From 83f774ab37c64dd43f29b7e972975522d55621fe Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 02:56:57 +0900 Subject: [PATCH 001/147] =?UTF-8?q?Traefik=20ingress=20=EC=95=8C=EB=A6=BC?= =?UTF-8?q?=20label=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/prometheus-rules.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 5a8c8fb7..838a009e 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -28,7 +28,7 @@ spec: summary: "Backend API 5xx rate is high" description: "Backend API 5xx responses are above 5% for 2 minutes." - alert: Ingress5xxSpike - expr: sum(rate(traefik_service_requests_total{service=~"code-place-dev-.*", code=~"5.."}[2m])) / clamp_min(sum(rate(traefik_service_requests_total{service=~"code-place-dev-.*"}[2m])), 1) > 0.05 + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-dev-.*", code=~"5.."}[2m])) / clamp_min(sum(rate(traefik_service_requests_total{exported_service=~"code-place-dev-.*"}[2m])), 1) > 0.05 for: 2m labels: namespace: code-place-dev @@ -38,7 +38,7 @@ spec: summary: "Ingress 5xx rate is high in dev" description: "Traefik 5xx responses for code-place-dev are above 5% for 2 minutes." - alert: Ingress5xxSpike - expr: sum(rate(traefik_service_requests_total{service=~"code-place-prod-.*", code=~"5.."}[2m])) / clamp_min(sum(rate(traefik_service_requests_total{service=~"code-place-prod-.*"}[2m])), 1) > 0.05 + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-prod-.*", code=~"5.."}[2m])) / clamp_min(sum(rate(traefik_service_requests_total{exported_service=~"code-place-prod-.*"}[2m])), 1) > 0.05 for: 2m labels: namespace: code-place-prod From d5997d0c045d8cfd55a79abca371a961afa697af Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 03:20:00 +0900 Subject: [PATCH 002/147] =?UTF-8?q?Grafana=20=EB=8C=80=EC=8B=9C=EB=B3=B4?= =?UTF-8?q?=EB=93=9C=20=EC=9A=B4=EC=98=81=20=EC=A7=80=ED=91=9C=20=EC=A0=95?= =?UTF-8?q?=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grafana-dashboard-codeplace.yaml | 175 ++++++++++++++++-- 1 file changed, 157 insertions(+), 18 deletions(-) diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 43e73b13..682634b1 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -13,6 +13,7 @@ data: "version": 1, "refresh": "30s", "tags": ["codeplace"], + "editable": true, "time": { "from": "now-1h", "to": "now" @@ -37,51 +38,189 @@ data: }, "panels": [ { - "type": "timeseries", - "title": "Backend Request Rate", - "gridPos": {"x": 0, "y": 0, "w": 8, "h": 8}, + "type": "stat", + "title": "Backend Up", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} requests/s"} + {"expr": "min by (namespace) (up{job=\"backend\", namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}"} ] }, { - "type": "timeseries", - "title": "Backend 5xx Rate", - "gridPos": {"x": 8, "y": 0, "w": 8, "h": 8}, + "type": "stat", + "title": "Request Rate", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "fieldConfig": {"defaults": {"unit": "reqps"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "5xx Ratio", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.01}, + {"color": "red", "value": 0.05} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "((sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\", status_code=~\"5..\"}[5m])) or (sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m])) * 0)) / clamp_min(sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m])), 0.001)", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "p95 Latency", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 2} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\", status_code=~\"5..\"}[5m]))", "legendFormat": "{{namespace}} 5xx/s"} + {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "Judge Waiting Queue", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 3}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "Judge Available", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum by (namespace) (codeplace_judge_server_available{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}"} ] }, { "type": "timeseries", - "title": "Backend p95 Latency", - "gridPos": {"x": 16, "y": 0, "w": 8, "h": 8}, + "title": "Backend Requests by Status", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "fieldConfig": {"defaults": {"unit": "reqps"}}, "targets": [ - {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p95"} + {"expr": "sum by (namespace, status_code) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{status_code}}"} ] }, { "type": "timeseries", + "title": "Backend Latency", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p95"}, + {"expr": "histogram_quantile(0.99, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p99"} + ] + }, + { + "type": "bargauge", "title": "Submission Status", - "gridPos": {"x": 0, "y": 8, "w": 8, "h": 8}, + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ {"expr": "codeplace_submission_status_count{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{status}}"} ] }, { "type": "timeseries", - "title": "Judge Waiting Queue", - "gridPos": {"x": 8, "y": 8, "w": 8, "h": 8}, + "title": "Judge Duration p95", + "gridPos": {"x": 0, "y": 12, "w": 8, "h": 8}, + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_submission_judge_duration_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} judge p95"} + ] + }, + { + "type": "timeseries", + "title": "Judge Heartbeat Max Age", + "gridPos": {"x": 8, "y": 12, "w": 8, "h": 8}, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 120}, + {"color": "red", "value": 180} + ] + } + } + }, "targets": [ - {"expr": "codeplace_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} waiting"} + {"expr": "max by (namespace) (codeplace_judge_server_last_heartbeat_age_seconds{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} max age"} ] }, { "type": "timeseries", - "title": "Judge Heartbeat Age", - "gridPos": {"x": 16, "y": 8, "w": 8, "h": 8}, + "title": "Judge Server Load", + "gridPos": {"x": 16, "y": 12, "w": 8, "h": 8}, + "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "codeplace_judge_server_last_heartbeat_age_seconds{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{hostname}}"} + {"expr": "topk(10, codeplace_judge_server_task_number{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{hostname}} tasks"} ] } ] From a61eb8c39acc9f2ec925ae6ec9c312dd084368fc Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 03:37:50 +0900 Subject: [PATCH 003/147] =?UTF-8?q?=ED=94=8C=EB=9E=AB=ED=8F=BC=EA=B3=BC=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EC=8A=A4=ED=86=A0=EC=96=B4=20?= =?UTF-8?q?=EA=B4=80=EC=B8=A1=20=EC=A7=80=ED=91=9C=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 7 +- .../grafana-dashboard-codeplace.yaml | 152 ++++++++++++++++++ kubernetes/monitoring/prometheus-rules.yaml | 38 ++++- 3 files changed, 195 insertions(+), 2 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index b8d0cf39..13068e47 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -111,7 +111,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. - `CeleryBeatDown`: beat Pod not ready 2분 지속. - `PodCrashLooping`: 주요 Pod restart 증가 5분 지속. +- `CodePlacePodNotReady`: 주요 앱/DB/Redis Pod not ready 5분 지속. +- `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. +- `CodePlaceContainerMemoryHigh`: 주요 컨테이너 memory limit 사용률 85% 초과 10분 지속. - `PVCAlmostFull`: PVC 사용률 85% 초과 10분 지속. +- `CodePlaceNodePressure`: node memory/disk/PID pressure 5분 지속. ## 5. 운영 확인 절차 @@ -123,7 +127,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 4. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` 5. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. -6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat panel을 확인합니다. +6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. 7. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. @@ -133,6 +137,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. - OpenTelemetry는 `OTEL_ENABLED=0` 기본값을 유지하고 dev/staging에서 먼저 활성화합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. +- PostgreSQL/Redis 세부 지표는 별도 exporter 없이 kube-state-metrics/kubelet 기반 readiness, restart, PVC 관측부터 제공합니다. connection, lock, Redis memory/client 같은 상세 지표는 exporter 도입 이후 확장합니다. ## 6. 검증 diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 682634b1..b915d49c 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -222,6 +222,158 @@ data: "targets": [ {"expr": "topk(10, codeplace_judge_server_task_number{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{hostname}} tasks"} ] + }, + { + "type": "row", + "title": "Platform", + "gridPos": {"x": 0, "y": 20, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "Pod Ready", + "gridPos": {"x": 0, "y": 21, "w": 6, "h": 5}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace, pod) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"})", "legendFormat": "{{namespace}} {{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Pod Restarts", + "gridPos": {"x": 6, "y": 21, "w": 6, "h": 5}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Container CPU Usage", + "gridPos": {"x": 12, "y": 21, "w": 6, "h": 5}, + "fieldConfig": {"defaults": {"unit": "cores"}}, + "targets": [ + {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + ] + }, + { + "type": "timeseries", + "title": "Container Memory Usage", + "gridPos": {"x": 18, "y": 21, "w": 6, "h": 5}, + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + ] + }, + { + "type": "timeseries", + "title": "PVC Usage", + "gridPos": {"x": 0, "y": 26, "w": 12, "h": 6}, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}), 1)", "legendFormat": "{{namespace}} {{persistentvolumeclaim}}"} + ] + }, + { + "type": "timeseries", + "title": "Node Pressure", + "gridPos": {"x": 12, "y": 26, "w": 12, "h": 6}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "kube_node_status_condition{condition=~\"MemoryPressure|DiskPressure|PIDPressure\", status=\"true\"}", "legendFormat": "{{node}} {{condition}}"} + ] + }, + { + "type": "row", + "title": "Data Stores", + "gridPos": {"x": 0, "y": 32, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "PostgreSQL Ready", + "gridPos": {"x": 0, "y": 33, "w": 6, "h": 5}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"postgres-[0-9]+\"})", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "Redis Ready", + "gridPos": {"x": 6, "y": 33, "w": 6, "h": 5}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"redis-.*|redis-replication-.*\"})", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL Restarts", + "gridPos": {"x": 12, "y": 33, "w": 6, "h": 5}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Redis Restarts", + "gridPos": {"x": 18, "y": 33, "w": 6, "h": 5}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + ] } ] } diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 838a009e..06183612 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -129,8 +129,35 @@ spec: annotations: summary: "CodePlace pod is restarting" description: "A CodePlace application pod has restarted within the last 5 minutes." + - alert: CodePlacePodNotReady + expr: max by (namespace, pod) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}) == 0 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace pod is not ready" + description: "A CodePlace application or datastore pod has not been ready for 5 minutes." + - alert: CodePlaceContainerCPUHigh + expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}), 0.001) > 0.8 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace container CPU usage is high" + description: "A CodePlace container has used more than 80% of its CPU limit for 10 minutes." + - alert: CodePlaceContainerMemoryHigh + expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="memory", unit="byte", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}), 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace container memory usage is high" + description: "A CodePlace container has used more than 85% of its memory limit for 10 minutes." - alert: PVCAlmostFull - expr: kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"} / kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"} > 0.85 + expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 for: 10m labels: severity: warning @@ -138,3 +165,12 @@ spec: annotations: summary: "PVC usage is above 85%" description: "A persistent volume claim has been above 85% usage for 10 minutes." + - alert: CodePlaceNodePressure + expr: kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure", status="true"} == 1 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Kubernetes node is under pressure" + description: "A Kubernetes node has reported memory, disk, or PID pressure for 5 minutes." From 38eb41eba7eab1e67291784b903053c671b8902e Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 03:39:47 +0900 Subject: [PATCH 004/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=95=8C?= =?UTF-8?q?=EB=A6=BC=20=EB=8C=80=EC=9D=91=20runbook=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-runbook/_index.md | 298 ++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 docs/content/developer/infra/observability-runbook/_index.md diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md new file mode 100644 index 00000000..ce42236f --- /dev/null +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -0,0 +1,298 @@ +--- +date: 2026-06-19T00:00:00+09:00 +draft: false +title: "Observability Runbook" +weight: 6 +--- + +{{< callout >}} +이 문서는 Kubernetes 운영 환경에서 CodePlace 알림을 받았을 때 원인 후보를 빠르게 좁히기 위한 1차 runbook입니다. Docker Swarm monitoring은 레거시로 유지합니다. +{{< /callout >}} + +## 공통 확인 + +알림을 받으면 먼저 scope를 확인합니다. + +```sh +kubectl get ns +kubectl -n monitoring get prometheusrule,alertmanagerconfig,servicemonitor +kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana' +``` + +Grafana에서는 `CodePlace Overview` dashboard에서 `namespace`를 알림의 namespace로 맞춥니다. + +Prometheus target 확인: + +```text +Status -> Targets -> backend +``` + +Alertmanager 확인: + +```text +Alerts -> alertname / namespace / priority +Status -> Configuration -> p0-discord / p1-discord +``` + +## P0 + +### BackendTargetDown + +증상: Prometheus가 backend `/metrics`를 1분 이상 scrape하지 못합니다. + +확인: + +```sh +kubectl -n get pod,svc,endpoints backend +kubectl -n describe pod -l app=backend +kubectl -n logs deploy/backend --tail=100 +kubectl -n monitoring get servicemonitor backend -o yaml +``` + +판단: + +- backend Pod가 없거나 not ready이면 배포/리소스 문제입니다. +- Service endpoint가 비어 있으면 label selector 또는 readiness 문제입니다. +- backend는 떠 있는데 `/metrics`만 실패하면 app 설정 또는 ServiceMonitor port/path 문제입니다. + +### Api5xxSpike + +증상: backend 5xx 비율이 2분 동안 5%를 넘습니다. + +확인: + +```promql +sum by (namespace, status_code) (rate(codeplace_http_requests_total{namespace=""}[5m])) +histogram_quantile(0.95, sum by (le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=""}[5m]))) +``` + +```sh +kubectl -n logs deploy/backend --tail=200 +kubectl -n get pod -l app=backend +kubectl -n describe pod -l app=backend +``` + +판단: + +- 5xx와 latency가 같이 오르면 backend/DB/Redis 병목 가능성이 큽니다. +- 5xx만 증가하면 최근 배포, exception, 외부 dependency를 우선 확인합니다. + +### Ingress5xxSpike + +증상: Traefik service 기준 5xx 비율이 2분 동안 5%를 넘습니다. + +확인: + +```promql +sum by (exported_service, code) (rate(traefik_service_requests_total{exported_service=~"-.*"}[5m])) +``` + +```sh +kubectl -n get ingress,svc,endpoints +kubectl -n kube-system logs deploy/traefik --tail=200 +``` + +판단: + +- backend 5xx도 같이 증가하면 app/backend 문제입니다. +- ingress 5xx만 증가하면 Traefik route, Service endpoint, upstream connection 문제를 의심합니다. + +### JudgeAllUnavailable + +증상: 사용 가능한 judge-server가 0개입니다. + +확인: + +```promql +codeplace_judge_server_available{namespace=""} +codeplace_judge_server_last_heartbeat_age_seconds{namespace=""} +codeplace_waiting_queue_length{namespace=""} +``` + +```sh +kubectl -n get pod -l app=judge-server +kubectl -n logs deploy/judge-server --tail=200 +kubectl -n logs deploy/celery-worker --tail=200 +``` + +판단: + +- judge Pod가 not ready이면 platform/deploy 문제입니다. +- Pod는 ready인데 heartbeat가 stale이면 judge-server와 backend 통신 또는 token 문제입니다. +- waiting queue가 같이 증가하면 사용자 제출 영향이 있습니다. + +### JudgeHeartbeatCritical + +증상: enabled judge-server heartbeat age가 180초를 넘습니다. + +확인: + +```promql +max by (namespace) (codeplace_judge_server_last_heartbeat_age_seconds{namespace=""}) +topk(10, codeplace_judge_server_task_number{namespace=""}) +``` + +```sh +kubectl -n get pod -l app=judge-server -o wide +kubectl -n describe pod -l app=judge-server +``` + +판단: + +- 특정 judge만 stale이면 해당 Pod/Node 문제입니다. +- 전체 judge가 stale이면 backend DB row, heartbeat endpoint, token 설정을 확인합니다. + +### PostgresUnavailable + +증상: PostgreSQL Pod가 1분 이상 ready가 아닙니다. + +확인: + +```sh +kubectl -n get pod -l cnpg.io/cluster=postgres +kubectl -n describe cluster postgres +kubectl -n describe pod -l cnpg.io/cluster=postgres +``` + +판단: + +- Pod scheduling 실패는 node/storage 문제입니다. +- Pod restart나 readiness 실패는 CNPG 상태와 PostgreSQL 로그를 확인합니다. + +### RedisUnavailable + +증상: Redis 또는 Sentinel Pod가 1분 이상 ready가 아닙니다. + +확인: + +```sh +kubectl -n get pod | grep redis +kubectl -n describe redissentinel redis +kubectl -n describe redisreplication redis-replication +kubectl -n logs -l app=redis-sentinel --tail=100 +``` + +판단: + +- Redis replication Pod 문제면 app cache/queue 영향이 큽니다. +- Sentinel 문제면 failover와 master discovery 영향이 큽니다. + +## P1 + +### ApiLatencyHigh + +확인: + +```promql +histogram_quantile(0.95, sum by (le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=""}[5m]))) +sum by (namespace) (codeplace_waiting_queue_length{namespace=""}) +``` + +backend latency와 DB/Redis readiness, CPU/memory를 같이 봅니다. + +### JudgeWaitingQueueBacklog + +확인: + +```promql +codeplace_waiting_queue_length{namespace=""} +sum by (namespace) (codeplace_judge_server_available{namespace=""}) +``` + +queue 증가와 judge available 감소가 같이 보이면 judge-server 문제입니다. judge는 정상인데 queue만 증가하면 celery-worker 처리량을 봅니다. + +### CeleryWorkerRestarting / CeleryBeatDown + +확인: + +```sh +kubectl -n get pod -l app=celery-worker +kubectl -n get pod -l app=celery-beat +kubectl -n logs deploy/celery-worker --tail=200 +kubectl -n logs deploy/celery-beat --tail=200 +``` + +### PodCrashLooping / CodePlacePodNotReady + +확인: + +```sh +kubectl -n get pod +kubectl -n describe pod +kubectl -n logs --previous --tail=200 +``` + +이미지 pull, secret mount, readiness, resource limit, node scheduling을 확인합니다. + +### CodePlaceContainerCPUHigh / CodePlaceContainerMemoryHigh + +확인: + +```promql +topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace="", container!=""}[5m]))) +topk(10, container_memory_working_set_bytes{namespace="", container!=""}) +``` + +CPU/memory가 지속적으로 높으면 replica 조정, limit 조정, 최근 배포 변경을 확인합니다. + +### PVCAlmostFull + +확인: + +```promql +max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=""}) / +max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=""}) +``` + +```sh +kubectl -n get pvc +kubectl -n describe pvc +``` + +### CodePlaceNodePressure + +확인: + +```sh +kubectl get node +kubectl describe node +kubectl top node +kubectl top pod -A --sort-by=memory +``` + +node pressure는 앱 문제가 아니라 cluster resource 문제일 가능성이 큽니다. + +## Test Alert + +Discord webhook 자체는 curl로 확인할 수 있습니다. Alertmanager routing은 임시 `PrometheusRule`로 확인합니다. + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: codeplace-test-alert + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + groups: + - name: codeplace.test + interval: 15s + rules: + - alert: CodePlaceTestAlert + expr: vector(1) + for: 30s + labels: + severity: critical + priority: P0 + namespace: code-place-dev + annotations: + summary: "CodePlace test alert" + description: "Discord alert routing test" +``` + +테스트 후 삭제합니다. + +```sh +kubectl -n monitoring delete prometheusrule codeplace-test-alert +``` From 203b580836aea04d473e3748f1b4c21907b0a3f4 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 03:43:11 +0900 Subject: [PATCH 005/147] =?UTF-8?q?Sentry=20=EA=B8=B0=EB=B3=B8=20=EA=B0=9C?= =?UTF-8?q?=EC=9D=B8=EC=A0=95=EB=B3=B4=20=EC=A0=84=EC=86=A1=20=EB=B9=84?= =?UTF-8?q?=ED=99=9C=EC=84=B1=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/oj/settings.py | 2 +- .../developer/infra/observability-improvement-plan/_index.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/oj/settings.py b/backend/oj/settings.py index 5bba42a0..b3c70235 100644 --- a/backend/oj/settings.py +++ b/backend/oj/settings.py @@ -29,7 +29,7 @@ if SENTRY_DSN: sentry_sdk.init( dsn=SENTRY_DSN, - send_default_pii=True, + send_default_pii=False, environment=get_env("SENTRY_ENVIRONMENT", get_env("OJ_ENV", "dev")), ) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 13068e47..9c05fadb 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -138,6 +138,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - OpenTelemetry는 `OTEL_ENABLED=0` 기본값을 유지하고 dev/staging에서 먼저 활성화합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. - PostgreSQL/Redis 세부 지표는 별도 exporter 없이 kube-state-metrics/kubelet 기반 readiness, restart, PVC 관측부터 제공합니다. connection, lock, Redis memory/client 같은 상세 지표는 exporter 도입 이후 확장합니다. +- Sentry backend SDK는 기본 PII 자동 전송을 비활성화합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. ## 6. 검증 From 179644f39f2863d61fe076eba5bcca428ea37afd Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 03:43:50 +0900 Subject: [PATCH 006/147] =?UTF-8?q?=ED=8A=B8=EB=A0=88=EC=9D=B4=EC=8B=B1=20?= =?UTF-8?q?=EB=8F=84=EC=9E=85=20=EC=A0=84=EC=A0=9C=20=EC=A1=B0=EA=B1=B4=20?= =?UTF-8?q?=EB=AC=B8=EC=84=9C=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 9c05fadb..ed546668 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -75,6 +75,16 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT 기본 sampling 값은 `OTEL_TRACES_SAMPLER_ARG=0.05`입니다. +현재 Kubernetes manifest에는 backend/celery의 `OTEL_EXPORTER_OTLP_ENDPOINT`가 `http://otel-collector.monitoring.svc.cluster.local:4317`로 잡혀 있습니다. 다만 `otel-collector`와 Tempo는 아직 배포 리소스로 포함하지 않습니다. 다음 조건이 정해지기 전에는 `OTEL_ENABLED=1`을 켜지 않습니다. + +- trace 저장 backend: Tempo 단일 binary, Tempo distributed, 외부 managed backend 중 선택. +- retention: dev/prod별 trace 보관 기간과 저장 용량. +- storage: filesystem, S3 호환 object storage, Longhorn PVC 중 선택. +- sampling: dev/prod sampling ratio와 error trace 우선 보존 정책. +- Grafana datasource: Tempo datasource 이름과 dashboard trace link 규칙. + +이 조건이 정해진 뒤 `otel-collector`와 Tempo를 `kubernetes/monitoring` 하위 Helm values 또는 별도 kustomize 리소스로 추가합니다. + ## 3. Kubernetes Monitoring 신규 monitoring 리소스는 `kubernetes/monitoring` 아래에 두고, 애플리케이션 `dev/prod` overlay에 포함하지 않습니다. @@ -139,6 +149,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. - PostgreSQL/Redis 세부 지표는 별도 exporter 없이 kube-state-metrics/kubelet 기반 readiness, restart, PVC 관측부터 제공합니다. connection, lock, Redis memory/client 같은 상세 지표는 exporter 도입 이후 확장합니다. - Sentry backend SDK는 기본 PII 자동 전송을 비활성화합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. +- Loki/Alloy 로그 수집과 Tempo tracing은 저장소/retention 결정을 먼저 요구하므로 현재 PR에서는 켜지 않습니다. ## 6. 검증 From 0c062cd6c086404d43cbe5ed70effa4547e02ecf Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 03:53:55 +0900 Subject: [PATCH 007/147] =?UTF-8?q?Loki=EC=99=80=20Alloy=20=EB=A1=9C?= =?UTF-8?q?=EA=B7=B8=20=EC=88=98=EC=A7=91=20=EC=84=A4=EC=A0=95=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 32 ++++- .../infra/observability-runbook/_index.md | 37 +++++- .../kube-prometheus-stack-values.yaml | 8 ++ kubernetes/monitoring/logs/README.md | 49 ++++++++ kubernetes/monitoring/logs/alloy-values.yaml | 117 ++++++++++++++++++ kubernetes/monitoring/logs/loki-values.yaml | 109 ++++++++++++++++ 6 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 kubernetes/monitoring/logs/README.md create mode 100644 kubernetes/monitoring/logs/alloy-values.yaml create mode 100644 kubernetes/monitoring/logs/loki-values.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index ed546668..bb48d28f 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -63,6 +63,17 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. +Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 운영 조건에서는 Loki를 Monolithic mode로 배포하고 Longhorn PVC 기반 filesystem storage를 사용합니다. + +- Loki release: `loki`, namespace: `monitoring`. +- Alloy release: `alloy`, namespace: `monitoring`. +- Loki storage: `filesystem` on Longhorn PVC. +- Loki retention: `code-place-dev` 3일, `code-place-prod` 7일. +- 수집 namespace: `code-place-dev`, `code-place-prod`, `monitoring`. +- Grafana datasource: 기존 kube-prometheus-stack Grafana에 `Loki` datasource를 추가합니다. + +이 구성은 클라우드 없는 단기 retention 기준선입니다. prod 로그량이 커지거나 로그 저장소 장애 도메인을 더 분리해야 하면, Loki는 내부에서 별도 운영하는 S3 호환 object storage, 예를 들어 독립 MinIO cluster, 로 이전합니다. Grafana Loki chart의 내장 MinIO subchart는 신규 운영 의존성으로 사용하지 않습니다. + ### Tracing OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OTEL_ENABLED=0`입니다. dev/staging에서 먼저 활성화하고 collector/Tempo/sampling 확인 후 prod 활성화를 결정합니다. @@ -94,6 +105,8 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. +- `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. +- `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. `alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. @@ -131,14 +144,18 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. 2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. -3. 애플리케이션은 환경별 overlay로 적용합니다. +3. Loki와 Alloy를 설치합니다. + - `helm upgrade --install loki grafana-community/loki --namespace monitoring --values kubernetes/monitoring/logs/loki-values.yaml` + - `helm upgrade --install alloy grafana/alloy --namespace monitoring --values kubernetes/monitoring/logs/alloy-values.yaml` +4. 애플리케이션은 환경별 overlay로 적용합니다. - dev: `kubectl apply -k kubernetes/overlays/dev` - prod: `kubectl apply -k kubernetes/overlays/prod` -4. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. +5. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` -5. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. -6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. -7. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +6. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. +7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. +8. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +9. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. @@ -147,9 +164,10 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. - OpenTelemetry는 `OTEL_ENABLED=0` 기본값을 유지하고 dev/staging에서 먼저 활성화합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. +- 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - PostgreSQL/Redis 세부 지표는 별도 exporter 없이 kube-state-metrics/kubelet 기반 readiness, restart, PVC 관측부터 제공합니다. connection, lock, Redis memory/client 같은 상세 지표는 exporter 도입 이후 확장합니다. - Sentry backend SDK는 기본 PII 자동 전송을 비활성화합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. -- Loki/Alloy 로그 수집과 Tempo tracing은 저장소/retention 결정을 먼저 요구하므로 현재 PR에서는 켜지 않습니다. +- Tempo tracing은 저장소/retention 결정을 먼저 요구하므로 현재 PR에서는 켜지 않습니다. ## 6. 검증 @@ -160,8 +178,10 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Metrics endpoint: Django test client 기준 `/metrics` 200 및 `codeplace_*` metric 포함 확인 - Kubernetes app render: `kubectl kustomize kubernetes/overlays/dev`, `kubectl kustomize kubernetes/overlays/prod` - Kubernetes monitoring render: `kubectl kustomize kubernetes/monitoring` +- Logs values YAML parse: `kubernetes/monitoring/logs/loki-values.yaml`, `kubernetes/monitoring/logs/alloy-values.yaml` `promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다. +`helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다. ### Discord Webhook Secret diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index ce42236f..5421edb7 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -16,7 +16,7 @@ weight: 6 ```sh kubectl get ns kubectl -n monitoring get prometheusrule,alertmanagerconfig,servicemonitor -kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana' +kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy' ``` Grafana에서는 `CodePlace Overview` dashboard에서 `namespace`를 알림의 namespace로 맞춥니다. @@ -34,6 +34,22 @@ Alerts -> alertname / namespace / priority Status -> Configuration -> p0-discord / p1-discord ``` +Loki 로그 확인: + +```logql +{namespace=""} +{namespace="", app="backend"} | json +{namespace="", container=~"backend|celery-worker|judge-server"} +``` + +Loki/Alloy 상태 확인: + +```sh +kubectl -n monitoring get pod,pvc | grep -E 'loki|alloy' +kubectl -n monitoring logs deploy/loki-gateway --tail=100 +kubectl -n monitoring logs daemonset/alloy --tail=100 +``` + ## P0 ### BackendTargetDown @@ -72,6 +88,13 @@ kubectl -n get pod -l app=backend kubectl -n describe pod -l app=backend ``` +Grafana Explore: + +```logql +{namespace="", app="backend"} | json | status_code >= 500 +{namespace="", app="backend"} |= "" +``` + 판단: - 5xx와 latency가 같이 오르면 backend/DB/Redis 병목 가능성이 큽니다. @@ -212,6 +235,12 @@ kubectl -n logs deploy/celery-worker --tail=200 kubectl -n logs deploy/celery-beat --tail=200 ``` +Grafana Explore: + +```logql +{namespace="", container=~"celery-worker|celery-beat"} +``` + ### PodCrashLooping / CodePlacePodNotReady 확인: @@ -222,6 +251,12 @@ kubectl -n describe pod kubectl -n logs --previous --tail=200 ``` +Grafana Explore: + +```logql +{namespace="", pod=""} +``` + 이미지 pull, secret mount, readiness, resource limit, node scheduling을 확인합니다. ### CodePlaceContainerCPUHigh / CodePlaceContainerMemoryHigh diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index e63df2ee..0be086b3 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -15,6 +15,14 @@ alertmanager: kubernetes.io/metadata.name: monitoring grafana: + additionalDataSources: + - name: Loki + type: loki + access: proxy + url: http://loki-gateway.monitoring.svc.cluster.local + isDefault: false + jsonData: + timeout: 60 ingress: enabled: true ingressClassName: traefik diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md new file mode 100644 index 00000000..0fb32be9 --- /dev/null +++ b/kubernetes/monitoring/logs/README.md @@ -0,0 +1,49 @@ +# CodePlace Logs Stack + +This directory contains Helm values for the Kubernetes logs stack. + +- Loki: monolithic mode, filesystem storage, Longhorn PVC. +- Alloy: DaemonSet log collector for `code-place-dev`, `code-place-prod`, and `monitoring`. +- Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. + +## Install or Upgrade + +Install or upgrade kube-prometheus-stack first so the Prometheus Operator CRDs are present and the Grafana `Loki` datasource is provisioned. Then install Loki and Alloy. + +```sh +helm repo add grafana-community https://grafana-community.github.io/helm-charts +helm repo add grafana https://grafana.github.io/helm-charts +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update + +helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ + --namespace monitoring \ + --values kubernetes/monitoring/kube-prometheus-stack-values.yaml + +helm upgrade --install loki grafana-community/loki \ + --namespace monitoring \ + --values kubernetes/monitoring/logs/loki-values.yaml + +helm upgrade --install alloy grafana/alloy \ + --namespace monitoring \ + --values kubernetes/monitoring/logs/alloy-values.yaml +``` + +## Verify + +```sh +kubectl -n monitoring get pod | grep -E 'loki|alloy|grafana' +kubectl -n monitoring get pvc | grep loki +``` + +Grafana should show a `Loki` datasource. Useful Explore queries: + +```logql +{namespace="code-place-dev"} +{namespace="code-place-prod", app="backend"} | json +{namespace="code-place-dev", container=~"backend|celery-worker|judge-server"} +``` + +## Notes + +This is not a horizontally scalable Loki design. It is the selected cloud-free baseline for short retention. If production log volume grows or logs must survive a node/PVC failure domain, move prod to an internally operated S3-compatible object store such as a separately managed MinIO cluster. diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml new file mode 100644 index 00000000..2f130085 --- /dev/null +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -0,0 +1,117 @@ +# Helm values for grafana/alloy. +# Alloy runs as a DaemonSet and collects logs from CodePlace and monitoring pods. + +controller: + type: daemonset + +alloy: + extraEnv: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + configMap: + create: true + content: |- + discovery.kubernetes "pod" { + role = "pod" + + selectors { + role = "pod" + field = "spec.nodeName=" + sys.env("NODE_NAME") + } + } + + discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pod.targets + + rule { + source_labels = ["__meta_kubernetes_namespace"] + regex = "code-place-dev|code-place-prod|monitoring" + action = "keep" + } + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + action = "replace" + target_label = "app" + } + + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = "(.+)" + action = "replace" + target_label = "app_kubernetes_io_name" + } + + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + action = "replace" + separator = "/" + target_label = "job" + } + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "__path__" + separator = "/" + replacement = "/var/log/pods/*$1/*.log" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_id"] + action = "replace" + target_label = "container_runtime" + regex = `^(\S+):\/\/.+$` + replacement = "$1" + } + } + + loki.source.kubernetes "pod_logs" { + targets = discovery.relabel.pod_logs.output + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.static_labels { + values = { + cluster = "codeplace" + } + } + + forward_to = [loki.write.default.receiver] + } + + loki.write "default" { + endpoint { + url = "http://loki-gateway.monitoring.svc.cluster.local/loki/api/v1/push" + } + } + + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi + +service: + serviceMonitor: + enabled: true + interval: 30s diff --git a/kubernetes/monitoring/logs/loki-values.yaml b/kubernetes/monitoring/logs/loki-values.yaml new file mode 100644 index 00000000..21cf4546 --- /dev/null +++ b/kubernetes/monitoring/logs/loki-values.yaml @@ -0,0 +1,109 @@ +# Helm values for grafana-community/loki. +# This is an on-premises, cloud-free Loki deployment. It intentionally uses +# Monolithic mode with a Longhorn-backed filesystem PVC instead of object storage. + +loki: + auth_enabled: false + commonConfig: + path_prefix: /var/loki + replication_factor: 1 + schemaConfig: + configs: + - from: "2026-06-19" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + storage: + type: filesystem + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + storage_config: + tsdb_shipper: + active_index_directory: /var/loki/tsdb-index + cache_location: /var/loki/tsdb-cache + filesystem: + directory: /var/loki/chunks + compactor: + working_directory: /var/loki/compactor + retention_enabled: true + delete_request_store: filesystem + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 168h + retention_stream: + - selector: '{namespace="code-place-dev"}' + priority: 1 + period: 72h + - selector: '{namespace="code-place-prod"}' + priority: 1 + period: 168h + ruler: + enable_api: true + +deploymentMode: Monolithic + +singleBinary: + replicas: 1 + persistence: + enabled: true + storageClass: longhorn + accessModes: + - ReadWriteOnce + size: 50Gi + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + memory: 2Gi + +gateway: + enabled: true + replicas: 1 + +lokiCanary: + enabled: true + +test: + enabled: true + +chunksCache: + enabled: false + +resultsCache: + enabled: false + +minio: + enabled: false + +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomPlanner: + replicas: 0 +bloomBuilder: + replicas: 0 +bloomGateway: + replicas: 0 From dfea20cb3103cd5eafd1d9376fe500afa3db6cb6 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 03:59:33 +0900 Subject: [PATCH 008/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=EC=8A=A4=ED=83=9D?= =?UTF-8?q?=20=EA=B4=80=EC=B8=A1=20=EC=95=8C=EB=A6=BC=EA=B3=BC=20=EB=8C=80?= =?UTF-8?q?=EC=8B=9C=EB=B3=B4=EB=93=9C=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 11 +- .../infra/observability-runbook/_index.md | 57 ++++++ .../monitoring/grafana-dashboard-logs.yaml | 175 ++++++++++++++++++ kubernetes/monitoring/kustomization.yaml | 1 + kubernetes/monitoring/logs/README.md | 6 + kubernetes/monitoring/logs/alloy-values.yaml | 9 +- kubernetes/monitoring/logs/loki-values.yaml | 8 + kubernetes/monitoring/prometheus-rules.yaml | 40 ++++ 8 files changed, 301 insertions(+), 6 deletions(-) create mode 100644 kubernetes/monitoring/grafana-dashboard-logs.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index bb48d28f..adec1fcd 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -107,6 +107,7 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. +- `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, 최근 backend error, judge/celery log 조회 dashboard. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. `alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. @@ -124,6 +125,7 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. - `PostgresUnavailable`: PostgreSQL Pod not ready 1분 지속. - `RedisUnavailable`: Redis Pod not ready 1분 지속. +- `LokiUnavailable`: Loki single-binary Pod not ready 1분 지속. ### P1 @@ -138,6 +140,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. - `CodePlaceContainerMemoryHigh`: 주요 컨테이너 memory limit 사용률 85% 초과 10분 지속. - `PVCAlmostFull`: PVC 사용률 85% 초과 10분 지속. +- `LokiPVCAlmostFull`: Loki Longhorn PVC 사용률 85% 초과 10분 지속. +- `AlloyDaemonSetUnavailable`: Alloy log collector가 모든 node에서 available하지 않음 5분 지속. +- `LokiGatewayUnavailable`: Loki gateway Pod not ready 2분 지속. - `CodePlaceNodePressure`: node memory/disk/PID pressure 5분 지속. ## 5. 운영 확인 절차 @@ -154,8 +159,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. -8. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -9. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, 최근 backend error, judge/celery log panel을 확인합니다. +9. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +10. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. @@ -179,6 +185,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Kubernetes app render: `kubectl kustomize kubernetes/overlays/dev`, `kubectl kustomize kubernetes/overlays/prod` - Kubernetes monitoring render: `kubectl kustomize kubernetes/monitoring` - Logs values YAML parse: `kubernetes/monitoring/logs/loki-values.yaml`, `kubernetes/monitoring/logs/alloy-values.yaml` +- Grafana dashboard JSON parse: `grafana-dashboard-codeplace.yaml`, `grafana-dashboard-logs.yaml` `promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다. `helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 5421edb7..c24ae8de 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -20,6 +20,7 @@ kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|al ``` Grafana에서는 `CodePlace Overview` dashboard에서 `namespace`를 알림의 namespace로 맞춥니다. +로그 수집이나 로그 기반 확인이 필요하면 `CodePlace Logs` dashboard를 함께 봅니다. Prometheus target 확인: @@ -200,6 +201,25 @@ kubectl -n logs -l app=redis-sentinel --tail=100 - Redis replication Pod 문제면 app cache/queue 영향이 큽니다. - Sentinel 문제면 failover와 master discovery 영향이 큽니다. +### LokiUnavailable + +증상: Loki single-binary Pod가 1분 이상 ready가 아닙니다. + +확인: + +```sh +kubectl -n monitoring get pod,pvc | grep loki +kubectl -n monitoring describe pod loki-0 +kubectl -n monitoring logs loki-0 --tail=200 +kubectl -n monitoring describe pvc -l app.kubernetes.io/name=loki +``` + +판단: + +- PVC attach/mount 실패면 Longhorn volume 상태를 먼저 봅니다. +- readiness만 실패하면 Loki config, retention, disk full, compactor error를 확인합니다. +- Loki가 내려가도 앱 요청 처리는 계속되지만 로그 수집/조회가 끊기므로 빠르게 복구합니다. + ## P1 ### ApiLatencyHigh @@ -284,6 +304,43 @@ kubectl -n get pvc kubectl -n describe pvc ``` +### LokiPVCAlmostFull + +확인: + +```promql +max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim=~".*loki.*"}) / +max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim=~".*loki.*"}) +``` + +```sh +kubectl -n monitoring get pvc | grep loki +kubectl -n monitoring describe pvc +``` + +조치: + +- dev/prod retention이 실제 기대대로 적용되는지 확인합니다. +- 당장 여유가 없으면 Longhorn PVC size를 증설합니다. +- 반복되면 retention 기간을 줄이거나 prod 로그 저장 방식을 내부 S3 호환 object storage로 분리합니다. + +### AlloyDaemonSetUnavailable / LokiGatewayUnavailable + +확인: + +```sh +kubectl -n monitoring get daemonset alloy +kubectl -n monitoring describe daemonset alloy +kubectl -n monitoring logs daemonset/alloy --tail=200 +kubectl -n monitoring get pod -l app.kubernetes.io/component=gateway +kubectl -n monitoring logs deploy/loki-gateway --tail=200 +``` + +판단: + +- Alloy가 특정 node에서만 비면 해당 node의 scheduling, taint, resource pressure를 봅니다. +- Loki gateway가 비면 Alloy write와 Grafana query가 실패할 수 있습니다. + ### CodePlaceNodePressure 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml new file mode 100644 index 00000000..c2ba5838 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -0,0 +1,175 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-logs + labels: + grafana_dashboard: "1" +data: + codeplace-logs.json: | + { + "uid": "codeplace-logs", + "title": "CodePlace Logs", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "logs"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "namespace", + "type": "custom", + "query": "code-place-dev,code-place-prod,monitoring", + "multi": true, + "includeAll": true, + "allValue": "code-place-dev|code-place-prod", + "current": { + "text": "code-place-dev", + "value": "code-place-dev" + } + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Loki Ready", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-[0-9]+\"})", "legendFormat": "loki"} + ] + }, + { + "type": "stat", + "title": "Alloy Nodes", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_daemonset_status_number_available{namespace=\"monitoring\", daemonset=\"alloy\"}", "legendFormat": "available"}, + {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"monitoring\", daemonset=\"alloy\"}", "legendFormat": "desired"} + ] + }, + { + "type": "stat", + "title": "Loki PVC Usage", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}), 1)", "legendFormat": "usage"} + ] + }, + { + "type": "stat", + "title": "Log Lines", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(count_over_time({namespace=~\"$namespace\"}[5m]))", "legendFormat": "lines"} + ] + }, + { + "type": "stat", + "title": "Backend Errors", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Loki", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"backend\"} | json | level=\"ERROR\" [5m]))", "legendFormat": "errors"} + ] + }, + { + "type": "logs", + "title": "Recent Backend Errors", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 10}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\", app=\"backend\"} | json | level=\"ERROR\""} + ] + }, + { + "type": "logs", + "title": "Judge and Celery Logs", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 10}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\", container=~\"celery-worker|celery-beat|judge-server\"}"} + ] + }, + { + "type": "logs", + "title": "Request ID Lookup", + "gridPos": {"x": 0, "y": 14, "w": 24, "h": 9}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\"} |= \"request_id\""} + ] + } + ] + } diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index 02e45234..bc79c927 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -6,4 +6,5 @@ resources: - ./alertmanager-config.yaml - ./backend-service-monitor.yaml - ./grafana-dashboard-codeplace.yaml + - ./grafana-dashboard-logs.yaml - ./prometheus-rules.yaml diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 0fb32be9..35f42ff8 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -5,6 +5,7 @@ This directory contains Helm values for the Kubernetes logs stack. - Loki: monolithic mode, filesystem storage, Longhorn PVC. - Alloy: DaemonSet log collector for `code-place-dev`, `code-place-prod`, and `monitoring`. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. +- Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. ## Install or Upgrade @@ -44,6 +45,11 @@ Grafana should show a `Loki` datasource. Useful Explore queries: {namespace="code-place-dev", container=~"backend|celery-worker|judge-server"} ``` +Grafana should also show: + +- `CodePlace Overview` +- `CodePlace Logs` + ## Notes This is not a horizontally scalable Loki design. It is the selected cloud-free baseline for short retention. If production log volume grows or logs must survive a node/PVC failure domain, move prod to an internally operated S3-compatible object store such as a separately managed MinIO cluster. diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml index 2f130085..8484a4fc 100644 --- a/kubernetes/monitoring/logs/alloy-values.yaml +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -111,7 +111,8 @@ alloy: limits: memory: 512Mi -service: - serviceMonitor: - enabled: true - interval: 30s +serviceMonitor: + enabled: true + additionalLabels: + release: kube-prometheus-stack + interval: 30s diff --git a/kubernetes/monitoring/logs/loki-values.yaml b/kubernetes/monitoring/logs/loki-values.yaml index 21cf4546..a98e4817 100644 --- a/kubernetes/monitoring/logs/loki-values.yaml +++ b/kubernetes/monitoring/logs/loki-values.yaml @@ -69,6 +69,14 @@ gateway: lokiCanary: enabled: true +monitoring: + serviceMonitor: + enabled: true + labels: + release: kube-prometheus-stack + interval: 15s + scrapeTimeout: 10s + test: enabled: true diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 06183612..60d34f11 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -82,6 +82,16 @@ spec: annotations: summary: "Redis pod is not ready" description: "At least one Redis pod has not been ready for 1 minute." + - alert: LokiUnavailable + expr: (min(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) + for: 1m + labels: + severity: critical + priority: P0 + namespace: monitoring + annotations: + summary: "Loki is not ready" + description: "The Loki single-binary pod has not been ready for 1 minute. Log ingestion and log queries may be unavailable." - name: codeplace.p1 interval: 30s rules: @@ -165,6 +175,36 @@ spec: annotations: summary: "PVC usage is above 85%" description: "A persistent volume claim has been above 85% usage for 10 minutes." + - alert: LokiPVCAlmostFull + expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim=~".*loki.*"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim=~".*loki.*"}), 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki PVC usage is above 85%" + description: "The Loki Longhorn PVC has been above 85% usage for 10 minutes. Log retention or PVC size must be adjusted before ingestion stops." + - alert: AlloyDaemonSetUnavailable + expr: (kube_daemonset_status_number_available{namespace="monitoring", daemonset="alloy"} < kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="alloy"}) or absent(kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="alloy"}) + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Alloy log collector is not running on every node" + description: "At least one node is missing an available Alloy DaemonSet pod for 5 minutes. Logs from that node may be missing." + - alert: LokiGatewayUnavailable + expr: (max by (namespace, pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki gateway is not ready" + description: "The Loki gateway has not been ready for 2 minutes. Alloy writes and Grafana Loki queries may fail." - alert: CodePlaceNodePressure expr: kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure", status="true"} == 1 for: 5m From 8e8be197e01e355626674c171f6b20eb8118ee23 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 04:01:51 +0900 Subject: [PATCH 009/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EB=A7=A4?= =?UTF-8?q?=EB=8B=88=ED=8E=98=EC=8A=A4=ED=8A=B8=20=EA=B2=80=EC=A6=9D=20?= =?UTF-8?q?=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 2 + .../infra/observability-runbook/_index.md | 1 + kubernetes/monitoring/logs/README.md | 8 ++ kubernetes/monitoring/validate.sh | 96 +++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100755 kubernetes/monitoring/validate.sh diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index adec1fcd..a3cf3a51 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -108,6 +108,7 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT - `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. - `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, 최근 backend error, judge/celery log 조회 dashboard. +- `validate.sh`: monitoring YAML, Grafana dashboard JSON, kustomize render, 선택적 promtool/Helm render 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. `alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. @@ -186,6 +187,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Kubernetes monitoring render: `kubectl kustomize kubernetes/monitoring` - Logs values YAML parse: `kubernetes/monitoring/logs/loki-values.yaml`, `kubernetes/monitoring/logs/alloy-values.yaml` - Grafana dashboard JSON parse: `grafana-dashboard-codeplace.yaml`, `grafana-dashboard-logs.yaml` +- Monitoring bundle validation: `bash kubernetes/monitoring/validate.sh` `promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다. `helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index c24ae8de..12a7f6fe 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -14,6 +14,7 @@ weight: 6 알림을 받으면 먼저 scope를 확인합니다. ```sh +bash kubernetes/monitoring/validate.sh kubectl get ns kubectl -n monitoring get prometheusrule,alertmanagerconfig,servicemonitor kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy' diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 35f42ff8..37c3df7b 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -32,6 +32,14 @@ helm upgrade --install alloy grafana/alloy \ ## Verify +Run the repository validation first: + +```sh +bash kubernetes/monitoring/validate.sh +``` + +If `helm` is installed and the chart repos are already added, the script also renders kube-prometheus-stack, Loki, and Alloy. Without `helm`, it still validates YAML, Grafana dashboard JSON, and kustomize output. + ```sh kubectl -n monitoring get pod | grep -E 'loki|alloy|grafana' kubectl -n monitoring get pvc | grep loki diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh new file mode 100755 index 00000000..682fe072 --- /dev/null +++ b/kubernetes/monitoring/validate.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MONITORING_DIR="${ROOT_DIR}/kubernetes/monitoring" + +require_cmd() { + local cmd="$1" + if ! command -v "${cmd}" >/dev/null 2>&1; then + echo "missing required command: ${cmd}" >&2 + exit 1 + fi +} + +optional_cmd() { + command -v "$1" >/dev/null 2>&1 +} + +require_cmd python3 +require_cmd kubectl + +echo "==> validating monitoring YAML and Grafana dashboard JSON" +python3 - <<'PY' +from pathlib import Path +import json +import sys + +try: + import yaml +except ImportError: + yaml = None + +root = Path("kubernetes/monitoring") +yaml_paths = [ + root / "kube-prometheus-stack-values.yaml", + root / "prometheus-rules.yaml", + root / "alertmanager-config.yaml", + root / "backend-service-monitor.yaml", + root / "logs" / "loki-values.yaml", + root / "logs" / "alloy-values.yaml", + root / "grafana-dashboard-codeplace.yaml", + root / "grafana-dashboard-logs.yaml", +] + +for path in yaml_paths: + text = path.read_text() + if yaml is None: + if path.name.startswith("grafana-dashboard-"): + print(f"missing Python module yaml; cannot extract dashboard JSON from {path}", file=sys.stderr) + sys.exit(1) + print(f"SKIP YAML parse {path}: Python module yaml not found") + continue + + data = yaml.safe_load(text) + print(f"YAML OK {path}") + if path.name.startswith("grafana-dashboard-"): + for key, value in data.get("data", {}).items(): + json.loads(value) + print(f"JSON OK {path}:{key}") +PY + +echo "==> rendering kustomize monitoring manifests" +kubectl kustomize "${MONITORING_DIR}" >/tmp/codeplace-monitoring-render.yaml +echo "KUSTOMIZE OK kubernetes/monitoring" + +if optional_cmd promtool; then + echo "==> checking Prometheus rules with promtool" + promtool check rules "${MONITORING_DIR}/prometheus-rules.yaml" +else + echo "SKIP promtool check: promtool not found" +fi + +if optional_cmd helm; then + echo "==> rendering Helm charts" + helm template kube-prometheus-stack prometheus-community/kube-prometheus-stack \ + --namespace monitoring \ + --values "${MONITORING_DIR}/kube-prometheus-stack-values.yaml" \ + >/tmp/codeplace-kube-prometheus-stack-render.yaml + echo "HELM OK kube-prometheus-stack" + + helm template loki grafana-community/loki \ + --namespace monitoring \ + --values "${MONITORING_DIR}/logs/loki-values.yaml" \ + >/tmp/codeplace-loki-render.yaml + echo "HELM OK loki" + + helm template alloy grafana/alloy \ + --namespace monitoring \ + --values "${MONITORING_DIR}/logs/alloy-values.yaml" \ + >/tmp/codeplace-alloy-render.yaml + echo "HELM OK alloy" +else + echo "SKIP helm template: helm not found" +fi + +echo "observability validation completed" From 6fa61b57edf1d8199643306270f7e26d7dd6f2c6 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 04:08:16 +0900 Subject: [PATCH 010/147] =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EC=8A=A4?= =?UTF-8?q?=ED=86=A0=EC=96=B4=20=EC=84=B8=EB=B6=80=20=EA=B4=80=EC=B8=A1=20?= =?UTF-8?q?=EC=A7=80=ED=91=9C=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 7 +- .../infra/observability-runbook/_index.md | 42 ++++++++++++ kubernetes/base/postgres/README.md | 12 ++++ kubernetes/base/redis/README.md | 3 + kubernetes/base/redis/redis.yaml | 24 +++++++ .../monitoring/datastore-pod-monitors.yaml | 41 ++++++++++++ .../grafana-dashboard-codeplace.yaml | 64 +++++++++++++++++- .../kube-prometheus-stack-values.yaml | 1 + kubernetes/monitoring/kustomization.yaml | 1 + kubernetes/monitoring/prometheus-rules.yaml | 67 +++++++++++++++++++ kubernetes/monitoring/validate.sh | 4 +- 11 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 kubernetes/monitoring/datastore-pod-monitors.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index a3cf3a51..ea0a5659 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -126,6 +126,8 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. - `PostgresUnavailable`: PostgreSQL Pod not ready 1분 지속. - `RedisUnavailable`: Redis Pod not ready 1분 지속. +- `PostgresExporterDown`: CNPG PostgreSQL exporter metric 수집 실패 2분 지속. +- `RedisExporterDown`: Redis exporter metric 수집 실패 2분 지속. - `LokiUnavailable`: Loki single-binary Pod not ready 1분 지속. ### P1 @@ -140,6 +142,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `CodePlacePodNotReady`: 주요 앱/DB/Redis Pod not ready 5분 지속. - `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. - `CodePlaceContainerMemoryHigh`: 주요 컨테이너 memory limit 사용률 85% 초과 10분 지속. +- `PostgresCollectorError`: CNPG PostgreSQL metrics collection error 5분 지속. +- `PostgresHADegraded`: PostgreSQL instance가 3개 node에 분산되지 않음 10분 지속. +- `RedisMemoryHigh`: Redis memory 사용률 85% 초과 10분 지속. - `PVCAlmostFull`: PVC 사용률 85% 초과 10분 지속. - `LokiPVCAlmostFull`: Loki Longhorn PVC 사용률 85% 초과 10분 지속. - `AlloyDaemonSetUnavailable`: Alloy log collector가 모든 node에서 available하지 않음 5분 지속. @@ -172,7 +177,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - OpenTelemetry는 `OTEL_ENABLED=0` 기본값을 유지하고 dev/staging에서 먼저 활성화합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. -- PostgreSQL/Redis 세부 지표는 별도 exporter 없이 kube-state-metrics/kubelet 기반 readiness, restart, PVC 관측부터 제공합니다. connection, lock, Redis memory/client 같은 상세 지표는 exporter 도입 이후 확장합니다. +- PostgreSQL은 CNPG instance exporter와 PodMonitor로 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor로 `redis_*` 지표를 수집합니다. - Sentry backend SDK는 기본 PII 자동 전송을 비활성화합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. - Tempo tracing은 저장소/retention 결정을 먼저 요구하므로 현재 PR에서는 켜지 않습니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 12a7f6fe..e3c09dbf 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -27,6 +27,8 @@ Prometheus target 확인: ```text Status -> Targets -> backend +Status -> Targets -> postgres +Status -> Targets -> redis ``` Alertmanager 확인: @@ -202,6 +204,29 @@ kubectl -n logs -l app=redis-sentinel --tail=100 - Redis replication Pod 문제면 app cache/queue 영향이 큽니다. - Sentinel 문제면 failover와 master discovery 영향이 큽니다. +### PostgresExporterDown / RedisExporterDown + +증상: DB/Redis Pod 자체가 ready여도 exporter metric scrape가 실패합니다. + +확인: + +```sh +kubectl -n monitoring get podmonitor postgres redis -o yaml +kubectl -n get pod -l cnpg.io/cluster=postgres +kubectl -n get pod | grep redis +``` + +```promql +cnpg_collector_up{namespace="", cluster="postgres"} +redis_up{namespace=""} +``` + +판단: + +- PostgreSQL readiness는 정상인데 `cnpg_collector_up`이 0이면 CNPG instance exporter 또는 metrics query 문제입니다. +- Redis Pod는 정상인데 `redis_up`이 0이면 exporter sidecar, Redis auth, Redis process 응답 문제입니다. +- PodMonitor target 자체가 없으면 Prometheus Operator selector 또는 Pod label/port 설정을 확인합니다. + ### LokiUnavailable 증상: Loki single-binary Pod가 1분 이상 ready가 아닙니다. @@ -291,6 +316,23 @@ topk(10, container_memory_working_set_bytes{namespace="", container!= CPU/memory가 지속적으로 높으면 replica 조정, limit 조정, 최근 배포 변경을 확인합니다. +### PostgresCollectorError / PostgresHADegraded / RedisMemoryHigh + +확인: + +```promql +cnpg_collector_last_collection_error{namespace="", cluster="postgres"} +cnpg_collector_nodes_used{namespace="", cluster="postgres"} +redis_memory_used_bytes{namespace=""} +redis_memory_max_bytes{namespace=""} +``` + +판단: + +- `PostgresCollectorError`는 metrics query 실패이므로 CNPG exporter 로그와 PostgreSQL 권한/상태를 확인합니다. +- `PostgresHADegraded`는 3개 instance가 서로 다른 node에 분산되지 않은 상태입니다. node 수, taint, Longhorn volume attach 상태를 확인합니다. +- `RedisMemoryHigh`는 Redis maxmemory 대비 사용량이 높은 상태입니다. eviction 정책, queue/backlog, cache key 증가를 확인합니다. + ### PVCAlmostFull 확인: diff --git a/kubernetes/base/postgres/README.md b/kubernetes/base/postgres/README.md index 358ecb11..454b2e74 100644 --- a/kubernetes/base/postgres/README.md +++ b/kubernetes/base/postgres/README.md @@ -5,6 +5,7 @@ ### 핵심 파일 - `postgres.yaml`: CNPG Operator를 통해 PostgreSQL 클러스터를 생성하기 위한 Custom Resource(CR) 정의 파일입니다. +- CNPG instance manager는 각 PostgreSQL Pod의 `metrics` 포트에서 Prometheus metric을 제공합니다. CodePlace monitoring은 별도 PodMonitor로 이 endpoint를 scrape합니다. --- @@ -60,3 +61,14 @@ kustomize build kubernetes/overlays/dev | kubectl apply -f - ``` > **관련 문서:** 더 자세한 정보는 [CloudNativePG 공식 문서](https://cloudnative-pg.io/documentation/current/)를 참고하세요. + +## 3. Monitoring + +PostgreSQL exporter scrape는 `kubernetes/monitoring/datastore-pod-monitors.yaml`에서 관리합니다. kube-prometheus-stack 설치 후 Prometheus Targets에서 `postgres` target이 healthy인지 확인합니다. + +주요 지표: + +- `cnpg_collector_up` +- `cnpg_collector_last_collection_error` +- `cnpg_collector_nodes_used` +- `cnpg_collector_wal_bytes` diff --git a/kubernetes/base/redis/README.md b/kubernetes/base/redis/README.md index cecacc15..aec37cdf 100644 --- a/kubernetes/base/redis/README.md +++ b/kubernetes/base/redis/README.md @@ -6,6 +6,7 @@ - `RedisReplication`: Redis 3개 Pod로 master/replica 구성을 만듭니다. - `RedisSentinel`: Sentinel 3개 Pod로 master 감시와 failover를 수행합니다. +- `redisExporter`: Redis/Sentinel Pod에 exporter sidecar를 붙여 Prometheus metric을 노출합니다. - 애플리케이션은 `redis-sentinel:26379`로 접속하고, master group 이름은 `mymaster`를 사용합니다. - `RedisSentinel` CR 이름은 `redis`입니다. Opstree Operator가 `-sentinel` Service를 생성하므로 실제 Sentinel Service 이름은 `redis-sentinel`입니다. @@ -57,3 +58,5 @@ REDIS_SENTINEL_HOSTS=redis-sentinel:26379 Redis와 Sentinel Pod는 `kubernetes.io/hostname` 기준으로 서로 다른 노드에 분산되도록 설정되어 있습니다. 이 구성은 최소 3개 이상의 스케줄 가능한 Kubernetes 노드를 전제로 합니다. 노드가 3개 미만이면 HA를 만족할 수 없으므로 Pod가 pending 상태로 남을 수 있습니다. Sentinel failover는 Redis master 장애 시 자동 승격을 수행하지만, failover 중에는 짧은 연결 재시도 구간이 생길 수 있습니다. 완전한 무중단에 가깝게 운영하려면 Longhorn replica와 Kubernetes 노드 장애 조건이 맞는지, 애플리케이션 retry 정책이 충분한지 함께 점검해야 합니다. + +Redis exporter는 `kubernetes/monitoring/datastore-pod-monitors.yaml`의 PodMonitor가 scrape합니다. kube-prometheus-stack 설치 후 Prometheus Targets에서 `redis` target이 healthy인지 확인합니다. diff --git a/kubernetes/base/redis/redis.yaml b/kubernetes/base/redis/redis.yaml index 0cb0f11b..3a562186 100644 --- a/kubernetes/base/redis/redis.yaml +++ b/kubernetes/base/redis/redis.yaml @@ -4,6 +4,18 @@ metadata: name: redis-replication spec: clusterSize: 3 + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:1.0 + imagePullPolicy: IfNotPresent + port: 9121 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi kubernetesConfig: image: quay.io/opstree/redis:v7.0.15 imagePullPolicy: IfNotPresent @@ -42,6 +54,18 @@ metadata: name: redis spec: clusterSize: 3 + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:1.0 + imagePullPolicy: IfNotPresent + port: 9121 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi pdb: enabled: true minAvailable: 2 diff --git a/kubernetes/monitoring/datastore-pod-monitors.yaml b/kubernetes/monitoring/datastore-pod-monitors.yaml new file mode 100644 index 00000000..911b5f9b --- /dev/null +++ b/kubernetes/monitoring/datastore-pod-monitors.yaml @@ -0,0 +1,41 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: postgres + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - code-place-dev + - code-place-prod + selector: + matchLabels: + cnpg.io/cluster: postgres + podMetricsEndpoints: + - port: metrics + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: redis + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - code-place-dev + - code-place-prod + selector: + matchExpressions: + - key: app + operator: In + values: + - redis-replication + - redis-sentinel + podMetricsEndpoints: + - portNumber: 9121 + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index b915d49c..622f09b5 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -357,10 +357,52 @@ data: {"expr": "min by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"redis-.*|redis-replication-.*\"})", "legendFormat": "{{namespace}}"} ] }, + { + "type": "stat", + "title": "PostgreSQL Exporter Up", + "gridPos": {"x": 12, "y": 33, "w": 6, "h": 5}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (cnpg_collector_up{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "Redis Exporter Up", + "gridPos": {"x": 18, "y": 33, "w": 6, "h": 5}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (redis_up{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}"} + ] + }, { "type": "timeseries", "title": "PostgreSQL Restarts", - "gridPos": {"x": 12, "y": 33, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 38, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -369,11 +411,29 @@ data: { "type": "timeseries", "title": "Redis Restarts", - "gridPos": {"x": 18, "y": 33, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 38, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} ] + }, + { + "type": "timeseries", + "title": "PostgreSQL WAL Bytes", + "gridPos": {"x": 12, "y": 38, "w": 6, "h": 5}, + "fieldConfig": {"defaults": {"unit": "Bps"}}, + "targets": [ + {"expr": "sum by (namespace) (rate(cnpg_collector_wal_bytes{namespace=~\"$namespace\", cluster=\"postgres\"}[5m]))", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "timeseries", + "title": "Redis Memory Usage", + "gridPos": {"x": 18, "y": 38, "w": 6, "h": 5}, + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}}"} + ] } ] } diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index 0be086b3..590eb083 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -3,6 +3,7 @@ prometheus: evaluationInterval: 15s scrapeInterval: 15s serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false ruleSelectorNilUsesHelmValues: false alertmanager: diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index bc79c927..cf1ad8d6 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -5,6 +5,7 @@ namespace: monitoring resources: - ./alertmanager-config.yaml - ./backend-service-monitor.yaml + - ./datastore-pod-monitors.yaml - ./grafana-dashboard-codeplace.yaml - ./grafana-dashboard-logs.yaml - ./prometheus-rules.yaml diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 60d34f11..4cc423c3 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -82,6 +82,46 @@ spec: annotations: summary: "Redis pod is not ready" description: "At least one Redis pod has not been ready for 1 minute." + - alert: PostgresExporterDown + expr: (min(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) == 0) or absent(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) + for: 2m + labels: + severity: critical + priority: P0 + namespace: code-place-dev + annotations: + summary: "PostgreSQL metrics exporter is down" + description: "Prometheus cannot collect CNPG PostgreSQL metrics for code-place-dev for 2 minutes." + - alert: PostgresExporterDown + expr: (min(cnpg_collector_up{namespace="code-place-prod", cluster="postgres"}) == 0) or absent(cnpg_collector_up{namespace="code-place-prod", cluster="postgres"}) + for: 2m + labels: + severity: critical + priority: P0 + namespace: code-place-prod + annotations: + summary: "PostgreSQL metrics exporter is down" + description: "Prometheus cannot collect CNPG PostgreSQL metrics for code-place-prod for 2 minutes." + - alert: RedisExporterDown + expr: (min(redis_up{namespace="code-place-dev"}) == 0) or absent(redis_up{namespace="code-place-dev"}) + for: 2m + labels: + severity: critical + priority: P0 + namespace: code-place-dev + annotations: + summary: "Redis metrics exporter is down" + description: "Prometheus cannot collect Redis exporter metrics for code-place-dev for 2 minutes." + - alert: RedisExporterDown + expr: (min(redis_up{namespace="code-place-prod"}) == 0) or absent(redis_up{namespace="code-place-prod"}) + for: 2m + labels: + severity: critical + priority: P0 + namespace: code-place-prod + annotations: + summary: "Redis metrics exporter is down" + description: "Prometheus cannot collect Redis exporter metrics for code-place-prod for 2 minutes." - alert: LokiUnavailable expr: (min(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) for: 1m @@ -166,6 +206,33 @@ spec: annotations: summary: "CodePlace container memory usage is high" description: "A CodePlace container has used more than 85% of its memory limit for 10 minutes." + - alert: PostgresCollectorError + expr: max by (namespace, pod) (cnpg_collector_last_collection_error{namespace=~"code-place-(dev|prod)", cluster="postgres"}) > 0 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL metrics collection is failing" + description: "CNPG PostgreSQL metrics collection has reported errors for 5 minutes." + - alert: PostgresHADegraded + expr: min by (namespace) (cnpg_collector_nodes_used{namespace=~"code-place-(dev|prod)", cluster="postgres"}) < 3 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL HA placement is degraded" + description: "CNPG reports fewer than 3 distinct nodes for PostgreSQL instances for 10 minutes." + - alert: RedisMemoryHigh + expr: max by (namespace, pod) (redis_memory_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "Redis memory usage is high" + description: "Redis exporter reports memory usage above 85% of configured max memory for 10 minutes." - alert: PVCAlmostFull expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 for: 10m diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 682fe072..77d91a93 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -36,6 +36,7 @@ yaml_paths = [ root / "prometheus-rules.yaml", root / "alertmanager-config.yaml", root / "backend-service-monitor.yaml", + root / "datastore-pod-monitors.yaml", root / "logs" / "loki-values.yaml", root / "logs" / "alloy-values.yaml", root / "grafana-dashboard-codeplace.yaml", @@ -51,9 +52,10 @@ for path in yaml_paths: print(f"SKIP YAML parse {path}: Python module yaml not found") continue - data = yaml.safe_load(text) + documents = list(yaml.safe_load_all(text)) print(f"YAML OK {path}") if path.name.startswith("grafana-dashboard-"): + data = documents[0] for key, value in data.get("data", {}).items(): json.loads(value) print(f"JSON OK {path}:{key}") From 999be2da21467919a701ec25627894ea67129af5 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 04:09:53 +0900 Subject: [PATCH 011/147] =?UTF-8?q?Redis=20=EB=A9=94=EB=AA=A8=EB=A6=AC=20?= =?UTF-8?q?=EC=95=8C=EB=A6=BC=20=EC=98=A4=ED=83=90=20=EB=B0=A9=EC=A7=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/developer/infra/observability-runbook/_index.md | 3 ++- kubernetes/monitoring/grafana-dashboard-codeplace.yaml | 3 ++- kubernetes/monitoring/prometheus-rules.yaml | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index e3c09dbf..e6d07903 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -331,7 +331,8 @@ redis_memory_max_bytes{namespace=""} - `PostgresCollectorError`는 metrics query 실패이므로 CNPG exporter 로그와 PostgreSQL 권한/상태를 확인합니다. - `PostgresHADegraded`는 3개 instance가 서로 다른 node에 분산되지 않은 상태입니다. node 수, taint, Longhorn volume attach 상태를 확인합니다. -- `RedisMemoryHigh`는 Redis maxmemory 대비 사용량이 높은 상태입니다. eviction 정책, queue/backlog, cache key 증가를 확인합니다. +- `RedisMemoryHigh`는 Redis `maxmemory`가 0보다 클 때만 동작합니다. `maxmemory=0`이면 Redis 자체 제한이 없으므로 container memory alert를 기준으로 보고, 제한을 둘지 별도로 결정합니다. +- Redis memory가 높으면 eviction 정책, queue/backlog, cache key 증가를 확인합니다. ### PVCAlmostFull diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 622f09b5..9b8302c1 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -432,7 +432,8 @@ data: "gridPos": {"x": 18, "y": 38, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ - {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}}"} + {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} used"}, + {"expr": "max by (namespace, pod) (redis_memory_max_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} maxmemory"} ] } ] diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 4cc423c3..d932ab67 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -225,14 +225,14 @@ spec: summary: "PostgreSQL HA placement is degraded" description: "CNPG reports fewer than 3 distinct nodes for PostgreSQL instances for 10 minutes." - alert: RedisMemoryHigh - expr: max by (namespace, pod) (redis_memory_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 + expr: (max by (namespace, pod) (redis_memory_used_bytes{namespace=~"code-place-(dev|prod)"}) / max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}) > 0.85) and on (namespace, pod) max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}) > 0 for: 10m labels: severity: warning priority: P1 annotations: summary: "Redis memory usage is high" - description: "Redis exporter reports memory usage above 85% of configured max memory for 10 minutes." + description: "Redis exporter reports memory usage above 85% of configured maxmemory for 10 minutes." - alert: PVCAlmostFull expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 for: 10m From 9a9780d32a7d808797c4a44743c9713a1fe28743 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 04:14:17 +0900 Subject: [PATCH 012/147] =?UTF-8?q?Sentry=20=EC=9D=B4=EB=B2=A4=ED=8A=B8=20?= =?UTF-8?q?=EB=AF=BC=EA=B0=90=EC=A0=95=EB=B3=B4=20=ED=95=84=ED=84=B0?= =?UTF-8?q?=EB=A7=81=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/oj/settings.py | 65 +++++++++++++++++-- .../observability-improvement-plan/_index.md | 4 +- .../infra/observability-runbook/_index.md | 2 + frontend/Dockerfile | 4 ++ frontend/src/utils/sentry.js | 48 ++++++++++++++ 5 files changed, 117 insertions(+), 6 deletions(-) diff --git a/backend/oj/settings.py b/backend/oj/settings.py index b3c70235..26c624f3 100644 --- a/backend/oj/settings.py +++ b/backend/oj/settings.py @@ -22,16 +22,71 @@ else: from .dev_settings import * +SENTRY_SENSITIVE_EXACT_FIELDS = { + "authorization", + "cookie", + "password", + "token", + "secret", + "code", + "src", +} +SENTRY_SENSITIVE_SUBSTRINGS = { + "authorization", + "cookie", + "password", + "token", + "secret", + "source_code", + "sourcecode", + "spj_code", + "src", +} + + +def _is_sentry_sensitive_key(key): + normalized = str(key).lower() + return ( + normalized in SENTRY_SENSITIVE_EXACT_FIELDS + or any(field in normalized for field in SENTRY_SENSITIVE_SUBSTRINGS) + ) + + +def _redact_sentry_value(value): + if isinstance(value, dict): + return { + key: "[REDACTED]" if _is_sentry_sensitive_key(key) else _redact_sentry_value(item) + for key, item in value.items() + } + if isinstance(value, list): + return [_redact_sentry_value(item) for item in value] + return value + + +def _before_send_sentry(event, hint): + event = _redact_sentry_value(event) + request = event.get("request") or {} + request_id = (request.get("headers") or {}).get("X-Request-ID") + if request_id: + event.setdefault("tags", {})["request_id"] = request_id + return event + + SENTRY_DSN = get_env( "SENTRY_DSN", "", ) if SENTRY_DSN: - sentry_sdk.init( - dsn=SENTRY_DSN, - send_default_pii=False, - environment=get_env("SENTRY_ENVIRONMENT", get_env("OJ_ENV", "dev")), - ) + SENTRY_OPTIONS = { + "dsn": SENTRY_DSN, + "send_default_pii": False, + "environment": get_env("SENTRY_ENVIRONMENT", get_env("OJ_ENV", "dev")), + "before_send": _before_send_sentry, + } + SENTRY_RELEASE = get_env("SENTRY_RELEASE", "") + if SENTRY_RELEASE: + SENTRY_OPTIONS["release"] = SENTRY_RELEASE + sentry_sdk.init(**SENTRY_OPTIONS) with open(os.path.join(DATA_DIR, "config", "secret.key"), "r") as f: SECRET_KEY = f.read() diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index ea0a5659..583894db 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -178,7 +178,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - PostgreSQL은 CNPG instance exporter와 PodMonitor로 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor로 `redis_*` 지표를 수집합니다. -- Sentry backend SDK는 기본 PII 자동 전송을 비활성화합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. +- Sentry backend SDK는 기본 PII 자동 전송을 비활성화하고 전송 직전 `authorization`, `cookie`, `password`, `token`, `secret`, 제출 source code 계열 필드를 redaction합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. +- Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. +- Backend/Celery Sentry release는 `SENTRY_RELEASE` env가 있으면 사용합니다. 이미지 태그와 동일한 commit SHA를 넣어 Grafana/Loki/Sentry 간 배포 기준을 맞추는 것을 권장합니다. - Tempo tracing은 저장소/retention 결정을 먼저 요구하므로 현재 PR에서는 켜지 않습니다. ## 6. 검증 diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index e6d07903..6e46439d 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -103,6 +103,8 @@ Grafana Explore: - 5xx와 latency가 같이 오르면 backend/DB/Redis 병목 가능성이 큽니다. - 5xx만 증가하면 최근 배포, exception, 외부 dependency를 우선 확인합니다. +- Sentry event가 있으면 `environment`, `release`, `request_id` tag를 기준으로 같은 시간대 Loki 로그와 대조합니다. +- Frontend Sentry 설정은 image build 시점에 결정됩니다. `SENTRY_DSN`, `USE_SENTRY`, `SENTRY_ENVIRONMENT`, `APP_VERSION` 변경 후에는 frontend image rebuild가 필요합니다. ### Ingress5xxSpike diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 1a5725ea..d1e53e1e 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -10,8 +10,12 @@ COPY . . ARG APP_VERSION ARG SENTRY_ENVIRONMENT +ARG SENTRY_DSN +ARG USE_SENTRY ENV VUE_APP_VERSION=$APP_VERSION ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT +ENV SENTRY_DSN=$SENTRY_DSN +ENV USE_SENTRY=$USE_SENTRY RUN npm run build:dll RUN npm run build diff --git a/frontend/src/utils/sentry.js b/frontend/src/utils/sentry.js index f6987606..07b60eb5 100644 --- a/frontend/src/utils/sentry.js +++ b/frontend/src/utils/sentry.js @@ -3,9 +3,57 @@ import { init, setContext } from "@sentry/vue" const SENTRY_DSN = "https://143814aaa2d6e0b4550b2e5effefe90d@o4511586463776768.ingest.us.sentry.io/4511586483634176" +const SENSITIVE_EXACT_FIELDS = [ + "authorization", + "cookie", + "password", + "token", + "secret", + "code", + "src", +] + +const SENSITIVE_SUBSTRINGS = [ + "authorization", + "cookie", + "password", + "token", + "secret", + "source_code", + "sourcecode", + "spj_code", + "src", +] + +function isSensitiveKey(key) { + const normalized = String(key).toLowerCase() + return ( + SENSITIVE_EXACT_FIELDS.indexOf(normalized) !== -1 || + SENSITIVE_SUBSTRINGS.some((field) => normalized.indexOf(field) !== -1) + ) +} + +function redact(value) { + if (Array.isArray(value)) { + return value.map((item) => redact(item)) + } + + if (value && typeof value === "object") { + return Object.keys(value).reduce((result, key) => { + result[key] = isSensitiveKey(key) ? "[REDACTED]" : redact(value[key]) + return result + }, {}) + } + + return value +} + const options = { release: process.env.VERSION, environment: process.env.SENTRY_ENVIRONMENT, + beforeSend(event) { + return redact(event) + }, denyUrls: [ // Chrome extensions /extensions\//i, From 1d8ecc6b183daa1cfe08b9e10d60a0ce92f086b9 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 04:16:27 +0900 Subject: [PATCH 013/147] =?UTF-8?q?Sentry=20=EB=A6=B4=EB=A6=AC=EC=A6=88=20?= =?UTF-8?q?=ED=83=9C=EA=B7=B8=20=EB=B0=B0=ED=8F=AC=20=EC=97=B0=EB=8F=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci2develop.yml | 14 +++++++++++++- .github/workflows/ci2production-and-release.yml | 14 +++++++++++++- .../overlays/dev/backend-deployment-patch.yaml | 2 ++ .../overlays/dev/celery-beat-deployment-patch.yaml | 2 ++ .../dev/celery-worker-deployment-patch.yaml | 2 ++ .../overlays/prod/backend-deployment-patch.yaml | 2 ++ .../prod/celery-beat-deployment-patch.yaml | 2 ++ .../prod/celery-worker-deployment-patch.yaml | 2 ++ 8 files changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index b6fe14fd..227f64ec 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -136,6 +136,15 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml + - name: Update Backend Sentry Release + if: needs.ci-backend-dev.result == 'success' + uses: mikefarah/yq@master + with: + cmd: | + yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/backend-deployment-patch.yaml + yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-worker-deployment-patch.yaml + yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-beat-deployment-patch.yaml + - name: Update Frontend Image Tag if: needs.ci-frontend-dev.result == 'success' uses: mikefarah/yq@master @@ -152,7 +161,10 @@ jobs: run: | git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git add kubernetes/overlays/dev/kustomization.yaml + git add kubernetes/overlays/dev/kustomization.yaml \ + kubernetes/overlays/dev/backend-deployment-patch.yaml \ + kubernetes/overlays/dev/celery-worker-deployment-patch.yaml \ + kubernetes/overlays/dev/celery-beat-deployment-patch.yaml # 변경사항이 없으면 종료 if git diff-index --quiet HEAD; then diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index 28e3db59..f0488ac1 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -165,6 +165,15 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml + - name: Update Backend Sentry Release + if: needs.ci-backend.result == 'success' + uses: mikefarah/yq@master + with: + cmd: | + yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/backend-deployment-patch.yaml + yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-worker-deployment-patch.yaml + yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-beat-deployment-patch.yaml + - name: Update Frontend Image Tag if: needs.ci-frontend.result == 'success' uses: mikefarah/yq@master @@ -181,7 +190,10 @@ jobs: run: | git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git add kubernetes/overlays/prod/kustomization.yaml + git add kubernetes/overlays/prod/kustomization.yaml \ + kubernetes/overlays/prod/backend-deployment-patch.yaml \ + kubernetes/overlays/prod/celery-worker-deployment-patch.yaml \ + kubernetes/overlays/prod/celery-beat-deployment-patch.yaml # 변경사항이 없으면 종료 if git diff-index --quiet HEAD; then diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index 29a24409..5bdbadfe 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -17,3 +17,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: SENTRY_RELEASE + value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index b4639921..12692cc1 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -20,3 +20,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: SENTRY_RELEASE + value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index 6a9b0ae2..fd033aa5 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -23,3 +23,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: SENTRY_RELEASE + value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" diff --git a/kubernetes/overlays/prod/backend-deployment-patch.yaml b/kubernetes/overlays/prod/backend-deployment-patch.yaml index de88b416..e4468851 100644 --- a/kubernetes/overlays/prod/backend-deployment-patch.yaml +++ b/kubernetes/overlays/prod/backend-deployment-patch.yaml @@ -24,3 +24,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" + - name: SENTRY_RELEASE + value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml index db60527a..bb4e4f49 100644 --- a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml @@ -20,3 +20,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" + - name: SENTRY_RELEASE + value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml index f98ee708..cb654565 100644 --- a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml @@ -23,3 +23,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" + - name: SENTRY_RELEASE + value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" From e8e3a55a2d4fc17455eedbe142d9a4bddddd1e3e Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 13:46:08 +0900 Subject: [PATCH 014/147] =?UTF-8?q?=ED=94=84=EB=A1=A0=ED=8A=B8=20=EC=9A=94?= =?UTF-8?q?=EC=B2=AD=20ID=20=EC=A0=84=ED=8C=8C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 3 +- .../infra/observability-runbook/_index.md | 1 + frontend/src/pages/admin/api.js | 2 + frontend/src/pages/oj/api.js | 2 + frontend/src/utils/request_id.js | 76 +++++++++++++++++++ frontend/src/utils/sentry.js | 6 ++ 6 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 frontend/src/utils/request_id.js diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 583894db..746a5a96 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -41,7 +41,7 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 ### Logs -backend는 `X-Request-ID`를 수용하고, 없으면 request ID를 생성한 뒤 응답 header로 반환합니다. +backend는 `X-Request-ID`를 수용하고, 없으면 request ID를 생성한 뒤 응답 header로 반환합니다. Frontend axios 요청은 `X-Request-ID`를 생성해 backend로 전달하고, backend가 반환한 request ID를 axios response/error와 Sentry request context에 보존합니다. Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설정합니다. JSON 로그 필드는 다음을 기준으로 합니다. @@ -179,6 +179,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - PostgreSQL은 CNPG instance exporter와 PodMonitor로 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor로 `redis_*` 지표를 수집합니다. - Sentry backend SDK는 기본 PII 자동 전송을 비활성화하고 전송 직전 `authorization`, `cookie`, `password`, `token`, `secret`, 제출 source code 계열 필드를 redaction합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. +- Frontend는 axios 요청마다 `X-Request-ID`를 전파하고 마지막 request ID를 `window.__CODEPLACE_LAST_REQUEST_ID__`와 Sentry request context에 저장합니다. - Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. - Backend/Celery Sentry release는 `SENTRY_RELEASE` env가 있으면 사용합니다. 이미지 태그와 동일한 commit SHA를 넣어 Grafana/Loki/Sentry 간 배포 기준을 맞추는 것을 권장합니다. - Tempo tracing은 저장소/retention 결정을 먼저 요구하므로 현재 PR에서는 켜지 않습니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 6e46439d..5abbb3a4 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -104,6 +104,7 @@ Grafana Explore: - 5xx와 latency가 같이 오르면 backend/DB/Redis 병목 가능성이 큽니다. - 5xx만 증가하면 최근 배포, exception, 외부 dependency를 우선 확인합니다. - Sentry event가 있으면 `environment`, `release`, `request_id` tag를 기준으로 같은 시간대 Loki 로그와 대조합니다. +- 사용자 브라우저에서 재현 중이면 `window.__CODEPLACE_LAST_REQUEST_ID__` 값을 확인해 같은 request ID로 Loki/backend JSON log를 조회합니다. - Frontend Sentry 설정은 image build 시점에 결정됩니다. `SENTRY_DSN`, `USE_SENTRY`, `SENTRY_ENVIRONMENT`, `APP_VERSION` 변경 후에는 frontend image rebuild가 필요합니다. ### Ingress5xxSpike diff --git a/frontend/src/pages/admin/api.js b/frontend/src/pages/admin/api.js index 526ee5cb..57b38d10 100644 --- a/frontend/src/pages/admin/api.js +++ b/frontend/src/pages/admin/api.js @@ -2,11 +2,13 @@ import Vue from "vue" import router from "./router" import axios from "axios" import utils from "@/utils/utils" +import { configureAxiosRequestId } from "@/utils/request_id" Vue.prototype.$http = axios axios.defaults.baseURL = "/api" axios.defaults.xsrfHeaderName = "X-CSRFToken" axios.defaults.xsrfCookieName = "csrftoken" +configureAxiosRequestId(axios) export default { // 登录 diff --git a/frontend/src/pages/oj/api.js b/frontend/src/pages/oj/api.js index 6a0ade9f..2154ee68 100644 --- a/frontend/src/pages/oj/api.js +++ b/frontend/src/pages/oj/api.js @@ -1,11 +1,13 @@ import Vue from "vue" import store from "@/store" import axios from "axios" +import { configureAxiosRequestId } from "@/utils/request_id" Vue.prototype.$http = axios axios.defaults.baseURL = "/api" axios.defaults.xsrfHeaderName = "X-CSRFToken" axios.defaults.xsrfCookieName = "csrftoken" +configureAxiosRequestId(axios) export default { getPopup() { diff --git a/frontend/src/utils/request_id.js b/frontend/src/utils/request_id.js new file mode 100644 index 00000000..b80246fe --- /dev/null +++ b/frontend/src/utils/request_id.js @@ -0,0 +1,76 @@ +import { setRequestIdContext } from "@/utils/sentry" + +const REQUEST_ID_HEADER = "X-Request-ID" +const RESPONSE_REQUEST_ID_HEADER = "x-request-id" + +function randomHex(length) { + if (window.crypto && window.crypto.getRandomValues) { + const values = new Uint8Array(Math.ceil(length / 2)) + window.crypto.getRandomValues(values) + return Array.from(values) + .map((value) => value.toString(16).padStart(2, "0")) + .join("") + .slice(0, length) + } + + return Math.random().toString(16).slice(2, 2 + length).padEnd(length, "0") +} + +function createRequestId() { + return `cp-${Date.now().toString(36)}-${randomHex(16)}` +} + +function rememberRequestId(requestId) { + if (!requestId) { + return + } + + window.__CODEPLACE_LAST_REQUEST_ID__ = requestId + setRequestIdContext(requestId) +} + +function responseRequestId(response) { + return ( + response && + response.headers && + (response.headers[RESPONSE_REQUEST_ID_HEADER] || response.headers[REQUEST_ID_HEADER]) + ) +} + +export function configureAxiosRequestId(axios) { + if (axios.__codeplaceRequestIdConfigured) { + return + } + axios.__codeplaceRequestIdConfigured = true + + axios.interceptors.request.use((config) => { + const requestId = createRequestId() + config.headers = config.headers || {} + config.headers[REQUEST_ID_HEADER] = requestId + config.metadata = { + ...(config.metadata || {}), + requestId, + } + rememberRequestId(requestId) + return config + }) + + axios.interceptors.response.use( + (response) => { + const requestId = + responseRequestId(response) || + (response.config && response.config.metadata && response.config.metadata.requestId) + response.requestId = requestId + rememberRequestId(requestId) + return response + }, + (error) => { + const requestId = + responseRequestId(error.response) || + (error.config && error.config.metadata && error.config.metadata.requestId) + error.requestId = requestId + rememberRequestId(requestId) + return Promise.reject(error) + }, + ) +} diff --git a/frontend/src/utils/sentry.js b/frontend/src/utils/sentry.js index 07b60eb5..4a135b2a 100644 --- a/frontend/src/utils/sentry.js +++ b/frontend/src/utils/sentry.js @@ -85,3 +85,9 @@ export function initSentry(Vue) { location: window.location.href, }) } + +export function setRequestIdContext(requestId) { + setContext("request", { + request_id: requestId, + }) +} From 01bf7475e987a111a5a3aabc48015077a8b57376 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 13:49:00 +0900 Subject: [PATCH 015/147] =?UTF-8?q?=EC=95=8C=EB=A6=BC=20=EC=9D=B4=EB=A9=94?= =?UTF-8?q?=EC=9D=BC=20fallback=20=EC=98=88=EC=8B=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 3 + .../infra/observability-runbook/_index.md | 3 + .../alertmanager-config-email.example.yaml | 72 +++++++++++++++++++ kubernetes/monitoring/validate.sh | 1 + 4 files changed, 79 insertions(+) create mode 100644 kubernetes/monitoring/alertmanager-config-email.example.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 746a5a96..eb4573fd 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -113,6 +113,8 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT `alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. +Email fallback은 SMTP host/from/to/auth가 필요하므로 기본 kustomization에는 포함하지 않습니다. SMTP 값이 정해지면 `kubernetes/monitoring/alertmanager-config-email.example.yaml`을 실제 값으로 복사하거나 패치해서 적용합니다. Prometheus Operator의 `AlertmanagerConfig`는 receiver에 `emailConfigs`를 지원하며, SMTP password는 같은 namespace의 Secret key로 참조합니다. + ## 4. 빠른 알림 정책 ### P0 @@ -155,6 +157,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. 2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. + Email fallback을 켤 경우 SMTP 정보를 별도로 정한 뒤 `alertmanager-email` Secret과 email AlertmanagerConfig를 추가 적용합니다. 3. Loki와 Alloy를 설치합니다. - `helm upgrade --install loki grafana-community/loki --namespace monitoring --values kubernetes/monitoring/logs/loki-values.yaml` - `helm upgrade --install alloy grafana/alloy --namespace monitoring --values kubernetes/monitoring/logs/alloy-values.yaml` diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 5abbb3a4..6bfe0e7d 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -38,6 +38,8 @@ Alerts -> alertname / namespace / priority Status -> Configuration -> p0-discord / p1-discord ``` +Email fallback을 켠 환경에서는 `Status -> Configuration`에서 `p0-email` / `p1-email` receiver도 확인합니다. + Loki 로그 확인: ```logql @@ -404,6 +406,7 @@ node pressure는 앱 문제가 아니라 cluster resource 문제일 가능성이 ## Test Alert Discord webhook 자체는 curl로 확인할 수 있습니다. Alertmanager routing은 임시 `PrometheusRule`로 확인합니다. +Email fallback은 SMTP 정보가 정해진 환경에서만 켭니다. `kubernetes/monitoring/alertmanager-config-email.example.yaml`의 placeholder를 실제 `smarthost`, `from`, `to`, `authUsername`으로 바꾸고 `alertmanager-email` Secret을 만든 뒤 적용합니다. ```yaml apiVersion: monitoring.coreos.com/v1 diff --git a/kubernetes/monitoring/alertmanager-config-email.example.yaml b/kubernetes/monitoring/alertmanager-config-email.example.yaml new file mode 100644 index 00000000..4c448539 --- /dev/null +++ b/kubernetes/monitoring/alertmanager-config-email.example.yaml @@ -0,0 +1,72 @@ +# Optional email fallback for CodePlace alerts. +# +# Do not add this file to kustomization.yaml until SMTP values are available. +# Required secret: +# +# kubectl -n monitoring create secret generic alertmanager-email \ +# --from-literal=smtp-auth-password="$SMTP_AUTH_PASSWORD" \ +# --dry-run=client -o yaml | kubectl apply -f - +# +# Replace smarthost, from, to, and authUsername before applying. +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: codeplace-alert-email-fallback + labels: + alertmanagerConfig: codeplace +spec: + route: + groupBy: + - alertname + - priority + - namespace + groupWait: 30s + groupInterval: 5m + repeatInterval: 1h + receiver: p1-email + routes: + - matchers: + - name: priority + value: P0 + groupWait: 10s + groupInterval: 1m + repeatInterval: 15m + receiver: p0-email + - matchers: + - name: priority + value: P1 + groupWait: 30s + groupInterval: 5m + repeatInterval: 1h + receiver: p1-email + receivers: + - name: p0-email + emailConfigs: + - sendResolved: true + smarthost: smtp.example.com:587 + from: codeplace-alerts@example.com + to: oncall@example.com + authUsername: codeplace-alerts@example.com + authPassword: + name: alertmanager-email + key: smtp-auth-password + requireTLS: true + headers: + - key: Subject + value: '[{{ .Status }}][P0] {{ .CommonLabels.alertname }}' + html: '{{ .CommonAnnotations.summary }}
{{ .CommonAnnotations.description }}' + - name: p1-email + emailConfigs: + - sendResolved: true + smarthost: smtp.example.com:587 + from: codeplace-alerts@example.com + to: oncall@example.com + authUsername: codeplace-alerts@example.com + authPassword: + name: alertmanager-email + key: smtp-auth-password + requireTLS: true + headers: + - key: Subject + value: '[{{ .Status }}][P1] {{ .CommonLabels.alertname }}' + html: '{{ .CommonAnnotations.summary }}
{{ .CommonAnnotations.description }}' diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 77d91a93..448540b1 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -35,6 +35,7 @@ yaml_paths = [ root / "kube-prometheus-stack-values.yaml", root / "prometheus-rules.yaml", root / "alertmanager-config.yaml", + root / "alertmanager-config-email.example.yaml", root / "backend-service-monitor.yaml", root / "datastore-pod-monitors.yaml", root / "logs" / "loki-values.yaml", From ee2e225ab137c00aa681497c0bf7463d3463a11d Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 13:51:22 +0900 Subject: [PATCH 016/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EB=A1=9C?= =?UTF-8?q?=EA=B7=B8=EC=99=80=20=EC=9A=94=EC=B2=AD=20ID=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/account/tests.py | 28 ++++++++++++++++++++++- backend/utils/tests.py | 49 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/backend/account/tests.py b/backend/account/tests.py index 653ca07c..43cab62d 100644 --- a/backend/account/tests.py +++ b/backend/account/tests.py @@ -8,7 +8,7 @@ from django.contrib import auth from django.db import DatabaseError from django.http.response import JsonResponse -from django.test import RequestFactory, TestCase +from django.test import RequestFactory, SimpleTestCase, TestCase from django.utils import decorators from django.utils.timezone import now from utils.api.tests import APIClient, APITestCase @@ -18,9 +18,35 @@ from .models import AdminType, ProblemPermission, User from .decorators import scheduler_only +from .middleware import RequestIDMiddleware from .tasks import calculate_user_score_basis, calculate_user_score_fluctuation +class RequestIDMiddlewareTest(SimpleTestCase): + + def setUp(self): + self.factory = RequestFactory() + + def test_generates_request_id_when_header_is_missing(self): + middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True})) + request = self.factory.get("/") + + response = middleware(request) + + self.assertEqual(len(request.request_id), 32) + self.assertEqual(response["X-Request-ID"], request.request_id) + + def test_accepts_existing_request_id_and_truncates_it(self): + request_id = "a" * 150 + middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True})) + request = self.factory.get("/", HTTP_X_REQUEST_ID=request_id) + + response = middleware(request) + + self.assertEqual(request.request_id, "a" * 128) + self.assertEqual(response["X-Request-ID"], request.request_id) + + class PermissionDecoratorTest(APITestCase): """ 데코레이터 테스트 diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 7e2f85ef..a775b282 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -1,12 +1,59 @@ import json +import logging import os from unittest.mock import patch, mock_open, MagicMock -from django.test import TestCase, override_settings +from django.test import SimpleTestCase, TestCase, override_settings from django.core.cache import cache +from utils.json_logging import CodePlaceJsonFormatter from utils.testcase_cache import TestCaseCacheManager +class CodePlaceJsonFormatterTest(SimpleTestCase): + + def test_preserves_status_code_and_redacts_sensitive_fields(self): + record = logging.LogRecord( + name="codeplace.request", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="request completed", + args=(), + exc_info=None, + ) + record.status_code = 500 + record.request_id = "request-1" + record.token = "secret-token" + record.source_code = "print('secret')" + + payload = json.loads(CodePlaceJsonFormatter().format(record)) + + self.assertEqual(payload["status_code"], 500) + self.assertEqual(payload["request_id"], "request-1") + self.assertNotIn("token", payload) + self.assertNotIn("source_code", payload) + + def test_redacts_nested_sensitive_fields(self): + formatter = CodePlaceJsonFormatter() + payload = { + "status_code": 200, + "data": { + "password": "secret", + "submission": { + "code": "print('secret')", + "language": "Python3", + }, + }, + } + + redacted = formatter._redact(payload) + + self.assertEqual(redacted["status_code"], 200) + self.assertEqual(redacted["data"]["password"], "[REDACTED]") + self.assertEqual(redacted["data"]["submission"]["code"], "[REDACTED]") + self.assertEqual(redacted["data"]["submission"]["language"], "Python3") + + class TestTestCaseCacheManager(TestCase): def setUp(self): From 41f539e4f79dd76bb02c11cd58015e327b7d31df Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 13:53:00 +0900 Subject: [PATCH 017/147] =?UTF-8?q?=ED=8A=B8=EB=A0=88=EC=9D=B4=EC=8B=B1=20?= =?UTF-8?q?=EB=B9=84=ED=99=9C=EC=84=B1=20=EA=B2=BD=EB=A1=9C=20=EC=95=88?= =?UTF-8?q?=EC=A0=84=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/judge/dispatcher.py | 4 ++-- backend/judge/tasks.py | 4 ++-- backend/utils/observability_tracing.py | 21 +++++++++++++++++++++ backend/utils/tests.py | 18 ++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/backend/judge/dispatcher.py b/backend/judge/dispatcher.py index 60e92bc2..231fde01 100644 --- a/backend/judge/dispatcher.py +++ b/backend/judge/dispatcher.py @@ -10,7 +10,6 @@ from django.db import transaction, IntegrityError from django.db.models import F from django.http import HttpResponseNotFound -from opentelemetry import trace from account.models import User, UserScore, UserSolved from conf.models import JudgeServer @@ -21,9 +20,10 @@ from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.constants import CacheKey, ProblemScore, ProblemField, Tier +from utils.observability_tracing import get_tracer logger = logging.getLogger(__name__) -tracer = trace.get_tracer(__name__) +tracer = get_tracer(__name__) # 继续处理在队列中的问题 diff --git a/backend/judge/tasks.py b/backend/judge/tasks.py index 483be700..bc58d51c 100644 --- a/backend/judge/tasks.py +++ b/backend/judge/tasks.py @@ -1,12 +1,12 @@ import celery -from opentelemetry import trace from account.models import User from utils.shortcuts import CELERY_TASK_ARGS +from utils.observability_tracing import get_tracer from submission.models import Submission from judge.dispatcher import JudgeDispatcher -tracer = trace.get_tracer(__name__) +tracer = get_tracer(__name__) @celery.shared_task(**CELERY_TASK_ARGS()) diff --git a/backend/utils/observability_tracing.py b/backend/utils/observability_tracing.py index 9702522f..26ad9182 100644 --- a/backend/utils/observability_tracing.py +++ b/backend/utils/observability_tracing.py @@ -1,10 +1,31 @@ import logging +from contextlib import nullcontext from utils.shortcuts import get_env logger = logging.getLogger(__name__) +class _NoOpSpan: + + def set_attribute(self, key, value): + return None + + +class _NoOpTracer: + + def start_as_current_span(self, name): + return nullcontext(_NoOpSpan()) + + +def get_tracer(name): + try: + from opentelemetry import trace + except Exception: + return _NoOpTracer() + return trace.get_tracer(name) + + def configure_opentelemetry(service_name): if get_env("OTEL_ENABLED", "0").lower() not in ("1", "true", "yes", "on"): return diff --git a/backend/utils/tests.py b/backend/utils/tests.py index a775b282..bdd4a751 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -6,9 +6,27 @@ from django.core.cache import cache from utils.json_logging import CodePlaceJsonFormatter +from utils.observability_tracing import get_tracer from utils.testcase_cache import TestCaseCacheManager +class ObservabilityTracingTest(SimpleTestCase): + + def test_get_tracer_returns_noop_tracer_when_opentelemetry_is_unavailable(self): + real_import = __import__ + + def fake_import(name, *args, **kwargs): + if name == "opentelemetry": + raise ImportError("missing opentelemetry") + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=fake_import): + tracer = get_tracer("test") + + with tracer.start_as_current_span("span") as span: + self.assertIsNone(span.set_attribute("key", "value")) + + class CodePlaceJsonFormatterTest(SimpleTestCase): def test_preserves_status_code_and_redacts_sensitive_fields(self): From a0adaabf995c58765c707872c99a6b9241f74522 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 13:55:32 +0900 Subject: [PATCH 018/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EB=A7=A4?= =?UTF-8?q?=EB=8B=88=ED=8E=98=EC=8A=A4=ED=8A=B8=20=EA=B2=80=EC=A6=9D=20?= =?UTF-8?q?=EA=B0=95=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 5 ++ kubernetes/monitoring/validate.sh | 82 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index eb4573fd..cef1b32b 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -199,6 +199,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Logs values YAML parse: `kubernetes/monitoring/logs/loki-values.yaml`, `kubernetes/monitoring/logs/alloy-values.yaml` - Grafana dashboard JSON parse: `grafana-dashboard-codeplace.yaml`, `grafana-dashboard-logs.yaml` - Monitoring bundle validation: `bash kubernetes/monitoring/validate.sh` + - YAML parse: kube-prometheus-stack values, PrometheusRule, AlertmanagerConfig, ServiceMonitor, PodMonitor, Loki/Alloy values. + - Grafana dashboard JSON parse와 dashboard shape check: uid/title/refresh/panel/target expr. + - PrometheusRule shape check: P0/P1 interval, alert priority/severity, summary/description, duplicate alert namespace label. + - AlertmanagerConfig shape check: groupBy, P0/P1 Discord receivers, webhook Secret reference. + - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. `promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다. `helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다. diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 448540b1..3ade2290 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -60,6 +60,88 @@ for path in yaml_paths: for key, value in data.get("data", {}).items(): json.loads(value) print(f"JSON OK {path}:{key}") + +if yaml is None: + sys.exit(0) + +rules = list(yaml.safe_load_all((root / "prometheus-rules.yaml").read_text()))[0] +rule_groups = {group["name"]: group for group in rules["spec"]["groups"]} +expected_intervals = { + "codeplace.p0": "15s", + "codeplace.p1": "30s", +} +for group_name, interval in expected_intervals.items(): + actual = rule_groups.get(group_name, {}).get("interval") + if actual != interval: + raise SystemExit(f"{group_name} interval must be {interval}, got {actual}") + +alert_instances = {} +for group in rules["spec"]["groups"]: + expected_priority = "P0" if group["name"].endswith(".p0") else "P1" if group["name"].endswith(".p1") else None + for rule in group.get("rules", []): + alert_name = rule.get("alert") + if not alert_name: + raise SystemExit(f"rule in {group['name']} is missing alert name") + if not rule.get("expr"): + raise SystemExit(f"{alert_name} is missing expr") + labels = rule.get("labels", {}) + annotations = rule.get("annotations", {}) + if labels.get("severity") not in {"critical", "warning"}: + raise SystemExit(f"{alert_name} is missing valid severity") + if labels.get("priority") not in {"P0", "P1"}: + raise SystemExit(f"{alert_name} is missing valid priority") + if expected_priority and labels.get("priority") != expected_priority: + raise SystemExit(f"{alert_name} priority must match {group['name']}") + for key in ("summary", "description"): + if not annotations.get(key): + raise SystemExit(f"{alert_name} is missing annotation {key}") + alert_instances.setdefault(alert_name, []).append(labels) + +for alert_name, labels_list in alert_instances.items(): + if len(labels_list) > 1 and any("namespace" not in labels for labels in labels_list): + raise SystemExit(f"duplicate alert {alert_name} must include static namespace labels") +print("PROMETHEUS RULE SHAPE OK") + +alertmanager = list(yaml.safe_load_all((root / "alertmanager-config.yaml").read_text()))[0] +route = alertmanager["spec"]["route"] +if set(route.get("groupBy", [])) != {"alertname", "priority", "namespace"}: + raise SystemExit("Alertmanager groupBy must be alertname, priority, namespace") +receivers = {receiver["name"]: receiver for receiver in alertmanager["spec"]["receivers"]} +for receiver_name in ("p0-discord", "p1-discord"): + receiver = receivers.get(receiver_name) + if not receiver: + raise SystemExit(f"missing receiver {receiver_name}") + configs = receiver.get("discordConfigs") or [] + if not configs: + raise SystemExit(f"{receiver_name} must define discordConfigs") + api_url = configs[0].get("apiURL", {}) + if api_url.get("name") != "alertmanager-contact-points" or api_url.get("key") != "webhook-url": + raise SystemExit(f"{receiver_name} must use alertmanager-contact-points/webhook-url") +print("ALERTMANAGER SHAPE OK") + +kustomization = yaml.safe_load((root / "kustomization.yaml").read_text()) +if "./alertmanager-config-email.example.yaml" in kustomization.get("resources", []): + raise SystemExit("email fallback example must not be included in default kustomization") +print("MONITORING KUSTOMIZATION SHAPE OK") + +for dashboard_path in (root / "grafana-dashboard-codeplace.yaml", root / "grafana-dashboard-logs.yaml"): + dashboard_map = yaml.safe_load(dashboard_path.read_text()) + for key, value in dashboard_map.get("data", {}).items(): + dashboard = json.loads(value) + if not dashboard.get("uid") or not dashboard.get("title"): + raise SystemExit(f"{dashboard_path}:{key} must define uid and title") + if dashboard.get("refresh") != "30s": + raise SystemExit(f"{dashboard_path}:{key} refresh must be 30s") + panels = dashboard.get("panels") or [] + if not panels: + raise SystemExit(f"{dashboard_path}:{key} must contain panels") + for panel in panels: + if not panel.get("title") or not panel.get("type") or not panel.get("gridPos"): + raise SystemExit(f"{dashboard_path}:{key} has panel missing title/type/gridPos") + for target in panel.get("targets", []): + if not target.get("expr"): + raise SystemExit(f"{dashboard_path}:{key}:{panel.get('title')} has target missing expr") +print("GRAFANA DASHBOARD SHAPE OK") PY echo "==> rendering kustomize monitoring manifests" From 71191bf9f875713a6e3c989f9908f7415a16d907 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 13:57:20 +0900 Subject: [PATCH 019/147] =?UTF-8?q?Judge=20=EC=84=9C=EB=B2=84=20=EA=B4=80?= =?UTF-8?q?=EC=B8=A1=20=EC=A7=80=ED=91=9C=20=ED=9A=8C=EA=B7=80=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/tests.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/backend/utils/tests.py b/backend/utils/tests.py index bdd4a751..e988d040 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -1,15 +1,67 @@ import json import logging import os +from datetime import timedelta from unittest.mock import patch, mock_open, MagicMock from django.test import SimpleTestCase, TestCase, override_settings from django.core.cache import cache +from django.utils import timezone from utils.json_logging import CodePlaceJsonFormatter +from utils.observability_metrics import CodePlaceCollector from utils.observability_tracing import get_tracer from utils.testcase_cache import TestCaseCacheManager +class CodePlaceCollectorTest(SimpleTestCase): + + def _samples_by_metric(self, metrics): + return {metric.name: metric.samples for metric in metrics} + + def test_judge_server_metrics_skip_disabled_servers(self): + now = timezone.now() + enabled = MagicMock( + hostname="enabled-judge", + is_disabled=False, + last_heartbeat=now, + status="normal", + task_number=2, + ) + disabled = MagicMock( + hostname="disabled-judge", + is_disabled=True, + last_heartbeat=now - timedelta(days=30), + status="abnormal", + task_number=99, + ) + + with patch("conf.models.JudgeServer.objects.all", return_value=[enabled, disabled]): + samples = self._samples_by_metric(list(CodePlaceCollector()._judge_server_metrics())) + + for metric_samples in samples.values(): + hostnames = {sample.labels["hostname"] for sample in metric_samples} + self.assertIn("enabled-judge", hostnames) + self.assertNotIn("disabled-judge", hostnames) + + def test_judge_server_metrics_mark_stale_enabled_server_unavailable(self): + stale = MagicMock( + hostname="stale-judge", + is_disabled=False, + last_heartbeat=timezone.now() - timedelta(minutes=10), + status="abnormal", + task_number=1, + ) + + with patch("conf.models.JudgeServer.objects.all", return_value=[stale]): + samples = self._samples_by_metric(list(CodePlaceCollector()._judge_server_metrics())) + + available = samples["codeplace_judge_server_available"][0] + heartbeat_age = samples["codeplace_judge_server_last_heartbeat_age_seconds"][0] + self.assertEqual(available.labels["hostname"], "stale-judge") + self.assertEqual(available.value, 0) + self.assertGreaterEqual(heartbeat_age.value, 599) + + class ObservabilityTracingTest(SimpleTestCase): def test_get_tracer_returns_noop_tracer_when_opentelemetry_is_unavailable(self): From 86702e9359d37bf90a23d9660e944361221ba774 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:00:26 +0900 Subject: [PATCH 020/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=8B=A4?= =?UTF-8?q?=ED=81=B4=EB=9F=AC=EC=8A=A4=ED=84=B0=20=EC=A0=90=EA=B2=80=20?= =?UTF-8?q?=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 2 + kubernetes/monitoring/logs/README.md | 13 +++ kubernetes/monitoring/smoke-check.sh | 98 +++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100755 kubernetes/monitoring/smoke-check.sh diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index cef1b32b..656e206f 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -204,6 +204,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - PrometheusRule shape check: P0/P1 interval, alert priority/severity, summary/description, duplicate alert namespace label. - AlertmanagerConfig shape check: groupBy, P0/P1 Discord receivers, webhook Secret reference. - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. +- Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh` + - Prometheus Operator CRD, monitoring namespace resource, webhook Secret, kube-prometheus-stack Pod readiness, optional Loki/Alloy, app namespace backend readiness를 읽기 전용으로 확인합니다. `promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다. `helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다. diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 37c3df7b..78bb4519 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -40,6 +40,19 @@ bash kubernetes/monitoring/validate.sh If `helm` is installed and the chart repos are already added, the script also renders kube-prometheus-stack, Loki, and Alloy. Without `helm`, it still validates YAML, Grafana dashboard JSON, and kustomize output. +After applying the monitoring resources to a live cluster, run the read-only smoke check: + +```sh +bash kubernetes/monitoring/smoke-check.sh +``` + +Use `MONITORING_NAMESPACE` and `CODEPLACE_NAMESPACES` to override the defaults: + +```sh +MONITORING_NAMESPACE=monitoring CODEPLACE_NAMESPACES="code-place-dev" \ + bash kubernetes/monitoring/smoke-check.sh +``` + ```sh kubectl -n monitoring get pod | grep -E 'loki|alloy|grafana' kubectl -n monitoring get pvc | grep loki diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh new file mode 100755 index 00000000..d5342163 --- /dev/null +++ b/kubernetes/monitoring/smoke-check.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="${MONITORING_NAMESPACE:-monitoring}" +APP_NAMESPACES="${CODEPLACE_NAMESPACES:-code-place-dev code-place-prod}" + +ok() { + echo "OK $*" +} + +fail() { + echo "FAIL $*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +resource_exists() { + kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1 +} + +require_cmd kubectl + +echo "==> checking Prometheus Operator CRDs" +for crd in \ + prometheusrules.monitoring.coreos.com \ + servicemonitors.monitoring.coreos.com \ + podmonitors.monitoring.coreos.com \ + alertmanagerconfigs.monitoring.coreos.com; do + kubectl get crd "$crd" >/dev/null + ok "CRD $crd" +done + +echo "==> checking monitoring namespace resources" +kubectl get namespace "$NAMESPACE" >/dev/null +ok "namespace $NAMESPACE" + +for resource in \ + "prometheusrule/codeplace-fast-alerts" \ + "servicemonitor/backend" \ + "podmonitor/postgres" \ + "podmonitor/redis" \ + "alertmanagerconfig/codeplace-alert-routing" \ + "configmap/grafana-dashboard-codeplace" \ + "configmap/grafana-dashboard-codeplace-logs"; do + kubectl -n "$NAMESPACE" get "$resource" >/dev/null + ok "$resource" +done + +if resource_exists "$NAMESPACE" secret alertmanager-contact-points; then + kubectl -n "$NAMESPACE" get secret alertmanager-contact-points \ + -o jsonpath='{.data.webhook-url}' | grep -q . \ + || fail "secret/alertmanager-contact-points missing webhook-url key" + ok "secret/alertmanager-contact-points webhook-url" +else + fail "secret/alertmanager-contact-points is missing" +fi + +echo "==> checking kube-prometheus-stack pods" +for selector in \ + "app.kubernetes.io/name=grafana" \ + "app.kubernetes.io/name=prometheus" \ + "app.kubernetes.io/name=alertmanager" \ + "app.kubernetes.io/name=kube-prometheus-stack-operator"; do + kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l "$selector" --timeout=30s + ok "pods ready for $selector" +done + +echo "==> checking optional logs stack" +if kubectl -n "$NAMESPACE" get pod -l app.kubernetes.io/name=loki >/dev/null 2>&1; then + kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l app.kubernetes.io/name=loki --timeout=30s + ok "loki pods ready" +else + echo "SKIP loki pods: app.kubernetes.io/name=loki not found" +fi + +if kubectl -n "$NAMESPACE" get daemonset alloy >/dev/null 2>&1; then + desired="$(kubectl -n "$NAMESPACE" get daemonset alloy -o jsonpath='{.status.desiredNumberScheduled}')" + available="$(kubectl -n "$NAMESPACE" get daemonset alloy -o jsonpath='{.status.numberAvailable}')" + [ "$desired" = "$available" ] || fail "daemonset/alloy available=$available desired=$desired" + ok "daemonset/alloy available=$available desired=$desired" +else + echo "SKIP daemonset/alloy: not found" +fi + +echo "==> checking application namespaces" +for app_namespace in $APP_NAMESPACES; do + kubectl get namespace "$app_namespace" >/dev/null + ok "namespace $app_namespace" + kubectl -n "$app_namespace" get svc backend >/dev/null + ok "$app_namespace service/backend" + kubectl -n "$app_namespace" wait --for=condition=Ready pod -l app=backend --timeout=30s + ok "$app_namespace backend pods ready" +done + +echo "observability smoke check completed" From 5eaeb905bf6ff330beb4bbd7d8fd83c0039ecb01 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:02:50 +0900 Subject: [PATCH 021/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20selector?= =?UTF-8?q?=20=EA=B2=80=EC=A6=9D=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 4 +- kubernetes/monitoring/smoke-check.sh | 29 +++++++++++- kubernetes/monitoring/validate.sh | 45 +++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 656e206f..ff54af1a 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -203,9 +203,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Grafana dashboard JSON parse와 dashboard shape check: uid/title/refresh/panel/target expr. - PrometheusRule shape check: P0/P1 interval, alert priority/severity, summary/description, duplicate alert namespace label. - AlertmanagerConfig shape check: groupBy, P0/P1 Discord receivers, webhook Secret reference. + - kube-prometheus-stack values shape check: Prometheus selector policy, AlertmanagerConfig selector, Loki datasource, dashboard sidecar label. + - scrape resource shape check: ServiceMonitor/PodMonitor selector label, scrape path/port/interval. - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. - Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh` - - Prometheus Operator CRD, monitoring namespace resource, webhook Secret, kube-prometheus-stack Pod readiness, optional Loki/Alloy, app namespace backend readiness를 읽기 전용으로 확인합니다. + - Prometheus Operator CRD, monitoring namespace resource, selector label, webhook Secret, kube-prometheus-stack Pod readiness, optional Loki/Alloy, app namespace backend service/port/readiness를 읽기 전용으로 확인합니다. `promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다. `helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다. diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index d5342163..7364c5be 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -21,6 +21,22 @@ resource_exists() { kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1 } +resource_count() { + kubectl -n "$1" get "$2" -l "$3" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | wc -w | tr -d ' ' +} + +require_label() { + local namespace="$1" + local resource="$2" + local name="$3" + local key="$4" + local expected="$5" + local actual + actual="$(kubectl -n "$namespace" get "$resource" "$name" -o "jsonpath={.metadata.labels['${key//./\\.}']}")" + [ "$actual" = "$expected" ] || fail "$resource/$name label $key expected=$expected actual=$actual" + ok "$resource/$name label $key=$expected" +} + require_cmd kubectl echo "==> checking Prometheus Operator CRDs" @@ -49,6 +65,14 @@ for resource in \ ok "$resource" done +require_label "$NAMESPACE" prometheusrule codeplace-fast-alerts release kube-prometheus-stack +require_label "$NAMESPACE" servicemonitor backend release kube-prometheus-stack +require_label "$NAMESPACE" podmonitor postgres release kube-prometheus-stack +require_label "$NAMESPACE" podmonitor redis release kube-prometheus-stack +require_label "$NAMESPACE" alertmanagerconfig codeplace-alert-routing alertmanagerConfig codeplace +require_label "$NAMESPACE" configmap grafana-dashboard-codeplace grafana_dashboard 1 +require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-logs grafana_dashboard 1 + if resource_exists "$NAMESPACE" secret alertmanager-contact-points; then kubectl -n "$NAMESPACE" get secret alertmanager-contact-points \ -o jsonpath='{.data.webhook-url}' | grep -q . \ @@ -69,7 +93,7 @@ for selector in \ done echo "==> checking optional logs stack" -if kubectl -n "$NAMESPACE" get pod -l app.kubernetes.io/name=loki >/dev/null 2>&1; then +if [ "$(resource_count "$NAMESPACE" pod app.kubernetes.io/name=loki)" -gt 0 ]; then kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l app.kubernetes.io/name=loki --timeout=30s ok "loki pods ready" else @@ -91,6 +115,9 @@ for app_namespace in $APP_NAMESPACES; do ok "namespace $app_namespace" kubectl -n "$app_namespace" get svc backend >/dev/null ok "$app_namespace service/backend" + kubectl -n "$app_namespace" get svc backend -o jsonpath='{.spec.ports[*].name}' | grep -qw api \ + || fail "$app_namespace service/backend missing api port" + ok "$app_namespace service/backend api port" kubectl -n "$app_namespace" wait --for=condition=Ready pod -l app=backend --timeout=30s ok "$app_namespace backend pods ready" done diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 3ade2290..a0fa0c92 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -119,6 +119,51 @@ for receiver_name in ("p0-discord", "p1-discord"): raise SystemExit(f"{receiver_name} must use alertmanager-contact-points/webhook-url") print("ALERTMANAGER SHAPE OK") +stack_values = yaml.safe_load((root / "kube-prometheus-stack-values.yaml").read_text()) +prometheus_spec = stack_values.get("prometheus", {}).get("prometheusSpec", {}) +for key in ( + "serviceMonitorSelectorNilUsesHelmValues", + "podMonitorSelectorNilUsesHelmValues", + "ruleSelectorNilUsesHelmValues", +): + if prometheus_spec.get(key) is not False: + raise SystemExit(f"kube-prometheus-stack prometheusSpec.{key} must be false") +alertmanager_selector = ( + stack_values.get("alertmanager", {}) + .get("alertmanagerSpec", {}) + .get("alertmanagerConfigSelector", {}) + .get("matchLabels", {}) +) +if alertmanager_selector.get("alertmanagerConfig") != "codeplace": + raise SystemExit("AlertmanagerConfig selector must match alertmanagerConfig=codeplace") +grafana_values = stack_values.get("grafana", {}) +datasources = grafana_values.get("additionalDataSources") or [] +if not any(ds.get("name") == "Loki" and ds.get("type") == "loki" for ds in datasources): + raise SystemExit("Grafana values must provision Loki datasource") +dashboard_sidecar = grafana_values.get("sidecar", {}).get("dashboards", {}) +if dashboard_sidecar.get("label") != "grafana_dashboard": + raise SystemExit("Grafana dashboard sidecar must select grafana_dashboard label") +print("KUBE-PROMETHEUS-STACK VALUES SHAPE OK") + +backend_sm = yaml.safe_load((root / "backend-service-monitor.yaml").read_text()) +if backend_sm.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": + raise SystemExit("backend ServiceMonitor must keep release=kube-prometheus-stack label") +endpoint = (backend_sm.get("spec", {}).get("endpoints") or [{}])[0] +if endpoint.get("port") != "api" or endpoint.get("path") != "/metrics" or endpoint.get("interval") != "15s": + raise SystemExit("backend ServiceMonitor must scrape api /metrics every 15s") +if backend_sm.get("spec", {}).get("selector", {}).get("matchLabels", {}).get("app") != "backend": + raise SystemExit("backend ServiceMonitor must select app=backend") + +pod_monitors = list(yaml.safe_load_all((root / "datastore-pod-monitors.yaml").read_text())) +for monitor in pod_monitors: + name = monitor.get("metadata", {}).get("name") + if monitor.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": + raise SystemExit(f"{name} PodMonitor must keep release=kube-prometheus-stack label") + endpoint = (monitor.get("spec", {}).get("podMetricsEndpoints") or [{}])[0] + if endpoint.get("interval") != "30s": + raise SystemExit(f"{name} PodMonitor must scrape every 30s") +print("SCRAPE RESOURCE SHAPE OK") + kustomization = yaml.safe_load((root / "kustomization.yaml").read_text()) if "./alertmanager-config-email.example.yaml" in kustomization.get("resources", []): raise SystemExit("email fallback example must not be included in default kustomization") From 06b943dc7cf238c3cf6cf2cfc03a4dbea7123f89 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:06:33 +0900 Subject: [PATCH 022/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=EC=A0=80=EC=9E=A5?= =?UTF-8?q?=EC=86=8C=20=EC=9A=B4=EC=98=81=20=EC=A0=84=EC=A0=9C=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 18 +++++++++-- .../monitoring/grafana-dashboard-logs.yaml | 11 ++++++- kubernetes/monitoring/logs/README.md | 14 ++++++++- kubernetes/monitoring/validate.sh | 30 +++++++++++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index ff54af1a..6edd0d71 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -72,7 +72,17 @@ Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라 - 수집 namespace: `code-place-dev`, `code-place-prod`, `monitoring`. - Grafana datasource: 기존 kube-prometheus-stack Grafana에 `Loki` datasource를 추가합니다. -이 구성은 클라우드 없는 단기 retention 기준선입니다. prod 로그량이 커지거나 로그 저장소 장애 도메인을 더 분리해야 하면, Loki는 내부에서 별도 운영하는 S3 호환 object storage, 예를 들어 독립 MinIO cluster, 로 이전합니다. Grafana Loki chart의 내장 MinIO subchart는 신규 운영 의존성으로 사용하지 않습니다. +이 구성은 클라우드 없는 단기 retention 기준선입니다. Longhorn PVC는 hostPath/emptyDir보다 낫습니다. Kubernetes에서 lifecycle을 관리할 수 있고, PVC 사용률과 Longhorn volume health를 운영자가 볼 수 있기 때문입니다. 다만 durable log archive는 아니므로 장애 분석용 단기 저장소로만 취급합니다. + +현재 유지해야 하는 로그 저장소 불변 조건은 다음과 같습니다. + +- Loki deployment mode는 `Monolithic`입니다. +- Loki storage type은 `filesystem`입니다. +- Loki PVC storageClass는 `longhorn`, size는 `50Gi`입니다. +- Loki retention은 dev 72h, prod 168h입니다. +- Grafana Loki chart의 내장 MinIO subchart는 사용하지 않습니다. + +prod 로그량이 커지면 먼저 noisy log를 줄이거나 retention을 줄입니다. 그래도 부족하면 Longhorn replica 상태와 node 여유 disk를 확인한 뒤 PVC를 증설합니다. 로그 저장소 장애 도메인을 더 분리해야 하면, Loki는 내부에서 별도 운영하는 S3 호환 object storage, 예를 들어 독립 MinIO cluster, 로 이전합니다. ### Tracing @@ -169,8 +179,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 6. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. 8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, 최근 backend error, judge/celery log panel을 확인합니다. -9. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -10. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +9. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. +10. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +11. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. @@ -204,6 +215,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - PrometheusRule shape check: P0/P1 interval, alert priority/severity, summary/description, duplicate alert namespace label. - AlertmanagerConfig shape check: groupBy, P0/P1 Discord receivers, webhook Secret reference. - kube-prometheus-stack values shape check: Prometheus selector policy, AlertmanagerConfig selector, Loki datasource, dashboard sidecar label. + - Loki storage shape check: Monolithic mode, filesystem storage, Longhorn PVC, 50Gi size, dev/prod retention, MinIO disabled. - scrape resource shape check: ServiceMonitor/PodMonitor selector label, scrape path/port/interval. - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. - Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh` diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index c2ba5838..8dd1edc8 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -31,6 +31,15 @@ data: "text": "code-place-dev", "value": "code-place-dev" } + }, + { + "name": "request_id", + "type": "textbox", + "query": ".*", + "current": { + "text": ".*", + "value": ".*" + } } ] }, @@ -168,7 +177,7 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=~\"$namespace\"} |= \"request_id\""} + {"expr": "{namespace=~\"$namespace\"} | json | request_id=~\"$request_id\""} ] } ] diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 78bb4519..bc7fb307 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -7,6 +7,8 @@ This directory contains Helm values for the Kubernetes logs stack. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. - Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. +This is the selected baseline while cloud object storage is unavailable. Longhorn PVC is better than local hostPath or emptyDir because it gives Kubernetes-managed persistence and operationally visible volume health without introducing a new storage service. It is still not a durable log archive and should be treated as short-retention troubleshooting storage. + ## Install or Upgrade Install or upgrade kube-prometheus-stack first so the Prometheus Operator CRDs are present and the Grafana `Loki` datasource is provisioned. Then install Loki and Alloy. @@ -73,4 +75,14 @@ Grafana should also show: ## Notes -This is not a horizontally scalable Loki design. It is the selected cloud-free baseline for short retention. If production log volume grows or logs must survive a node/PVC failure domain, move prod to an internally operated S3-compatible object store such as a separately managed MinIO cluster. +This is not a horizontally scalable Loki design. It is the selected cloud-free baseline for short retention. + +Keep these invariants unless the storage design changes intentionally: + +- Loki stays in `Monolithic` deployment mode. +- Loki uses filesystem storage on a Longhorn PVC. +- The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. +- Retention stays short: dev 72h, prod 168h. +- The Loki chart's MinIO subchart stays disabled. If object storage becomes available, use a separately operated S3-compatible service and migrate Loki storage deliberately. + +If production log volume grows, first reduce noisy application logs or shorten retention. Increase the Longhorn PVC only after confirming available node storage and Longhorn replica health. If logs must survive a node/PVC failure domain, move prod to an internally operated S3-compatible object store such as a separately managed MinIO cluster. diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index a0fa0c92..4a898a55 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -145,6 +145,36 @@ if dashboard_sidecar.get("label") != "grafana_dashboard": raise SystemExit("Grafana dashboard sidecar must select grafana_dashboard label") print("KUBE-PROMETHEUS-STACK VALUES SHAPE OK") +loki_values = yaml.safe_load((root / "logs" / "loki-values.yaml").read_text()) +loki_config = loki_values.get("loki", {}) +if loki_values.get("deploymentMode") != "Monolithic": + raise SystemExit("Loki must stay in Monolithic mode for the on-prem baseline") +if loki_config.get("storage", {}).get("type") != "filesystem": + raise SystemExit("Loki must use filesystem storage while cloud object storage is unavailable") +if loki_config.get("commonConfig", {}).get("replication_factor") != 1: + raise SystemExit("Loki single-binary filesystem storage must keep replication_factor=1") +persistence = loki_values.get("singleBinary", {}).get("persistence", {}) +if not persistence.get("enabled"): + raise SystemExit("Loki singleBinary persistence must be enabled") +if persistence.get("storageClass") != "longhorn": + raise SystemExit("Loki persistence must use Longhorn storageClass") +if persistence.get("size") != "50Gi": + raise SystemExit("Loki PVC size must stay explicit at 50Gi") +limits = loki_config.get("limits_config", {}) +if limits.get("retention_period") != "168h": + raise SystemExit("Loki default retention must be 168h") +retention_by_selector = { + stream.get("selector"): stream.get("period") + for stream in limits.get("retention_stream", []) +} +if retention_by_selector.get('{namespace="code-place-dev"}') != "72h": + raise SystemExit("Loki dev retention must be 72h") +if retention_by_selector.get('{namespace="code-place-prod"}') != "168h": + raise SystemExit("Loki prod retention must be 168h") +if loki_values.get("minio", {}).get("enabled") is not False: + raise SystemExit("Loki chart MinIO subchart must stay disabled") +print("LOKI ON-PREM STORAGE SHAPE OK") + backend_sm = yaml.safe_load((root / "backend-service-monitor.yaml").read_text()) if backend_sm.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": raise SystemExit("backend ServiceMonitor must keep release=kube-prometheus-stack label") From b4edefa808e43e2613c8b80a29d046f7cc1c471e Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:08:03 +0900 Subject: [PATCH 023/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20CI=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/observability-validation.yml | 36 +++++++++++++++++++ .../observability-improvement-plan/_index.md | 2 ++ 2 files changed, 38 insertions(+) create mode 100644 .github/workflows/observability-validation.yml diff --git a/.github/workflows/observability-validation.yml b/.github/workflows/observability-validation.yml new file mode 100644 index 00000000..9da62936 --- /dev/null +++ b/.github/workflows/observability-validation.yml @@ -0,0 +1,36 @@ +name: Observability Validation + +on: + pull_request: + branches: + - develop + - main + paths: + - ".github/workflows/observability-validation.yml" + - "backend/**" + - "frontend/**" + - "kubernetes/**" + - "docs/content/developer/infra/observability-*/**" + workflow_dispatch: + +jobs: + validate-observability: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up kubectl + uses: azure/setup-kubectl@v4 + + - name: Set up Python YAML support + run: python3 -m pip install --user PyYAML + + - name: Validate monitoring bundle + run: bash kubernetes/monitoring/validate.sh + + - name: Render dev application manifests + run: kubectl kustomize kubernetes/overlays/dev >/tmp/codeplace-dev-render.yaml + + - name: Render prod application manifests + run: kubectl kustomize kubernetes/overlays/prod >/tmp/codeplace-prod-render.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 6edd0d71..2fa71a0a 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -218,6 +218,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Loki storage shape check: Monolithic mode, filesystem storage, Longhorn PVC, 50Gi size, dev/prod retention, MinIO disabled. - scrape resource shape check: ServiceMonitor/PodMonitor selector label, scrape path/port/interval. - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. +- GitHub Actions observability validation: `.github/workflows/observability-validation.yml` + - PR에서 `kubernetes/**`, 관측성 관련 backend/frontend/docs 변경 시 monitoring bundle validation과 dev/prod kustomize render를 실행합니다. - Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh` - Prometheus Operator CRD, monitoring namespace resource, selector label, webhook Secret, kube-prometheus-stack Pod readiness, optional Loki/Alloy, app namespace backend service/port/readiness를 읽기 전용으로 확인합니다. From 44969cb8190023beb8528260c6e0c6297f86582d Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:10:41 +0900 Subject: [PATCH 024/147] =?UTF-8?q?=EA=B9=83=ED=97=88=EB=B8=8C=20=EC=95=A1?= =?UTF-8?q?=EC=85=98=20=EC=A0=9C=EA=B1=B0=EC=99=80=20=EB=A9=94=ED=8A=B8?= =?UTF-8?q?=EB=A6=AD=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/observability-validation.yml | 36 ---------------- backend/utils/tests.py | 41 +++++++++++++++++++ .../observability-improvement-plan/_index.md | 2 - 3 files changed, 41 insertions(+), 38 deletions(-) delete mode 100644 .github/workflows/observability-validation.yml diff --git a/.github/workflows/observability-validation.yml b/.github/workflows/observability-validation.yml deleted file mode 100644 index 9da62936..00000000 --- a/.github/workflows/observability-validation.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Observability Validation - -on: - pull_request: - branches: - - develop - - main - paths: - - ".github/workflows/observability-validation.yml" - - "backend/**" - - "frontend/**" - - "kubernetes/**" - - "docs/content/developer/infra/observability-*/**" - workflow_dispatch: - -jobs: - validate-observability: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up kubectl - uses: azure/setup-kubectl@v4 - - - name: Set up Python YAML support - run: python3 -m pip install --user PyYAML - - - name: Validate monitoring bundle - run: bash kubernetes/monitoring/validate.sh - - - name: Render dev application manifests - run: kubectl kustomize kubernetes/overlays/dev >/tmp/codeplace-dev-render.yaml - - - name: Render prod application manifests - run: kubectl kustomize kubernetes/overlays/prod >/tmp/codeplace-prod-render.yaml diff --git a/backend/utils/tests.py b/backend/utils/tests.py index e988d040..042deea9 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -6,6 +6,7 @@ from django.test import SimpleTestCase, TestCase, override_settings from django.core.cache import cache from django.utils import timezone +from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily from utils.json_logging import CodePlaceJsonFormatter from utils.observability_metrics import CodePlaceCollector @@ -62,6 +63,46 @@ def test_judge_server_metrics_mark_stale_enabled_server_unavailable(self): self.assertGreaterEqual(heartbeat_age.value, 599) +class CodePlaceMetricsEndpointTest(SimpleTestCase): + + def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): + submission_metric = GaugeMetricFamily( + "codeplace_submission_status_count", + "Current number of submissions grouped by judge status.", + labels=["status"], + ) + submission_metric.add_metric(["accepted"], 0) + waiting_queue_metric = GaugeMetricFamily( + "codeplace_waiting_queue_length", + "Number of submissions waiting because no judge-server was available.", + ) + waiting_queue_metric.add_metric([], 0) + judge_metric = GaugeMetricFamily( + "codeplace_judge_server_available", + "Whether each judge-server is enabled and has a fresh heartbeat.", + labels=["hostname"], + ) + judge_metric.add_metric(["judge-1"], 1) + judge_duration_metric = HistogramMetricFamily( + "codeplace_submission_judge_duration_seconds", + "Judge duration for submissions completed in the last 10 minutes.", + ) + judge_duration_metric.add_metric([], [("+Inf", 0)], 0) + + with patch.object(CodePlaceCollector, "_submission_status_count", return_value=[submission_metric]), \ + patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ + patch.object(CodePlaceCollector, "_judge_server_metrics", return_value=[judge_metric]), \ + patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric): + response = self.client.get("/metrics") + + self.assertEqual(response.status_code, 200) + body = response.content.decode() + self.assertIn("django_http_requests_total_by_method_total", body) + self.assertIn("codeplace_submission_status_count", body) + self.assertIn("codeplace_waiting_queue_length", body) + self.assertIn("codeplace_judge_server_available", body) + + class ObservabilityTracingTest(SimpleTestCase): def test_get_tracer_returns_noop_tracer_when_opentelemetry_is_unavailable(self): diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 2fa71a0a..6edd0d71 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -218,8 +218,6 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Loki storage shape check: Monolithic mode, filesystem storage, Longhorn PVC, 50Gi size, dev/prod retention, MinIO disabled. - scrape resource shape check: ServiceMonitor/PodMonitor selector label, scrape path/port/interval. - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. -- GitHub Actions observability validation: `.github/workflows/observability-validation.yml` - - PR에서 `kubernetes/**`, 관측성 관련 backend/frontend/docs 변경 시 monitoring bundle validation과 dev/prod kustomize render를 실행합니다. - Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh` - Prometheus Operator CRD, monitoring namespace resource, selector label, webhook Secret, kube-prometheus-stack Pod readiness, optional Loki/Alloy, app namespace backend service/port/readiness를 읽기 전용으로 확인합니다. From d802e396b79a360a73f033fab177ca5eecb5001f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:22:25 +0900 Subject: [PATCH 025/147] =?UTF-8?q?=EC=98=A8=ED=94=84=EB=A0=88=EB=AF=B8?= =?UTF-8?q?=EC=8A=A4=20=ED=8A=B8=EB=A0=88=EC=9D=B4=EC=8B=B1=20=EC=8A=A4?= =?UTF-8?q?=ED=83=9D=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 33 ++-- .../infra/observability-runbook/_index.md | 41 +++++ .../monitoring/grafana-dashboard-traces.yaml | 146 ++++++++++++++++++ .../kube-prometheus-stack-values.yaml | 16 ++ kubernetes/monitoring/kustomization.yaml | 3 + kubernetes/monitoring/otel-collector.yaml | 143 +++++++++++++++++ kubernetes/monitoring/prometheus-rules.yaml | 30 ++++ kubernetes/monitoring/tempo.yaml | 142 +++++++++++++++++ .../dev/backend-deployment-patch.yaml | 2 + .../dev/celery-beat-deployment-patch.yaml | 2 + .../dev/celery-worker-deployment-patch.yaml | 2 + 11 files changed, 548 insertions(+), 12 deletions(-) create mode 100644 kubernetes/monitoring/grafana-dashboard-traces.yaml create mode 100644 kubernetes/monitoring/otel-collector.yaml create mode 100644 kubernetes/monitoring/tempo.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 6edd0d71..ce69648e 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -86,7 +86,7 @@ prod 로그량이 커지면 먼저 noisy log를 줄이거나 retention을 줄입 ### Tracing -OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OTEL_ENABLED=0`입니다. dev/staging에서 먼저 활성화하고 collector/Tempo/sampling 확인 후 prod 활성화를 결정합니다. +OpenTelemetry는 앱 초기화 코드에 내장되어 있고, base/prod 기본값은 `OTEL_ENABLED=0`입니다. dev overlay에서만 `OTEL_ENABLED=1`로 켜서 collector/Tempo/sampling을 먼저 확인합니다. 자동 계측 대상은 Django, requests, psycopg2, Redis, Celery입니다. 제출/채점 경로에는 manual span을 추가합니다. @@ -96,15 +96,16 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT 기본 sampling 값은 `OTEL_TRACES_SAMPLER_ARG=0.05`입니다. -현재 Kubernetes manifest에는 backend/celery의 `OTEL_EXPORTER_OTLP_ENDPOINT`가 `http://otel-collector.monitoring.svc.cluster.local:4317`로 잡혀 있습니다. 다만 `otel-collector`와 Tempo는 아직 배포 리소스로 포함하지 않습니다. 다음 조건이 정해지기 전에는 `OTEL_ENABLED=1`을 켜지 않습니다. +현재 Kubernetes manifest에는 backend/celery의 `OTEL_EXPORTER_OTLP_ENDPOINT`가 `http://otel-collector.monitoring.svc.cluster.local:4317`로 잡혀 있습니다. monitoring namespace에는 OTel Collector와 Tempo 단일 binary를 배포합니다. -- trace 저장 backend: Tempo 단일 binary, Tempo distributed, 외부 managed backend 중 선택. -- retention: dev/prod별 trace 보관 기간과 저장 용량. -- storage: filesystem, S3 호환 object storage, Longhorn PVC 중 선택. -- sampling: dev/prod sampling ratio와 error trace 우선 보존 정책. -- Grafana datasource: Tempo datasource 이름과 dashboard trace link 규칙. +- OTel Collector release: raw Kubernetes Deployment, namespace `monitoring`. +- Tempo deployment mode: single binary, namespace `monitoring`. +- Tempo storage: filesystem on Longhorn PVC. +- Tempo PVC: `tempo-data`, 20Gi. +- Tempo retention: 72h. +- Grafana datasource: 기존 kube-prometheus-stack Grafana에 `Tempo` datasource를 추가합니다. -이 조건이 정해진 뒤 `otel-collector`와 Tempo를 `kubernetes/monitoring` 하위 Helm values 또는 별도 kustomize 리소스로 추가합니다. +prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collector/Tempo 리소스 사용량을 확인한 뒤 별도 overlay에서 `OTEL_ENABLED=1`로 승격합니다. ## 3. Kubernetes Monitoring @@ -118,6 +119,9 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT - `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. - `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, 최근 backend error, judge/celery log 조회 dashboard. +- `otel-collector.yaml`: backend/celery OTLP trace를 받아 Tempo로 전송하는 OpenTelemetry Collector. +- `tempo.yaml`: Tempo single binary, Longhorn PVC 기반 trace 저장소. +- `grafana-dashboard-traces.yaml`: Collector/Tempo readiness, span ingest/export, Tempo PVC usage dashboard. - `validate.sh`: monitoring YAML, Grafana dashboard JSON, kustomize render, 선택적 promtool/Helm render 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. @@ -161,6 +165,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `LokiPVCAlmostFull`: Loki Longhorn PVC 사용률 85% 초과 10분 지속. - `AlloyDaemonSetUnavailable`: Alloy log collector가 모든 node에서 available하지 않음 5분 지속. - `LokiGatewayUnavailable`: Loki gateway Pod not ready 2분 지속. +- `OTelCollectorUnavailable`: OpenTelemetry Collector Pod not ready 2분 지속. +- `TempoUnavailable`: Tempo Pod not ready 2분 지속. +- `TempoPVCAlmostFull`: Tempo PVC 사용률 85% 초과 10분. - `CodePlaceNodePressure`: node memory/disk/PID pressure 5분 지속. ## 5. 운영 확인 절차 @@ -180,15 +187,17 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. 8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, 최근 backend error, judge/celery log panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. -10. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -11. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +10. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. +11. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +12. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. +13. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. - `/metrics`는 cluster 내부 scrape 전용이며 외부 Ingress에 노출하지 않습니다. - Discord webhook URL은 Git에 저장하지 않습니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. -- OpenTelemetry는 `OTEL_ENABLED=0` 기본값을 유지하고 dev/staging에서 먼저 활성화합니다. +- OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지하고 dev overlay에서 먼저 활성화합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - PostgreSQL은 CNPG instance exporter와 PodMonitor로 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor로 `redis_*` 지표를 수집합니다. @@ -196,7 +205,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Frontend는 axios 요청마다 `X-Request-ID`를 전파하고 마지막 request ID를 `window.__CODEPLACE_LAST_REQUEST_ID__`와 Sentry request context에 저장합니다. - Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. - Backend/Celery Sentry release는 `SENTRY_RELEASE` env가 있으면 사용합니다. 이미지 태그와 동일한 commit SHA를 넣어 Grafana/Loki/Sentry 간 배포 기준을 맞추는 것을 권장합니다. -- Tempo tracing은 저장소/retention 결정을 먼저 요구하므로 현재 PR에서는 켜지 않습니다. +- Tempo tracing은 monitoring namespace에 배포하지만, prod app trace export는 dev 확인 후 별도 승격합니다. ## 6. 검증 diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 6bfe0e7d..618d6684 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -56,6 +56,17 @@ kubectl -n monitoring logs deploy/loki-gateway --tail=100 kubectl -n monitoring logs daemonset/alloy --tail=100 ``` +Trace 상태 확인: + +```sh +kubectl -n monitoring get pod,svc,pvc | grep -E 'otel-collector|tempo' +kubectl -n monitoring logs deploy/otel-collector --tail=100 +kubectl -n monitoring logs deploy/tempo --tail=100 +kubectl -n code-place-dev get deploy backend celery-worker celery-beat -o yaml | grep -A1 OTEL_ENABLED +``` + +Grafana에서는 `CodePlace Traces` dashboard와 `Tempo` datasource Explore를 확인합니다. dev에서 먼저 `service.name=codeplace-backend`와 `service.name=codeplace-celery` trace가 조회되어야 합니다. + ## P0 ### BackendTargetDown @@ -253,6 +264,36 @@ kubectl -n monitoring describe pvc -l app.kubernetes.io/name=loki ## P1 +### OTelCollectorUnavailable / TempoUnavailable / TempoPVCAlmostFull + +확인: + +```sh +kubectl -n monitoring get pod,svc,pvc | grep -E 'otel-collector|tempo' +kubectl -n monitoring describe pod -l app=otel-collector +kubectl -n monitoring describe pod -l app=tempo +kubectl -n monitoring logs deploy/otel-collector --tail=200 +kubectl -n monitoring logs deploy/tempo --tail=200 +kubectl -n monitoring describe pvc tempo-data +``` + +PromQL: + +```promql +kube_pod_status_ready{namespace="monitoring", pod=~"otel-collector-.*|tempo-.*", condition="true"} +max(kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) / +max(kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) +sum(rate(otelcol_receiver_accepted_spans_total{namespace="monitoring"}[5m])) +sum(rate(otelcol_exporter_send_failed_spans_total{namespace="monitoring"}[5m])) +``` + +판단: + +- Collector가 down이면 dev backend/celery trace export가 실패합니다. 앱 요청 처리는 계속되지만 trace 관측이 끊깁니다. +- Tempo가 down이면 Collector가 span export 실패를 기록합니다. +- Tempo PVC가 85%를 넘으면 retention 단축, noisy trace sampling 조정, PVC 증설 중 하나를 먼저 결정합니다. +- prod는 기본 `OTEL_ENABLED=0`입니다. prod trace가 없다고 해서 장애는 아니며, dev에서 Collector/Tempo를 먼저 확인한 뒤 prod 승격 여부를 결정합니다. + ### ApiLatencyHigh 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-traces.yaml b/kubernetes/monitoring/grafana-dashboard-traces.yaml new file mode 100644 index 00000000..20363275 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-traces.yaml @@ -0,0 +1,146 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-traces + labels: + grafana_dashboard: "1" +data: + codeplace-traces.json: | + { + "uid": "codeplace-traces", + "title": "CodePlace Traces", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "traces"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "Tempo Ready", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"tempo-.*\"})", "legendFormat": "tempo"} + ] + }, + { + "type": "stat", + "title": "OTel Collector Ready", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"otel-collector-.*\"})", "legendFormat": "otel-collector"} + ] + }, + { + "type": "stat", + "title": "Trace Receive Rate", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "spans/s"} + ] + }, + { + "type": "stat", + "title": "Trace Export Failures", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(increase(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "failed"} + ] + }, + { + "type": "stat", + "title": "Tempo PVC Usage", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}), 1)", "legendFormat": "usage"} + ] + }, + { + "type": "timeseries", + "title": "Collector Spans", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (receiver, transport) (rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "{{receiver}} {{transport}} accepted"}, + {"expr": "sum by (exporter) (rate(otelcol_exporter_sent_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "{{exporter}} sent"} + ] + }, + { + "type": "timeseries", + "title": "Collector Refused or Failed Spans", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum(rate(otelcol_receiver_refused_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "refused"}, + {"expr": "sum(rate(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "failed"} + ] + } + ] + } diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index 590eb083..1bfe4d4c 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -18,12 +18,28 @@ alertmanager: grafana: additionalDataSources: - name: Loki + uid: Loki type: loki access: proxy url: http://loki-gateway.monitoring.svc.cluster.local isDefault: false jsonData: timeout: 60 + - name: Tempo + uid: Tempo + type: tempo + access: proxy + url: http://tempo.monitoring.svc.cluster.local:3200 + isDefault: false + jsonData: + tracesToLogsV2: + datasourceUid: Loki + spanStartTimeShift: "-5m" + spanEndTimeShift: "5m" + filterByTraceID: false + filterBySpanID: false + customQuery: true + query: "{namespace=~\"code-place-dev|code-place-prod\"} |= \"${__span.traceId}\"" ingress: enabled: true ingressClassName: traefik diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index cf1ad8d6..fa7734b0 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -8,4 +8,7 @@ resources: - ./datastore-pod-monitors.yaml - ./grafana-dashboard-codeplace.yaml - ./grafana-dashboard-logs.yaml + - ./grafana-dashboard-traces.yaml + - ./otel-collector.yaml - ./prometheus-rules.yaml + - ./tempo.yaml diff --git a/kubernetes/monitoring/otel-collector.yaml b/kubernetes/monitoring/otel-collector.yaml new file mode 100644 index 00000000..3975b52e --- /dev/null +++ b/kubernetes/monitoring/otel-collector.yaml @@ -0,0 +1,143 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + processors: + memory_limiter: + check_interval: 5s + limit_mib: 384 + spike_limit_mib: 128 + batch: + timeout: 5s + send_batch_size: 512 + + exporters: + otlp/tempo: + endpoint: tempo.monitoring.svc.cluster.local:4317 + tls: + insecure: true + + extensions: + health_check: + endpoint: 0.0.0.0:13133 + + service: + extensions: + - health_check + pipelines: + traces: + receivers: + - otlp + processors: + - memory_limiter + - batch + exporters: + - otlp/tempo +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-collector + labels: + app: otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app: otel-collector + template: + metadata: + labels: + app: otel-collector + spec: + containers: + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.128.0 + args: + - --config=/etc/otelcol/config.yaml + ports: + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + - name: health + containerPort: 13133 + - name: metrics + containerPort: 8888 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi + readinessProbe: + httpGet: + path: / + port: health + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + httpGet: + path: / + port: health + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + volumeMounts: + - name: config + mountPath: /etc/otelcol + volumes: + - name: config + configMap: + name: otel-collector-config +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + labels: + app: otel-collector +spec: + selector: + app: otel-collector + ports: + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + - name: otlp-http + port: 4318 + targetPort: otlp-http + - name: health + port: 13133 + targetPort: health + - name: metrics + port: 8888 + targetPort: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: otel-collector + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: otel-collector + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index d932ab67..1522ef94 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -272,6 +272,36 @@ spec: annotations: summary: "Loki gateway is not ready" description: "The Loki gateway has not been ready for 2 minutes. Alloy writes and Grafana Loki queries may fail." + - alert: OTelCollectorUnavailable + expr: (max by (namespace, pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "OpenTelemetry Collector is not ready" + description: "The OpenTelemetry Collector has not been ready for 2 minutes. Enabled services cannot export traces." + - alert: TempoUnavailable + expr: (max by (namespace, pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Tempo is not ready" + description: "Tempo has not been ready for 2 minutes. Trace queries and trace ingestion may fail." + - alert: TempoPVCAlmostFull + expr: max(kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}), 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Tempo PVC usage is above 85%" + description: "The Tempo Longhorn PVC has been above 85% usage for 10 minutes. Trace retention or PVC size must be adjusted." - alert: CodePlaceNodePressure expr: kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure", status="true"} == 1 for: 5m diff --git a/kubernetes/monitoring/tempo.yaml b/kubernetes/monitoring/tempo.yaml new file mode 100644 index 00000000..050328af --- /dev/null +++ b/kubernetes/monitoring/tempo.yaml @@ -0,0 +1,142 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo-config +data: + tempo.yaml: | + server: + http_listen_port: 3200 + + distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + ingester: + max_block_duration: 5m + + compactor: + compaction: + block_retention: 72h + + storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tempo-data +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 20Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tempo + labels: + app: tempo +spec: + replicas: 1 + selector: + matchLabels: + app: tempo + template: + metadata: + labels: + app: tempo + spec: + containers: + - name: tempo + image: grafana/tempo:2.8.1 + args: + - -config.file=/etc/tempo/tempo.yaml + ports: + - name: http + containerPort: 3200 + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + readinessProbe: + httpGet: + path: /ready + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + volumeMounts: + - name: config + mountPath: /etc/tempo + - name: data + mountPath: /var/tempo + volumes: + - name: config + configMap: + name: tempo-config + - name: data + persistentVolumeClaim: + claimName: tempo-data +--- +apiVersion: v1 +kind: Service +metadata: + name: tempo + labels: + app: tempo +spec: + selector: + app: tempo + ports: + - name: http + port: 3200 + targetPort: http + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + - name: otlp-http + port: 4318 + targetPort: otlp-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: tempo + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: tempo + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index 5bdbadfe..7a98516d 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -19,3 +19,5 @@ spec: value: "dev" - name: SENTRY_RELEASE value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" + - name: OTEL_ENABLED + value: "1" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index 12692cc1..3712eff3 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -22,3 +22,5 @@ spec: value: "dev" - name: SENTRY_RELEASE value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" + - name: OTEL_ENABLED + value: "1" diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index fd033aa5..0a8b217a 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -25,3 +25,5 @@ spec: value: "dev" - name: SENTRY_RELEASE value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" + - name: OTEL_ENABLED + value: "1" From 6b66ee5a9c5f35f1ef7ac3652ea3cb484d1c864c Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:24:50 +0900 Subject: [PATCH 026/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=EC=99=80=20=ED=8A=B8?= =?UTF-8?q?=EB=A0=88=EC=9D=B4=EC=8A=A4=20=EC=83=81=EA=B4=80=EA=B4=80?= =?UTF-8?q?=EA=B3=84=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/json_logging.py | 2 ++ backend/utils/observability_tracing.py | 18 ++++++++++ backend/utils/tests.py | 33 +++++++++++++++++++ .../observability-improvement-plan/_index.md | 4 ++- .../infra/observability-runbook/_index.md | 6 ++++ 5 files changed, 62 insertions(+), 1 deletion(-) diff --git a/backend/utils/json_logging.py b/backend/utils/json_logging.py index 6468c0c3..ae02f5fb 100644 --- a/backend/utils/json_logging.py +++ b/backend/utils/json_logging.py @@ -3,6 +3,7 @@ from datetime import datetime, timezone from utils.observability_context import get_request_id +from utils.observability_tracing import get_current_trace_context SENSITIVE_EXACT_FIELDS = { @@ -37,6 +38,7 @@ def format(self, record): "message": record.getMessage(), "request_id": getattr(record, "request_id", None) or get_request_id(), } + payload.update(get_current_trace_context()) for key in ( "method", "path", diff --git a/backend/utils/observability_tracing.py b/backend/utils/observability_tracing.py index 26ad9182..eab2c9fb 100644 --- a/backend/utils/observability_tracing.py +++ b/backend/utils/observability_tracing.py @@ -26,6 +26,24 @@ def get_tracer(name): return trace.get_tracer(name) +def get_current_trace_context(): + try: + from opentelemetry import trace + except Exception: + return {} + + try: + span_context = trace.get_current_span().get_span_context() + except Exception: + return {} + if not getattr(span_context, "is_valid", False): + return {} + return { + "trace_id": format(span_context.trace_id, "032x"), + "span_id": format(span_context.span_id, "016x"), + } + + def configure_opentelemetry(service_name): if get_env("OTEL_ENABLED", "0").lower() not in ("1", "true", "yes", "on"): return diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 042deea9..0738a374 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -119,6 +119,19 @@ def fake_import(name, *args, **kwargs): with tracer.start_as_current_span("span") as span: self.assertIsNone(span.set_attribute("key", "value")) + def test_get_current_trace_context_returns_empty_when_opentelemetry_is_unavailable(self): + from utils.observability_tracing import get_current_trace_context + + real_import = __import__ + + def fake_import(name, *args, **kwargs): + if name == "opentelemetry": + raise ImportError("missing opentelemetry") + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=fake_import): + self.assertEqual(get_current_trace_context(), {}) + class CodePlaceJsonFormatterTest(SimpleTestCase): @@ -144,6 +157,26 @@ def test_preserves_status_code_and_redacts_sensitive_fields(self): self.assertNotIn("token", payload) self.assertNotIn("source_code", payload) + def test_includes_trace_context_when_available(self): + record = logging.LogRecord( + name="codeplace.request", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="request completed", + args=(), + exc_info=None, + ) + + with patch("utils.json_logging.get_current_trace_context", return_value={ + "trace_id": "0" * 31 + "1", + "span_id": "0" * 15 + "2", + }): + payload = json.loads(CodePlaceJsonFormatter().format(record)) + + self.assertEqual(payload["trace_id"], "0" * 31 + "1") + self.assertEqual(payload["span_id"], "0" * 15 + "2") + def test_redacts_nested_sensitive_fields(self): formatter = CodePlaceJsonFormatter() payload = { diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index ce69648e..502d9441 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -58,8 +58,10 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 - `remote_addr` - `task_id` - `submission_id` +- `trace_id` +- `span_id` -민감 정보는 로그에 싣지 않습니다. token, password, cookie, OAuth secret, authorization header, 제출 source code는 저장 대상이 아닙니다. +민감 정보는 로그에 싣지 않습니다. token, password, cookie, OAuth secret, authorization header, 제출 source code는 저장 대상이 아닙니다. OpenTelemetry가 활성화된 요청/작업 로그에는 현재 span의 `trace_id`, `span_id`를 추가해 Tempo trace와 Loki JSON log를 연결합니다. frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 618d6684..741c4293 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -66,6 +66,12 @@ kubectl -n code-place-dev get deploy backend celery-worker celery-beat -o yaml | ``` Grafana에서는 `CodePlace Traces` dashboard와 `Tempo` datasource Explore를 확인합니다. dev에서 먼저 `service.name=codeplace-backend`와 `service.name=codeplace-celery` trace가 조회되어야 합니다. +Tempo에서 trace를 선택한 뒤 같은 `trace_id`를 Loki에서 조회합니다. + +```logql +{namespace="", app="backend"} | json | trace_id="" +{namespace="", container=~"celery-worker|celery-beat"} | json | trace_id="" +``` ## P0 From 3d39fb8dc5cfff7ccfc273e00a5402ca9cf90436 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:31:33 +0900 Subject: [PATCH 027/147] =?UTF-8?q?Celery=20=EC=9E=91=EC=97=85=20=EA=B4=80?= =?UTF-8?q?=EC=B8=A1=20=EC=A7=80=ED=91=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/oj/celery.py | 2 + backend/utils/celery_observability.py | 97 ++++++++++++++++ backend/utils/json_logging.py | 3 + backend/utils/observability_metrics.py | 106 ++++++++++++++++++ backend/utils/tests.py | 72 +++++++++++- .../observability-improvement-plan/_index.md | 10 +- .../infra/observability-runbook/_index.md | 24 ++++ .../grafana-dashboard-codeplace.yaml | 73 ++++++++---- kubernetes/monitoring/prometheus-rules.yaml | 27 +++++ 9 files changed, 393 insertions(+), 21 deletions(-) create mode 100644 backend/utils/celery_observability.py diff --git a/backend/oj/celery.py b/backend/oj/celery.py index b9890384..f5ca162b 100644 --- a/backend/oj/celery.py +++ b/backend/oj/celery.py @@ -1,6 +1,7 @@ import os import celery +from utils.celery_observability import configure_celery_observability from utils.observability_tracing import configure_opentelemetry os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oj.settings') @@ -9,3 +10,4 @@ app = celery.Celery('scheduler') app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks() +configure_celery_observability() diff --git a/backend/utils/celery_observability.py b/backend/utils/celery_observability.py new file mode 100644 index 00000000..d4f62aa3 --- /dev/null +++ b/backend/utils/celery_observability.py @@ -0,0 +1,97 @@ +import logging +import time + +from celery import signals + +from utils.cache import cache + +logger = logging.getLogger("codeplace.celery_task") + +CELERY_TASK_TOTAL_KEY = "observability:celery:task_total" +CELERY_TASK_RUNTIME_BUCKET_KEY = "observability:celery:task_runtime_bucket" +CELERY_TASK_RUNTIME_COUNT_KEY = "observability:celery:task_runtime_count" +CELERY_TASK_RUNTIME_SUM_KEY = "observability:celery:task_runtime_sum" +CELERY_TASK_RUNTIME_LAST_KEY = "observability:celery:task_runtime_last" +CELERY_TASK_RUNTIME_BUCKETS = (1, 3, 5, 10, 30, 60, 120, 300, 600, float("inf")) + +_task_start_times = {} +_configured = False + + +def configure_celery_observability(): + global _configured + if _configured: + return + signals.task_prerun.connect(_on_task_prerun, weak=False) + signals.task_postrun.connect(_on_task_postrun, weak=False) + signals.task_retry.connect(_on_task_retry, weak=False) + _configured = True + + +def _task_name(task): + return getattr(task, "name", None) or getattr(task, "__name__", None) or "unknown" + + +def _field(*parts): + return "|".join(str(part) for part in parts) + + +def _hincrbyfloat(key, field, amount): + client = cache.client.get_client(write=True) + return client.hincrbyfloat(key, field, amount) + + +def _hsetfloat(key, field, value): + client = cache.client.get_client(write=True) + return client.hset(key, field, value) + + +def _record_task(task_name, status, duration_seconds=None): + status = (status or "unknown").lower() + try: + cache.hincrby(CELERY_TASK_TOTAL_KEY, _field(task_name, status), 1) + if duration_seconds is not None: + _hincrbyfloat(CELERY_TASK_RUNTIME_SUM_KEY, task_name, duration_seconds) + cache.hincrby(CELERY_TASK_RUNTIME_COUNT_KEY, task_name, 1) + _hsetfloat(CELERY_TASK_RUNTIME_LAST_KEY, _field(task_name, status), duration_seconds) + for bucket in CELERY_TASK_RUNTIME_BUCKETS: + if duration_seconds <= bucket: + bucket_label = "+Inf" if bucket == float("inf") else str(bucket) + cache.hincrby(CELERY_TASK_RUNTIME_BUCKET_KEY, _field(task_name, bucket_label), 1) + except Exception: + logger.exception("failed to record celery task metric", extra={ + "task_name": task_name, + "task_status": status, + }) + + +def _on_task_prerun(task_id=None, task=None, **kwargs): + if task_id: + _task_start_times[task_id] = time.monotonic() + + +def _on_task_postrun(task_id=None, task=None, state=None, **kwargs): + task_name = _task_name(task) + started_at = _task_start_times.pop(task_id, None) + duration_seconds = None + if started_at is not None: + duration_seconds = max(time.monotonic() - started_at, 0) + _record_task(task_name, state, duration_seconds) + logger.info("celery task completed", extra={ + "task_id": task_id, + "task_name": task_name, + "task_status": (state or "unknown").lower(), + "duration_ms": round(duration_seconds * 1000, 2) if duration_seconds is not None else None, + }) + + +def _on_task_retry(request=None, reason=None, **kwargs): + task_name = getattr(request, "task", None) or "unknown" + task_id = getattr(request, "id", None) + _record_task(task_name, "retry") + logger.warning("celery task retry", extra={ + "task_id": task_id, + "task_name": task_name, + "task_status": "retry", + "retry_reason": str(reason) if reason else None, + }) diff --git a/backend/utils/json_logging.py b/backend/utils/json_logging.py index ae02f5fb..62b213ad 100644 --- a/backend/utils/json_logging.py +++ b/backend/utils/json_logging.py @@ -47,7 +47,10 @@ def format(self, record): "user_id", "remote_addr", "task_id", + "task_name", + "task_status", "submission_id", + "retry_reason", ): value = getattr(record, key, None) if value is not None: diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index e5114494..4835ab94 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -8,6 +8,14 @@ from submission.models import JudgeStatus, Submission from utils.cache import cache +from utils.celery_observability import ( + CELERY_TASK_RUNTIME_BUCKET_KEY, + CELERY_TASK_RUNTIME_BUCKETS, + CELERY_TASK_RUNTIME_COUNT_KEY, + CELERY_TASK_RUNTIME_LAST_KEY, + CELERY_TASK_RUNTIME_SUM_KEY, + CELERY_TASK_TOTAL_KEY, +) from utils.constants import CacheKey logger = logging.getLogger(__name__) @@ -71,12 +79,28 @@ def describe(self): "codeplace_submission_judge_duration_seconds", "Judge duration for submissions completed in the last 10 minutes.", ) + yield GaugeMetricFamily( + "codeplace_celery_task_count", + "Total Celery tasks grouped by task name and status.", + labels=["task_name", "status"], + ) + yield HistogramMetricFamily( + "codeplace_celery_task_runtime_seconds", + "Celery task runtime grouped by task name.", + labels=["task_name"], + ) + yield GaugeMetricFamily( + "codeplace_celery_task_last_runtime_seconds", + "Latest observed Celery task runtime grouped by task name and status.", + labels=["task_name", "status"], + ) def collect(self): yield from self._submission_status_count() yield self._waiting_queue_length() yield from self._judge_server_metrics() yield self._judge_duration_histogram() + yield from self._celery_task_metrics() def _submission_status_count(self): metric = GaugeMetricFamily( @@ -167,6 +191,88 @@ def _judge_duration_histogram(self): metric.add_metric([], cumulative_buckets, sum(durations)) return metric + def _celery_task_metrics(self): + total_metric = GaugeMetricFamily( + "codeplace_celery_task_count", + "Total Celery tasks grouped by task name and status.", + labels=["task_name", "status"], + ) + runtime_metric = HistogramMetricFamily( + "codeplace_celery_task_runtime_seconds", + "Celery task runtime grouped by task name.", + labels=["task_name"], + ) + last_runtime_metric = GaugeMetricFamily( + "codeplace_celery_task_last_runtime_seconds", + "Latest observed Celery task runtime grouped by task name and status.", + labels=["task_name", "status"], + ) + try: + task_totals = self._redis_hash(CELERY_TASK_TOTAL_KEY) + runtime_buckets = self._redis_hash(CELERY_TASK_RUNTIME_BUCKET_KEY) + runtime_counts = self._redis_hash(CELERY_TASK_RUNTIME_COUNT_KEY) + runtime_sums = self._redis_hash(CELERY_TASK_RUNTIME_SUM_KEY) + last_runtimes = self._redis_hash(CELERY_TASK_RUNTIME_LAST_KEY) + except Exception as e: + logger.warning("Failed to collect Celery task metrics: %s", e) + yield total_metric + yield runtime_metric + yield last_runtime_metric + return + + for field, value in task_totals.items(): + task_name, status = self._split_field(field, 2) + total_metric.add_metric([task_name, status], float(value)) + + task_names = set(runtime_counts) | set(runtime_sums) + for field in runtime_buckets: + task_name, _ = self._split_field(field, 2) + task_names.add(task_name) + for task_name in sorted(task_names): + cumulative_buckets = [] + for bucket in CELERY_TASK_RUNTIME_BUCKETS: + bucket_label = "+Inf" if bucket == float("inf") else str(bucket) + value = float(runtime_buckets.get(self._join_field(task_name, bucket_label), 0)) + cumulative_buckets.append((bucket_label, value)) + runtime_metric.add_metric( + [task_name], + cumulative_buckets, + float(runtime_sums.get(task_name, 0)), + ) + + for field, value in last_runtimes.items(): + task_name, status = self._split_field(field, 2) + last_runtime_metric.add_metric([task_name, status], float(value)) + + yield total_metric + yield runtime_metric + yield last_runtime_metric + + @staticmethod + def _redis_hash(key): + values = cache.hgetall(key) or {} + return { + CodePlaceCollector._decode(field): CodePlaceCollector._decode(value) + for field, value in values.items() + } + + @staticmethod + def _decode(value): + if isinstance(value, bytes): + return value.decode() + return str(value) + + @staticmethod + def _split_field(field, expected_parts): + parts = field.split("|", expected_parts - 1) + if len(parts) != expected_parts: + return tuple(parts + ["unknown"] * (expected_parts - len(parts))) + return tuple(parts) + + @staticmethod + def _join_field(*parts): + return "|".join(str(part) for part in parts) + def register_codeplace_metrics(): if getattr(register_codeplace_metrics, "_registered", False): diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 0738a374..bae756c0 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -8,6 +8,7 @@ from django.utils import timezone from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily +from utils import celery_observability from utils.json_logging import CodePlaceJsonFormatter from utils.observability_metrics import CodePlaceCollector from utils.observability_tracing import get_tracer @@ -62,6 +63,74 @@ def test_judge_server_metrics_mark_stale_enabled_server_unavailable(self): self.assertEqual(available.value, 0) self.assertGreaterEqual(heartbeat_age.value, 599) + def test_celery_task_metrics_are_collected_from_redis(self): + hashes = { + celery_observability.CELERY_TASK_TOTAL_KEY: { + b"judge.tasks.judge_task|success": b"3", + b"judge.tasks.judge_task|failure": b"1", + }, + celery_observability.CELERY_TASK_RUNTIME_BUCKET_KEY: { + b"judge.tasks.judge_task|1": b"2", + b"judge.tasks.judge_task|3": b"3", + b"judge.tasks.judge_task|+Inf": b"3", + }, + celery_observability.CELERY_TASK_RUNTIME_COUNT_KEY: { + b"judge.tasks.judge_task": b"3", + }, + celery_observability.CELERY_TASK_RUNTIME_SUM_KEY: { + b"judge.tasks.judge_task": b"4.5", + }, + celery_observability.CELERY_TASK_RUNTIME_LAST_KEY: { + b"judge.tasks.judge_task|success": b"1.5", + }, + } + + with patch("utils.observability_metrics.cache.hgetall", side_effect=lambda key: hashes.get(key, {})): + samples = self._samples_by_metric(list(CodePlaceCollector()._celery_task_metrics())) + + total_samples = samples["codeplace_celery_task_count"] + self.assertEqual(total_samples[0].labels, { + "task_name": "judge.tasks.judge_task", + "status": "success", + }) + self.assertEqual(total_samples[0].value, 3) + runtime_samples = samples["codeplace_celery_task_runtime_seconds"] + bucket_samples = [sample for sample in runtime_samples if sample.name.endswith("_bucket")] + self.assertTrue(any(sample.labels["le"] == "+Inf" and sample.value == 3 for sample in bucket_samples)) + self.assertTrue(any(sample.name.endswith("_sum") and sample.value == 4.5 for sample in runtime_samples)) + last_runtime = samples["codeplace_celery_task_last_runtime_seconds"][0] + self.assertEqual(last_runtime.value, 1.5) + + +class CeleryObservabilityTest(SimpleTestCase): + + def test_record_task_writes_total_and_runtime_hashes(self): + redis_client = MagicMock() + with patch("utils.celery_observability.cache.hincrby") as hincrby, \ + patch("utils.celery_observability.cache.client.get_client", return_value=redis_client): + celery_observability._record_task("judge.tasks.judge_task", "success", 2.5) + + hincrby.assert_any_call( + celery_observability.CELERY_TASK_TOTAL_KEY, + "judge.tasks.judge_task|success", + 1, + ) + hincrby.assert_any_call( + celery_observability.CELERY_TASK_RUNTIME_COUNT_KEY, + "judge.tasks.judge_task", + 1, + ) + redis_client.hincrbyfloat.assert_any_call( + celery_observability.CELERY_TASK_RUNTIME_SUM_KEY, + "judge.tasks.judge_task", + 2.5, + ) + redis_client.hset.assert_any_call( + celery_observability.CELERY_TASK_RUNTIME_LAST_KEY, + "judge.tasks.judge_task|success", + 2.5, + ) + class CodePlaceMetricsEndpointTest(SimpleTestCase): @@ -92,7 +161,8 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): with patch.object(CodePlaceCollector, "_submission_status_count", return_value=[submission_metric]), \ patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ patch.object(CodePlaceCollector, "_judge_server_metrics", return_value=[judge_metric]), \ - patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric): + patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric), \ + patch.object(CodePlaceCollector, "_celery_task_metrics", return_value=[]): response = self.client.get("/metrics") self.assertEqual(response.status_code, 200) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 502d9441..83dd6cf8 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -36,6 +36,9 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_judge_server_available{hostname}` - `codeplace_judge_server_last_heartbeat_age_seconds{hostname}` - `codeplace_judge_server_task_number{hostname}` +- `codeplace_celery_task_count{task_name,status}` +- `codeplace_celery_task_runtime_seconds{task_name}` +- `codeplace_celery_task_last_runtime_seconds{task_name,status}` `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. @@ -58,6 +61,8 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 - `remote_addr` - `task_id` - `submission_id` +- `task_name` +- `task_status` - `trace_id` - `span_id` @@ -156,6 +161,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. - `CeleryBeatDown`: beat Pod not ready 2분 지속. +- `CeleryTaskFailures`: task failure 발생 2분 지속. +- `CeleryTaskRetries`: task retry 3회 초과/10분, 5분 지속. +- `CeleryTaskRuntimeHigh`: task p95 runtime 120초 초과 10분 지속. - `PodCrashLooping`: 주요 Pod restart 증가 5분 지속. - `CodePlacePodNotReady`: 주요 앱/DB/Redis Pod not ready 5분 지속. - `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. @@ -186,7 +194,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 5. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. -7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. +7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. 8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, 최근 backend error, judge/celery log panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 10. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 741c4293..d80d088e 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -339,6 +339,30 @@ Grafana Explore: {namespace="", container=~"celery-worker|celery-beat"} ``` +### CeleryTaskFailures / CeleryTaskRetries / CeleryTaskRuntimeHigh + +확인: + +```promql +sum by (task_name, status) (increase(codeplace_celery_task_count{namespace=""}[10m])) +histogram_quantile(0.95, sum by (task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=""}[10m]))) +codeplace_celery_task_last_runtime_seconds{namespace=""} +``` + +Grafana Explore: + +```logql +{namespace="", container=~"celery-worker|celery-beat"} | json | task_status=~"failure|retry" +{namespace="", container=~"celery-worker|celery-beat"} | json | task_name="" +``` + +판단: + +- 특정 task만 실패하면 최근 코드 변경, 입력 데이터, 외부 의존성을 우선 확인합니다. +- runtime만 증가하면 DB/Redis 지연, lock, worker CPU/memory, queue backlog를 같이 봅니다. +- `judge.tasks.judge_task` 실패와 `JudgeWaitingQueueBacklog`가 같이 발생하면 judge-server availability와 Redis `waiting_queue`를 우선 확인합니다. +- task metric은 worker가 Redis에 누적하고 backend `/metrics` collector가 노출합니다. worker 로그에는 `task_id`, `task_name`, `task_status`, `duration_ms`, `trace_id`가 포함됩니다. + ### PodCrashLooping / CodePlacePodNotReady 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 9b8302c1..a6dc577f 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -184,10 +184,45 @@ data: {"expr": "codeplace_submission_status_count{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{status}}"} ] }, + { + "type": "row", + "title": "Async Jobs", + "gridPos": {"x": 0, "y": 12, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "Celery Task Throughput", + "gridPos": {"x": 0, "y": 13, "w": 8, "h": 7}, + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (namespace, task_name, status) (rate(codeplace_celery_task_count{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{task_name}} {{status}}"} + ] + }, + { + "type": "timeseries", + "title": "Celery Task Runtime p95", + "gridPos": {"x": 8, "y": 13, "w": 8, "h": 7}, + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (namespace, task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} {{task_name}} p95"} + ] + }, + { + "type": "bargauge", + "title": "Celery Last Runtime", + "gridPos": {"x": 16, "y": 13, "w": 8, "h": 7}, + "fieldConfig": {"defaults": {"unit": "s"}}, + "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace_celery_task_last_runtime_seconds{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{task_name}} {{status}}"} + ] + }, { "type": "timeseries", "title": "Judge Duration p95", - "gridPos": {"x": 0, "y": 12, "w": 8, "h": 8}, + "gridPos": {"x": 0, "y": 20, "w": 8, "h": 8}, "fieldConfig": {"defaults": {"unit": "s"}}, "targets": [ {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_submission_judge_duration_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} judge p95"} @@ -196,7 +231,7 @@ data: { "type": "timeseries", "title": "Judge Heartbeat Max Age", - "gridPos": {"x": 8, "y": 12, "w": 8, "h": 8}, + "gridPos": {"x": 8, "y": 20, "w": 8, "h": 8}, "fieldConfig": { "defaults": { "unit": "s", @@ -217,7 +252,7 @@ data: { "type": "timeseries", "title": "Judge Server Load", - "gridPos": {"x": 16, "y": 12, "w": 8, "h": 8}, + "gridPos": {"x": 16, "y": 20, "w": 8, "h": 8}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "topk(10, codeplace_judge_server_task_number{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{hostname}} tasks"} @@ -226,14 +261,14 @@ data: { "type": "row", "title": "Platform", - "gridPos": {"x": 0, "y": 20, "w": 24, "h": 1}, + "gridPos": {"x": 0, "y": 28, "w": 24, "h": 1}, "collapsed": false, "panels": [] }, { "type": "stat", "title": "Pod Ready", - "gridPos": {"x": 0, "y": 21, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 29, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -254,7 +289,7 @@ data: { "type": "timeseries", "title": "Pod Restarts", - "gridPos": {"x": 6, "y": 21, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -263,7 +298,7 @@ data: { "type": "timeseries", "title": "Container CPU Usage", - "gridPos": {"x": 12, "y": 21, "w": 6, "h": 5}, + "gridPos": {"x": 12, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "cores"}}, "targets": [ {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} @@ -272,7 +307,7 @@ data: { "type": "timeseries", "title": "Container Memory Usage", - "gridPos": {"x": 18, "y": 21, "w": 6, "h": 5}, + "gridPos": {"x": 18, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} @@ -281,7 +316,7 @@ data: { "type": "timeseries", "title": "PVC Usage", - "gridPos": {"x": 0, "y": 26, "w": 12, "h": 6}, + "gridPos": {"x": 0, "y": 34, "w": 12, "h": 6}, "fieldConfig": { "defaults": { "unit": "percentunit", @@ -302,7 +337,7 @@ data: { "type": "timeseries", "title": "Node Pressure", - "gridPos": {"x": 12, "y": 26, "w": 12, "h": 6}, + "gridPos": {"x": 12, "y": 34, "w": 12, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "kube_node_status_condition{condition=~\"MemoryPressure|DiskPressure|PIDPressure\", status=\"true\"}", "legendFormat": "{{node}} {{condition}}"} @@ -311,14 +346,14 @@ data: { "type": "row", "title": "Data Stores", - "gridPos": {"x": 0, "y": 32, "w": 24, "h": 1}, + "gridPos": {"x": 0, "y": 40, "w": 24, "h": 1}, "collapsed": false, "panels": [] }, { "type": "stat", "title": "PostgreSQL Ready", - "gridPos": {"x": 0, "y": 33, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 41, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -339,7 +374,7 @@ data: { "type": "stat", "title": "Redis Ready", - "gridPos": {"x": 6, "y": 33, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 41, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -360,7 +395,7 @@ data: { "type": "stat", "title": "PostgreSQL Exporter Up", - "gridPos": {"x": 12, "y": 33, "w": 6, "h": 5}, + "gridPos": {"x": 12, "y": 41, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -381,7 +416,7 @@ data: { "type": "stat", "title": "Redis Exporter Up", - "gridPos": {"x": 18, "y": 33, "w": 6, "h": 5}, + "gridPos": {"x": 18, "y": 41, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -402,7 +437,7 @@ data: { "type": "timeseries", "title": "PostgreSQL Restarts", - "gridPos": {"x": 0, "y": 38, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 46, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -411,7 +446,7 @@ data: { "type": "timeseries", "title": "Redis Restarts", - "gridPos": {"x": 6, "y": 38, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 46, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -420,7 +455,7 @@ data: { "type": "timeseries", "title": "PostgreSQL WAL Bytes", - "gridPos": {"x": 12, "y": 38, "w": 6, "h": 5}, + "gridPos": {"x": 12, "y": 46, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "Bps"}}, "targets": [ {"expr": "sum by (namespace) (rate(cnpg_collector_wal_bytes{namespace=~\"$namespace\", cluster=\"postgres\"}[5m]))", "legendFormat": "{{namespace}}"} @@ -429,7 +464,7 @@ data: { "type": "timeseries", "title": "Redis Memory Usage", - "gridPos": {"x": 18, "y": 38, "w": 6, "h": 5}, + "gridPos": {"x": 18, "y": 46, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} used"}, diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 1522ef94..d27bd890 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -170,6 +170,33 @@ spec: annotations: summary: "Celery beat is not ready" description: "The celery-beat pod has not been ready for 2 minutes." + - alert: CeleryTaskFailures + expr: sum by (namespace, task_name) (increase(codeplace_celery_task_count{namespace=~"code-place-(dev|prod)", status="failure"}[10m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "Celery task failures detected" + description: "A Celery task has failed in the last 10 minutes." + - alert: CeleryTaskRetries + expr: sum by (namespace, task_name) (increase(codeplace_celery_task_count{namespace=~"code-place-(dev|prod)", status="retry"}[10m])) > 3 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Celery task retries are elevated" + description: "A Celery task retried more than 3 times in 10 minutes." + - alert: CeleryTaskRuntimeHigh + expr: histogram_quantile(0.95, sum by (namespace, task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=~"code-place-(dev|prod)"}[10m]))) > 120 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "Celery task runtime is high" + description: "Celery task p95 runtime has been above 120 seconds for 10 minutes." - alert: PodCrashLooping expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*"}[5m])) > 0 for: 5m From eb7f405eb5fded196d0efa58322c23eb873a7982 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:33:47 +0900 Subject: [PATCH 028/147] =?UTF-8?q?Celery=20=EC=9E=91=EC=97=85=20request?= =?UTF-8?q?=20ID=20=EC=A0=84=ED=8C=8C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/celery_observability.py | 21 ++++++++++++++ backend/utils/tests.py | 28 +++++++++++++++++++ .../observability-improvement-plan/_index.md | 2 +- .../infra/observability-runbook/_index.md | 1 + 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/backend/utils/celery_observability.py b/backend/utils/celery_observability.py index d4f62aa3..1b09ccc0 100644 --- a/backend/utils/celery_observability.py +++ b/backend/utils/celery_observability.py @@ -4,6 +4,7 @@ from celery import signals from utils.cache import cache +from utils.observability_context import get_request_id, reset_request_id, set_request_id logger = logging.getLogger("codeplace.celery_task") @@ -15,6 +16,7 @@ CELERY_TASK_RUNTIME_BUCKETS = (1, 3, 5, 10, 30, 60, 120, 300, 600, float("inf")) _task_start_times = {} +_request_id_tokens = {} _configured = False @@ -22,6 +24,7 @@ def configure_celery_observability(): global _configured if _configured: return + signals.before_task_publish.connect(_on_before_task_publish, weak=False) signals.task_prerun.connect(_on_task_prerun, weak=False) signals.task_postrun.connect(_on_task_postrun, weak=False) signals.task_retry.connect(_on_task_retry, weak=False) @@ -65,9 +68,24 @@ def _record_task(task_name, status, duration_seconds=None): }) +def _on_before_task_publish(headers=None, **kwargs): + request_id = get_request_id() + if request_id and headers is not None: + headers["x-request-id"] = request_id + + +def _request_headers(task): + request = getattr(task, "request", None) + headers = getattr(request, "headers", None) or {} + return headers if isinstance(headers, dict) else {} + + def _on_task_prerun(task_id=None, task=None, **kwargs): if task_id: _task_start_times[task_id] = time.monotonic() + request_id = _request_headers(task).get("x-request-id") + if request_id and task_id: + _request_id_tokens[task_id] = set_request_id(request_id) def _on_task_postrun(task_id=None, task=None, state=None, **kwargs): @@ -83,6 +101,9 @@ def _on_task_postrun(task_id=None, task=None, state=None, **kwargs): "task_status": (state or "unknown").lower(), "duration_ms": round(duration_seconds * 1000, 2) if duration_seconds is not None else None, }) + token = _request_id_tokens.pop(task_id, None) + if token is not None: + reset_request_id(token) def _on_task_retry(request=None, reason=None, **kwargs): diff --git a/backend/utils/tests.py b/backend/utils/tests.py index bae756c0..f0c23377 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -10,6 +10,7 @@ from utils import celery_observability from utils.json_logging import CodePlaceJsonFormatter +from utils.observability_context import get_request_id, reset_request_id, set_request_id from utils.observability_metrics import CodePlaceCollector from utils.observability_tracing import get_tracer from utils.testcase_cache import TestCaseCacheManager @@ -104,6 +105,10 @@ def test_celery_task_metrics_are_collected_from_redis(self): class CeleryObservabilityTest(SimpleTestCase): + def tearDown(self): + celery_observability._task_start_times.clear() + celery_observability._request_id_tokens.clear() + def test_record_task_writes_total_and_runtime_hashes(self): redis_client = MagicMock() with patch("utils.celery_observability.cache.hincrby") as hincrby, \ @@ -131,6 +136,29 @@ def test_record_task_writes_total_and_runtime_hashes(self): 2.5, ) + def test_before_task_publish_propagates_current_request_id(self): + token = set_request_id("request-123") + headers = {} + try: + celery_observability._on_before_task_publish(headers=headers) + finally: + reset_request_id(token) + + self.assertEqual(headers["x-request-id"], "request-123") + + def test_task_prerun_and_postrun_restore_request_id_context(self): + task = MagicMock() + task.name = "judge.tasks.judge_task" + task.request.headers = {"x-request-id": "request-456"} + + with patch("utils.celery_observability._record_task"), \ + patch("utils.celery_observability.logger"): + celery_observability._on_task_prerun(task_id="task-1", task=task) + self.assertEqual(get_request_id(), "request-456") + celery_observability._on_task_postrun(task_id="task-1", task=task, state="SUCCESS") + + self.assertIsNone(get_request_id()) + class CodePlaceMetricsEndpointTest(SimpleTestCase): diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 83dd6cf8..ae88e770 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -44,7 +44,7 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 ### Logs -backend는 `X-Request-ID`를 수용하고, 없으면 request ID를 생성한 뒤 응답 header로 반환합니다. Frontend axios 요청은 `X-Request-ID`를 생성해 backend로 전달하고, backend가 반환한 request ID를 axios response/error와 Sentry request context에 보존합니다. +backend는 `X-Request-ID`를 수용하고, 없으면 request ID를 생성한 뒤 응답 header로 반환합니다. Frontend axios 요청은 `X-Request-ID`를 생성해 backend로 전달하고, backend가 반환한 request ID를 axios response/error와 Sentry request context에 보존합니다. API 요청에서 Celery task를 enqueue할 때 request ID를 Celery header로 전파하고, worker 실행 중 같은 request ID를 JSON log context에 복원합니다. Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설정합니다. JSON 로그 필드는 다음을 기준으로 합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index d80d088e..5c46da92 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -354,6 +354,7 @@ Grafana Explore: ```logql {namespace="", container=~"celery-worker|celery-beat"} | json | task_status=~"failure|retry" {namespace="", container=~"celery-worker|celery-beat"} | json | task_name="" +{namespace="", container=~"celery-worker|celery-beat"} | json | request_id="" ``` 판단: From aac001e09fb6ddf8a9fe044dba9aeb17ee1077c5 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:37:20 +0900 Subject: [PATCH 029/147] =?UTF-8?q?=ED=94=84=EB=A1=A0=ED=8A=B8=20nginx=20?= =?UTF-8?q?=EC=9A=94=EC=B2=AD=20=EB=A1=9C=EA=B7=B8=20=EC=83=81=EA=B4=80?= =?UTF-8?q?=EA=B4=80=EA=B3=84=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 4 +- .../infra/observability-runbook/_index.md | 2 + frontend/deploy/kube_nginx/README.md | 6 +++ frontend/deploy/kube_nginx/api_proxy.conf | 1 + frontend/deploy/kube_nginx/nginx.conf | 26 ++++++++++-- .../monitoring/grafana-dashboard-logs.yaml | 40 ++++++++++++++++++- 6 files changed, 73 insertions(+), 6 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index ae88e770..9fb7d3cc 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -68,7 +68,7 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 민감 정보는 로그에 싣지 않습니다. token, password, cookie, OAuth secret, authorization header, 제출 source code는 저장 대상이 아닙니다. OpenTelemetry가 활성화된 요청/작업 로그에는 현재 span의 `trace_id`, `span_id`를 추가해 Tempo trace와 Loki JSON log를 연결합니다. -frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. +frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. access log는 Loki에서 파싱 가능한 JSON 형식이며, `request_id`, `method`, `path`, `status_code`, `duration_seconds`, `remote_addr`, `user_agent`, `x_forwarded_for`를 포함합니다. nginx는 들어온 `X-Request-ID`를 유지하고, 없으면 nginx `$request_id`를 생성해 응답 header와 backend proxy header로 전달합니다. Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 운영 조건에서는 Loki를 Monolithic mode로 배포하고 Longhorn PVC 기반 filesystem storage를 사용합니다. @@ -195,7 +195,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. -8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, 최근 backend error, judge/celery log panel을 확인합니다. +8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, 최근 backend error, frontend 4xx/5xx, judge/celery log panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 10. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. 11. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 5c46da92..3dfd6197 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -44,6 +44,7 @@ Loki 로그 확인: ```logql {namespace=""} +{namespace="", app="frontend"} | json {namespace="", app="backend"} | json {namespace="", container=~"backend|celery-worker|judge-server"} ``` @@ -114,6 +115,7 @@ kubectl -n describe pod -l app=backend Grafana Explore: ```logql +{namespace="", app="frontend"} | json | status_code >= 500 {namespace="", app="backend"} | json | status_code >= 500 {namespace="", app="backend"} |= "" ``` diff --git a/frontend/deploy/kube_nginx/README.md b/frontend/deploy/kube_nginx/README.md index 2bf65c87..0d58fbef 100644 --- a/frontend/deploy/kube_nginx/README.md +++ b/frontend/deploy/kube_nginx/README.md @@ -8,4 +8,10 @@ K3s로 마이그레이션한 후, TLS (Transport Layer Security) 종료에 대 따라서 이 Nginx 인스턴스는 더 이상 TLS 인증 및 종료를 수행하지 않습니다. 대신, Traefik이 모든 인바운드 TLS 트래픽을 처리한 후 암호화되지 않은 HTTP 트래픽을 이 Nginx 서비스로 프록시합니다. +### 관측성 + +Kubernetes용 Nginx는 access log를 JSON 형식으로 `/dev/stdout`에 출력하고 error log를 `/dev/stderr`에 출력합니다. +들어온 `X-Request-ID`가 있으면 그대로 유지하고, 없으면 Nginx `$request_id`를 생성해 응답 header와 backend proxy header로 전달합니다. +이 값은 frontend nginx log, backend JSON log, Celery task log를 Loki에서 같은 `request_id`로 조회하는 데 사용합니다. + 참고로, TLS 처리를 포함했던 이전 Nginx 설정은 `frontend/deploy/nginx/` 디렉토리에서 확인할 수 있습니다. diff --git a/frontend/deploy/kube_nginx/api_proxy.conf b/frontend/deploy/kube_nginx/api_proxy.conf index 4310b9c8..edf00cd0 100644 --- a/frontend/deploy/kube_nginx/api_proxy.conf +++ b/frontend/deploy/kube_nginx/api_proxy.conf @@ -1,6 +1,7 @@ proxy_pass http://backend; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $http_host; +proxy_set_header X-Request-ID $codeplace_request_id; client_max_body_size 200M; proxy_http_version 1.1; proxy_set_header Connection ''; diff --git a/frontend/deploy/kube_nginx/nginx.conf b/frontend/deploy/kube_nginx/nginx.conf index 5f21ba6c..110a502a 100644 --- a/frontend/deploy/kube_nginx/nginx.conf +++ b/frontend/deploy/kube_nginx/nginx.conf @@ -21,9 +21,28 @@ http { gzip_vary on; gzip_types application/javascript text/css; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; + map $http_x_request_id $codeplace_request_id { + default $http_x_request_id; + "" $request_id; + } + + log_format main escape=json + '{' + '"timestamp":"$time_iso8601",' + '"level":"INFO",' + '"logger":"frontend.nginx",' + '"message":"request completed",' + '"request_id":"$codeplace_request_id",' + '"method":"$request_method",' + '"path":"$uri",' + '"status_code":$status,' + '"duration_seconds":$request_time,' + '"remote_addr":"$remote_addr",' + '"body_bytes_sent":$body_bytes_sent,' + '"referer":"$http_referer",' + '"user_agent":"$http_user_agent",' + '"x_forwarded_for":"$http_x_forwarded_for"' + '}'; access_log /dev/stdout main; @@ -35,6 +54,7 @@ http { add_header X-XSS-Protection "1; mode=block" always; add_header X-Frame-Options SAMEORIGIN always; add_header X-Content-Type-Options nosniff always; + add_header X-Request-ID $codeplace_request_id always; server { listen 8000 default_server; diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index 8dd1edc8..20ad7596 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -135,6 +135,29 @@ data: {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"backend\"} | json | level=\"ERROR\" [5m]))", "legendFormat": "errors"} ] }, + { + "type": "stat", + "title": "Frontend 5xx", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Loki", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 500 [5m]))", "legendFormat": "5xx"} + ] + }, { "type": "logs", "title": "Recent Backend Errors", @@ -165,10 +188,25 @@ data: {"expr": "{namespace=~\"$namespace\", container=~\"celery-worker|celery-beat|judge-server\"}"} ] }, + { + "type": "logs", + "title": "Frontend 4xx/5xx Logs", + "gridPos": {"x": 0, "y": 14, "w": 24, "h": 8}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 400"} + ] + }, { "type": "logs", "title": "Request ID Lookup", - "gridPos": {"x": 0, "y": 14, "w": 24, "h": 9}, + "gridPos": {"x": 0, "y": 22, "w": 24, "h": 9}, "datasource": "Loki", "options": { "showLabels": false, From 0f14c152459867e0745b8ee61dcd926e5719d7a2 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:40:49 +0900 Subject: [PATCH 030/147] =?UTF-8?q?Ingress=20=EB=A1=9C=EA=B7=B8=20?= =?UTF-8?q?=EC=88=98=EC=A7=91=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 6 ++-- .../infra/observability-runbook/_index.md | 9 ++++++ .../monitoring/grafana-dashboard-logs.yaml | 32 ++++++++++++++++--- kubernetes/monitoring/logs/README.md | 3 +- kubernetes/monitoring/logs/alloy-values.yaml | 7 ++-- kubernetes/setup/traefik/traefik-config.yaml | 6 ++++ 6 files changed, 53 insertions(+), 10 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 9fb7d3cc..f45ce379 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -68,6 +68,8 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 민감 정보는 로그에 싣지 않습니다. token, password, cookie, OAuth secret, authorization header, 제출 source code는 저장 대상이 아닙니다. OpenTelemetry가 활성화된 요청/작업 로그에는 현재 span의 `trace_id`, `span_id`를 추가해 Tempo trace와 Loki JSON log를 연결합니다. +Traefik은 JSON access log를 출력하고, `X-Request-ID`, `X-Forwarded-For` header만 keep합니다. Alloy는 `kube-system` 전체가 아니라 Traefik Pod 로그만 추가 수집합니다. + frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. access log는 Loki에서 파싱 가능한 JSON 형식이며, `request_id`, `method`, `path`, `status_code`, `duration_seconds`, `remote_addr`, `user_agent`, `x_forwarded_for`를 포함합니다. nginx는 들어온 `X-Request-ID`를 유지하고, 없으면 nginx `$request_id`를 생성해 응답 header와 backend proxy header로 전달합니다. Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 운영 조건에서는 Loki를 Monolithic mode로 배포하고 Longhorn PVC 기반 filesystem storage를 사용합니다. @@ -76,7 +78,7 @@ Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라 - Alloy release: `alloy`, namespace: `monitoring`. - Loki storage: `filesystem` on Longhorn PVC. - Loki retention: `code-place-dev` 3일, `code-place-prod` 7일. -- 수집 namespace: `code-place-dev`, `code-place-prod`, `monitoring`. +- 수집 대상: `code-place-dev`, `code-place-prod`, `monitoring`, `kube-system`의 Traefik Pod. - Grafana datasource: 기존 kube-prometheus-stack Grafana에 `Loki` datasource를 추가합니다. 이 구성은 클라우드 없는 단기 retention 기준선입니다. Longhorn PVC는 hostPath/emptyDir보다 낫습니다. Kubernetes에서 lifecycle을 관리할 수 있고, PVC 사용률과 Longhorn volume health를 운영자가 볼 수 있기 때문입니다. 다만 durable log archive는 아니므로 장애 분석용 단기 저장소로만 취급합니다. @@ -195,7 +197,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. -8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, 최근 backend error, frontend 4xx/5xx, judge/celery log panel을 확인합니다. +8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, judge/celery log panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 10. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. 11. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 3dfd6197..9c68d776 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -44,6 +44,7 @@ Loki 로그 확인: ```logql {namespace=""} +{namespace="kube-system", app_kubernetes_io_name="traefik"} | json {namespace="", app="frontend"} | json {namespace="", app="backend"} | json {namespace="", container=~"backend|celery-worker|judge-server"} @@ -141,6 +142,14 @@ sum by (exported_service, code) (rate(traefik_service_requests_total{exported_se ```sh kubectl -n get ingress,svc,endpoints kubectl -n kube-system logs deploy/traefik --tail=200 +kubectl -n kube-system get helmchartconfig traefik -o yaml +``` + +Grafana Explore: + +```logql +{namespace="kube-system", app_kubernetes_io_name="traefik"} | json | DownstreamStatus >= 500 +{namespace="kube-system", app_kubernetes_io_name="traefik"} |= "" ``` 판단: diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index 20ad7596..0d86af4b 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -158,10 +158,33 @@ data: {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 500 [5m]))", "legendFormat": "5xx"} ] }, + { + "type": "stat", + "title": "Ingress 5xx", + "gridPos": {"x": 0, "y": 4, "w": 4, "h": 4}, + "datasource": "Loki", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(count_over_time({namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | DownstreamStatus >= 500 [5m]))", "legendFormat": "5xx"} + ] + }, { "type": "logs", "title": "Recent Backend Errors", - "gridPos": {"x": 0, "y": 4, "w": 12, "h": 10}, + "gridPos": {"x": 4, "y": 4, "w": 10, "h": 10}, "datasource": "Loki", "options": { "showLabels": false, @@ -176,7 +199,7 @@ data: { "type": "logs", "title": "Judge and Celery Logs", - "gridPos": {"x": 12, "y": 4, "w": 12, "h": 10}, + "gridPos": {"x": 14, "y": 4, "w": 10, "h": 10}, "datasource": "Loki", "options": { "showLabels": false, @@ -190,7 +213,7 @@ data: }, { "type": "logs", - "title": "Frontend 4xx/5xx Logs", + "title": "Ingress and Frontend 4xx/5xx Logs", "gridPos": {"x": 0, "y": 14, "w": 24, "h": 8}, "datasource": "Loki", "options": { @@ -200,7 +223,8 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 400"} + {"expr": "{namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | DownstreamStatus >= 400", "legendFormat": "traefik"}, + {"expr": "{namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 400", "legendFormat": "frontend"} ] }, { diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index bc7fb307..9eec1bba 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -3,7 +3,7 @@ This directory contains Helm values for the Kubernetes logs stack. - Loki: monolithic mode, filesystem storage, Longhorn PVC. -- Alloy: DaemonSet log collector for `code-place-dev`, `code-place-prod`, and `monitoring`. +- Alloy: DaemonSet log collector for `code-place-dev`, `code-place-prod`, `monitoring`, and Traefik pods in `kube-system`. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. - Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. @@ -66,6 +66,7 @@ Grafana should show a `Loki` datasource. Useful Explore queries: {namespace="code-place-dev"} {namespace="code-place-prod", app="backend"} | json {namespace="code-place-dev", container=~"backend|celery-worker|judge-server"} +{namespace="kube-system", app_kubernetes_io_name="traefik"} | json | DownstreamStatus >= 400 ``` Grafana should also show: diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml index 8484a4fc..93644695 100644 --- a/kubernetes/monitoring/logs/alloy-values.yaml +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -1,5 +1,5 @@ # Helm values for grafana/alloy. -# Alloy runs as a DaemonSet and collects logs from CodePlace and monitoring pods. +# Alloy runs as a DaemonSet and collects CodePlace, monitoring, and Traefik logs. controller: type: daemonset @@ -26,8 +26,9 @@ alloy: targets = discovery.kubernetes.pod.targets rule { - source_labels = ["__meta_kubernetes_namespace"] - regex = "code-place-dev|code-place-prod|monitoring" + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name"] + separator = "/" + regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/traefik" action = "keep" } diff --git a/kubernetes/setup/traefik/traefik-config.yaml b/kubernetes/setup/traefik/traefik-config.yaml index 0278c1dc..15094644 100644 --- a/kubernetes/setup/traefik/traefik-config.yaml +++ b/kubernetes/setup/traefik/traefik-config.yaml @@ -9,6 +9,12 @@ spec: - "--certificatesresolvers.default.acme.email=" - "--certificatesresolvers.default.acme.storage=/data/acme.json" - "--certificatesresolvers.default.acme.tlschallenge=true" # 443 포트를 통해 인증 + - "--accesslog=true" + - "--accesslog.format=json" + - "--accesslog.fields.defaultmode=keep" + - "--accesslog.fields.headers.defaultmode=drop" + - "--accesslog.fields.headers.names.X-Request-ID=keep" + - "--accesslog.fields.headers.names.X-Forwarded-For=keep" service: enabled: true From 798a0d914be65b648036b9989ba2774aaa045408 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:47:48 +0900 Subject: [PATCH 031/147] =?UTF-8?q?Longhorn=20=EC=8A=A4=ED=86=A0=EB=A6=AC?= =?UTF-8?q?=EC=A7=80=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 28 +- .../infra/observability-runbook/_index.md | 70 ++++- .../monitoring/grafana-dashboard-storage.yaml | 243 ++++++++++++++++++ kubernetes/monitoring/kustomization.yaml | 2 + .../monitoring/longhorn-service-monitor.yaml | 18 ++ kubernetes/monitoring/prometheus-rules.yaml | 80 ++++++ kubernetes/setup/longhorn/README.md | 19 ++ 7 files changed, 455 insertions(+), 5 deletions(-) create mode 100644 kubernetes/monitoring/grafana-dashboard-storage.yaml create mode 100644 kubernetes/monitoring/longhorn-service-monitor.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index f45ce379..bf4ab49d 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -17,7 +17,7 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 - **Submission/Judge Path:** submission 생성, Celery `judge_task`, Redis `waiting_queue`, judge-server heartbeat, judge-server `/judge` 호출, 결과 저장. - **Async Jobs:** celery-worker, celery-beat, scheduled tasks. - **Data Stores:** PostgreSQL/CNPG, Redis Sentinel. -- **Platform:** backend/frontend/celery/judge Pod, node, PVC. +- **Platform/Storage:** backend/frontend/celery/judge Pod, node, PVC, Longhorn volume/node/disk. - **Client/Error:** frontend runtime error, backend exception, request ID 기반 로그. ## 2. 구현 내용 @@ -93,6 +93,14 @@ Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라 prod 로그량이 커지면 먼저 noisy log를 줄이거나 retention을 줄입니다. 그래도 부족하면 Longhorn replica 상태와 node 여유 disk를 확인한 뒤 PVC를 증설합니다. 로그 저장소 장애 도메인을 더 분리해야 하면, Loki는 내부에서 별도 운영하는 S3 호환 object storage, 예를 들어 독립 MinIO cluster, 로 이전합니다. +Longhorn 자체도 kube-prometheus-stack에서 scrape합니다. `longhorn-system` namespace의 `longhorn-manager` Service를 ServiceMonitor로 연결하고, Grafana `CodePlace Storage` dashboard에서 다음 항목을 봅니다. + +- Longhorn manager ready count. +- faulted/degraded/read-only volume count. +- volume robustness와 actual/capacity size. +- Longhorn node/disk 사용률. +- Longhorn manager CPU/memory. + ### Tracing OpenTelemetry는 앱 초기화 코드에 내장되어 있고, base/prod 기본값은 `OTEL_ENABLED=0`입니다. dev overlay에서만 `OTEL_ENABLED=1`로 켜서 collector/Tempo/sampling을 먼저 확인합니다. @@ -131,6 +139,8 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `otel-collector.yaml`: backend/celery OTLP trace를 받아 Tempo로 전송하는 OpenTelemetry Collector. - `tempo.yaml`: Tempo single binary, Longhorn PVC 기반 trace 저장소. - `grafana-dashboard-traces.yaml`: Collector/Tempo readiness, span ingest/export, Tempo PVC usage dashboard. +- `longhorn-service-monitor.yaml`: Longhorn manager `/metrics` scrape, interval 30s. +- `grafana-dashboard-storage.yaml`: Longhorn volume/node/disk 상태와 storage capacity dashboard. - `validate.sh`: monitoring YAML, Grafana dashboard JSON, kustomize render, 선택적 promtool/Helm render 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. @@ -154,6 +164,9 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `PostgresExporterDown`: CNPG PostgreSQL exporter metric 수집 실패 2분 지속. - `RedisExporterDown`: Redis exporter metric 수집 실패 2분 지속. - `LokiUnavailable`: Loki single-binary Pod not ready 1분 지속. +- `LonghornManagerDown`: Longhorn manager 전부 not ready 2분 지속. +- `LonghornVolumeFaulted`: Longhorn volume faulted 1분 지속. +- `LonghornVolumeReadOnly`: Longhorn volume filesystem read-only 1분 지속. ### P1 @@ -180,6 +193,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `OTelCollectorUnavailable`: OpenTelemetry Collector Pod not ready 2분 지속. - `TempoUnavailable`: Tempo Pod not ready 2분 지속. - `TempoPVCAlmostFull`: Tempo PVC 사용률 85% 초과 10분. +- `LonghornVolumeDegraded`: Longhorn volume degraded 5분 지속. +- `LonghornNodeNotReady`: Longhorn node not ready 5분 지속. +- `LonghornDiskNotReady`: Longhorn disk not ready 5분 지속. +- `LonghornNodeStorageHigh`: Longhorn node storage 사용/예약률 85% 초과 10분. +- `LonghornDiskUsageHigh`: Longhorn disk 사용/예약률 85% 초과 10분. - `CodePlaceNodePressure`: node memory/disk/PID pressure 5분 지속. ## 5. 운영 확인 절차 @@ -200,9 +218,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, judge/celery log panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 10. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. -11. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -12. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. -13. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +11. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. +12. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +13. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. +14. Prometheus target에서 `longhorn` ServiceMonitor가 healthy인지 확인합니다. +15. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 9c68d776..1a5e5236 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -17,11 +17,13 @@ weight: 6 bash kubernetes/monitoring/validate.sh kubectl get ns kubectl -n monitoring get prometheusrule,alertmanagerconfig,servicemonitor -kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy' +kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy|otel|tempo' +kubectl -n longhorn-system get pod,svc ``` Grafana에서는 `CodePlace Overview` dashboard에서 `namespace`를 알림의 namespace로 맞춥니다. 로그 수집이나 로그 기반 확인이 필요하면 `CodePlace Logs` dashboard를 함께 봅니다. +PVC/volume 문제가 의심되면 `CodePlace Storage` dashboard에서 Longhorn 상태를 먼저 확인합니다. Prometheus target 확인: @@ -29,6 +31,7 @@ Prometheus target 확인: Status -> Targets -> backend Status -> Targets -> postgres Status -> Targets -> redis +Status -> Targets -> longhorn ``` Alertmanager 확인: @@ -75,6 +78,20 @@ Tempo에서 trace를 선택한 뒤 같은 `trace_id`를 Loki에서 조회합니 {namespace="", container=~"celery-worker|celery-beat"} | json | trace_id="" ``` +Longhorn 상태 확인: + +```sh +kubectl -n longhorn-system get pod,svc +kubectl -n monitoring get servicemonitor longhorn -o yaml +``` + +```promql +longhorn_volume_robustness +longhorn_volume_file_system_read_only +(longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1) +(longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1) +``` + ## P0 ### BackendTargetDown @@ -279,6 +296,32 @@ kubectl -n monitoring describe pvc -l app.kubernetes.io/name=loki - readiness만 실패하면 Loki config, retention, disk full, compactor error를 확인합니다. - Loki가 내려가도 앱 요청 처리는 계속되지만 로그 수집/조회가 끊기므로 빠르게 복구합니다. +### LonghornManagerDown / LonghornVolumeFaulted / LonghornVolumeReadOnly + +증상: Longhorn manager가 전부 ready가 아니거나, volume이 faulted/read-only 상태입니다. + +확인: + +```sh +kubectl -n longhorn-system get pod,svc +kubectl -n longhorn-system describe pod -l app=longhorn-manager +kubectl -n longhorn-system logs -l app=longhorn-manager --tail=200 +``` + +```promql +sum(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) +longhorn_volume_robustness +longhorn_volume_file_system_read_only +longhorn_volume_actual_size_bytes +longhorn_volume_capacity_bytes +``` + +판단: + +- manager가 전부 down이면 Longhorn UI/API와 metrics가 끊깁니다. node pressure, image pull, network, Longhorn system Pod 상태를 먼저 봅니다. +- volume faulted는 PVC-backed workload의 storage availability 문제입니다. affected `pvc_namespace`/`pvc`를 찾아 해당 workload 상태와 Longhorn replica 상태를 확인합니다. +- read-only volume은 쓰기 실패로 바로 이어질 수 있습니다. Loki/Tempo/PostgreSQL/Redis PVC가 대상이면 해당 관측/데이터 경로 장애로 취급합니다. + ## P1 ### OTelCollectorUnavailable / TempoUnavailable / TempoPVCAlmostFull @@ -436,6 +479,31 @@ kubectl -n get pvc kubectl -n describe pvc ``` +### LonghornVolumeDegraded / LonghornNodeNotReady / LonghornDiskNotReady / LonghornNodeStorageHigh / LonghornDiskUsageHigh + +확인: + +```sh +kubectl -n longhorn-system get pod +kubectl get node +kubectl describe node +``` + +```promql +longhorn_volume_robustness == 2 +longhorn_node_status{condition="ready"} +longhorn_disk_status{condition="ready"} +(longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1) +(longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1) +``` + +판단: + +- degraded volume은 replica rebuild 중이거나 node/disk 여유가 부족한 상태일 수 있습니다. +- node/disk not ready는 앱 Pod 장애보다 먼저 storage layer 문제로 봅니다. +- storage usage가 85%를 넘으면 Loki/Tempo retention 축소, noisy 로그/trace 감소, Longhorn disk/PVC 증설 중 하나를 결정합니다. + 클라우드 object storage가 없는 현재 조건에서는 Longhorn 여유 용량이 로그/트레이스 retention의 상한입니다. + ### LokiPVCAlmostFull 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-storage.yaml b/kubernetes/monitoring/grafana-dashboard-storage.yaml new file mode 100644 index 00000000..e12e121e --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-storage.yaml @@ -0,0 +1,243 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-storage + labels: + grafana_dashboard: "1" +data: + codeplace-storage.json: | + { + "uid": "codeplace-storage", + "title": "CodePlace Storage", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "storage", "longhorn"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "Longhorn Managers Ready", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"longhorn-system\", condition=\"true\", pod=~\"longhorn-manager-.*\"})", "legendFormat": "ready"} + ] + }, + { + "type": "stat", + "title": "Faulted Volumes", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "count(longhorn_volume_robustness == 3) or vector(0)", "legendFormat": "faulted"} + ] + }, + { + "type": "stat", + "title": "Degraded Volumes", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "count(longhorn_volume_robustness == 2) or vector(0)", "legendFormat": "degraded"} + ] + }, + { + "type": "stat", + "title": "Read-only Volumes", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(longhorn_volume_file_system_read_only) or vector(0)", "legendFormat": "read-only"} + ] + }, + { + "type": "stat", + "title": "Max Node Storage", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max((longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1))", "legendFormat": "usage"} + ] + }, + { + "type": "stat", + "title": "Max Disk Usage", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max((longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1))", "legendFormat": "usage"} + ] + }, + { + "type": "timeseries", + "title": "Volume Robustness", + "description": "1 healthy, 2 degraded, 3 faulted", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}}, + "targets": [ + {"expr": "longhorn_volume_robustness", "legendFormat": "{{pvc_namespace}}/{{pvc}}"} + ] + }, + { + "type": "timeseries", + "title": "Volume Actual Size", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "longhorn_volume_actual_size_bytes", "legendFormat": "{{pvc_namespace}}/{{pvc}} actual"}, + {"expr": "longhorn_volume_capacity_bytes", "legendFormat": "{{pvc_namespace}}/{{pvc}} capacity"} + ] + }, + { + "type": "timeseries", + "title": "Node Storage Usage", + "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "(longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1)", "legendFormat": "{{node}}"} + ] + }, + { + "type": "timeseries", + "title": "Disk Usage", + "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "(longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1)", "legendFormat": "{{node}} {{disk}}"} + ] + }, + { + "type": "timeseries", + "title": "Longhorn Manager CPU", + "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "m"}}, + "targets": [ + {"expr": "longhorn_manager_cpu_usage_millicpu", "legendFormat": "{{node}}"} + ] + }, + { + "type": "timeseries", + "title": "Longhorn Manager Memory", + "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "longhorn_manager_memory_usage_bytes", "legendFormat": "{{node}}"} + ] + } + ] + } diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index fa7734b0..183850b7 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -8,7 +8,9 @@ resources: - ./datastore-pod-monitors.yaml - ./grafana-dashboard-codeplace.yaml - ./grafana-dashboard-logs.yaml + - ./grafana-dashboard-storage.yaml - ./grafana-dashboard-traces.yaml + - ./longhorn-service-monitor.yaml - ./otel-collector.yaml - ./prometheus-rules.yaml - ./tempo.yaml diff --git a/kubernetes/monitoring/longhorn-service-monitor.yaml b/kubernetes/monitoring/longhorn-service-monitor.yaml new file mode 100644 index 00000000..c667b68e --- /dev/null +++ b/kubernetes/monitoring/longhorn-service-monitor.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: longhorn + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - longhorn-system + selector: + matchLabels: + app: longhorn-manager + endpoints: + - port: manager + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index d27bd890..c9d1e2b0 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -132,6 +132,36 @@ spec: annotations: summary: "Loki is not ready" description: "The Loki single-binary pod has not been ready for 1 minute. Log ingestion and log queries may be unavailable." + - alert: LonghornManagerDown + expr: (sum(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) == 0) or absent(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) + for: 2m + labels: + severity: critical + priority: P0 + namespace: longhorn-system + annotations: + summary: "Longhorn managers are not ready" + description: "No Longhorn manager pod has been ready for 2 minutes. PVC health and Longhorn metrics may be unavailable." + - alert: LonghornVolumeFaulted + expr: longhorn_volume_robustness == 3 + for: 1m + labels: + severity: critical + priority: P0 + namespace: longhorn-system + annotations: + summary: "Longhorn volume is faulted" + description: "A Longhorn volume is faulted for 1 minute. A PVC-backed workload may lose storage availability." + - alert: LonghornVolumeReadOnly + expr: longhorn_volume_file_system_read_only == 1 + for: 1m + labels: + severity: critical + priority: P0 + namespace: longhorn-system + annotations: + summary: "Longhorn volume is read-only" + description: "A Longhorn volume filesystem is read-only for 1 minute. Writes from the attached workload may fail." - name: codeplace.p1 interval: 30s rules: @@ -329,6 +359,56 @@ spec: annotations: summary: "Tempo PVC usage is above 85%" description: "The Tempo Longhorn PVC has been above 85% usage for 10 minutes. Trace retention or PVC size must be adjusted." + - alert: LonghornVolumeDegraded + expr: longhorn_volume_robustness == 2 + for: 5m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn volume is degraded" + description: "A Longhorn volume has been degraded for 5 minutes. Replica rebuild, node health, and disk pressure should be checked." + - alert: LonghornNodeNotReady + expr: longhorn_node_status{condition="ready"} == 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn node is not ready" + description: "A Longhorn node has not been ready for 5 minutes." + - alert: LonghornDiskNotReady + expr: longhorn_disk_status{condition="ready"} == 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn disk is not ready" + description: "A Longhorn disk has not been ready for 5 minutes." + - alert: LonghornNodeStorageHigh + expr: (longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn node storage usage is high" + description: "A Longhorn node has used or reserved more than 85% of available storage for 10 minutes." + - alert: LonghornDiskUsageHigh + expr: (longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn disk usage is high" + description: "A Longhorn disk has used or reserved more than 85% of capacity for 10 minutes." - alert: CodePlaceNodePressure expr: kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure", status="true"} == 1 for: 5m diff --git a/kubernetes/setup/longhorn/README.md b/kubernetes/setup/longhorn/README.md index d6dd5287..c652952a 100644 --- a/kubernetes/setup/longhorn/README.md +++ b/kubernetes/setup/longhorn/README.md @@ -88,3 +88,22 @@ kubectl get pods -n longhorn-system -w 물리 노드가 3대 미만인 경우(예: 2대), 복제본을 배치할 공간이 부족하여 볼륨 상태가 Degraded로 표시될 수 있습니다. 이 경우 Longhorn UI(Settings > General > Default Replica Count)에서 기본 복제본 수를 2로 변경해야 합니다. + +### 5. Monitoring + +kube-prometheus-stack이 설치된 클러스터에서는 CodePlace monitoring kustomization이 Longhorn manager metric을 scrape합니다. + +```sh +kubectl apply -k kubernetes/monitoring +kubectl -n monitoring get servicemonitor longhorn +``` + +Prometheus target에서 `longhorn` scrape가 healthy인지 확인하고, Grafana의 `CodePlace Storage` dashboard에서 다음 상태를 확인합니다. + +- Longhorn manager ready count. +- faulted/degraded/read-only volume count. +- volume robustness와 actual/capacity size. +- Longhorn node/disk 사용률. +- Longhorn manager CPU/memory. + +Longhorn alert는 CodePlace alert rule에 포함됩니다. volume `faulted` 또는 `read-only`는 P0, volume `degraded`와 node/disk readiness 또는 usage 문제는 P1로 처리합니다. From a350d3e4877dfb21d322efd8173fcd36f2481810 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 14:59:27 +0900 Subject: [PATCH 032/147] =?UTF-8?q?=EA=B3=B5=EA=B0=9C=20=EC=97=94=EB=93=9C?= =?UTF-8?q?=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=EA=B4=80=EC=B8=A1=EC=84=B1=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 33 ++- .../infra/observability-runbook/_index.md | 64 +++++- kubernetes/monitoring/blackbox-exporter.yaml | 103 +++++++++ .../grafana-dashboard-public-endpoints.yaml | 209 ++++++++++++++++++ .../kube-prometheus-stack-values.yaml | 1 + kubernetes/monitoring/kustomization.yaml | 3 + kubernetes/monitoring/prometheus-rules.yaml | 58 +++++ .../monitoring/public-endpoint-probes.yaml | 45 ++++ 8 files changed, 505 insertions(+), 11 deletions(-) create mode 100644 kubernetes/monitoring/blackbox-exporter.yaml create mode 100644 kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml create mode 100644 kubernetes/monitoring/public-endpoint-probes.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index bf4ab49d..6de7a7b1 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -13,7 +13,7 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 관측 대상은 운영자가 장애 발생 후 1분 안에 알림을 받고, 5분 안에 원인 후보를 좁힐 수 있도록 다음 경로로 나눕니다. -- **User Path:** Traefik/Ingress, frontend nginx, backend Django API. +- **User Path:** synthetic public HTTPS probe, Traefik/Ingress, frontend nginx, backend Django API. - **Submission/Judge Path:** submission 생성, Celery `judge_task`, Redis `waiting_queue`, judge-server heartbeat, judge-server `/judge` 호출, 결과 저장. - **Async Jobs:** celery-worker, celery-beat, scheduled tasks. - **Data Stores:** PostgreSQL/CNPG, Redis Sentinel. @@ -26,6 +26,8 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니다. 이 엔드포인트는 ServiceMonitor가 cluster 내부에서 scrape하며 외부 Ingress에는 연결하지 않습니다. +공개 사용자 경로는 Blackbox exporter 기반 synthetic probe로 별도 확인합니다. 이 probe는 `https://k3s.code-place-dev.site/`, `https://code.pusan.ac.kr/`를 30초 간격으로 호출해 DNS, Traefik TLS termination, frontend nginx, 기본 라우팅을 함께 검증합니다. + 추가된 CodePlace custom metrics는 다음과 같습니다. - `codeplace_http_requests_total{method,endpoint,status_code}` @@ -129,9 +131,12 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto 신규 monitoring 리소스는 `kubernetes/monitoring` 아래에 두고, 애플리케이션 `dev/prod` overlay에 포함하지 않습니다. - `backend-service-monitor.yaml`: backend `/metrics` scrape, interval 15s. +- `blackbox-exporter.yaml`: public endpoint synthetic probe용 Blackbox exporter. +- `public-endpoint-probes.yaml`: dev/prod 공개 HTTPS URL Probe, interval 30s. - `prometheus-rules.yaml`: P0/P1 fast alert rules. - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. +- `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. @@ -157,6 +162,8 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `BackendTargetDown`: backend scrape 실패 1분 지속. - `Api5xxSpike`: backend 5xx 비율 5% 초과 2분 지속. - `Ingress5xxSpike`: Traefik 5xx 비율 5% 초과 2분 지속. +- `PublicEndpointDown`: prod 공개 HTTPS endpoint probe 실패 1분 지속. +- `PublicEndpointTLSCritical`: prod 공개 HTTPS 인증서 만료 3일 이내. - `JudgeAllUnavailable`: 사용 가능한 judge-server 0개 1분 지속. - `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. - `PostgresUnavailable`: PostgreSQL Pod not ready 1분 지속. @@ -173,6 +180,10 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `ApiLatencyHigh`: p95 latency 2초 초과 5분 지속. +- `PublicEndpointDown`: dev 공개 HTTPS endpoint probe 실패 2분 지속. +- `PublicEndpointLatencyHigh`: 공개 HTTPS probe duration 2초 초과 5분 지속. +- `PublicEndpointTLSExpiringSoon`: 공개 HTTPS 인증서 만료 14일 이내. +- `BlackboxExporterUnavailable`: synthetic probe exporter Pod not ready 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. - `CeleryBeatDown`: beat Pod not ready 2분 지속. @@ -214,19 +225,21 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 5. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. -7. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. -8. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, judge/celery log panel을 확인합니다. -9. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. -10. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. -11. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. -12. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -13. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. -14. Prometheus target에서 `longhorn` ServiceMonitor가 healthy인지 확인합니다. -15. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +7. Prometheus target에서 `backend`, `longhorn`, `blackbox-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. +8. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. +9. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. +10. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, judge/celery log panel을 확인합니다. +11. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. +12. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. +13. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. +14. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +15. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. +16. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. - `/metrics`는 cluster 내부 scrape 전용이며 외부 Ingress에 노출하지 않습니다. +- 공개 URL synthetic probe는 monitoring namespace 내부에서 외부 도메인으로 나가는 HTTPS 요청입니다. DNS, outbound network, TLS, Traefik/frontend 경로를 함께 검증하지만 브라우저 JS runtime error를 대체하지는 않습니다. - Discord webhook URL은 Git에 저장하지 않습니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. - OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지하고 dev overlay에서 먼저 활성화합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 1a5e5236..21ba5834 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -17,11 +17,12 @@ weight: 6 bash kubernetes/monitoring/validate.sh kubectl get ns kubectl -n monitoring get prometheusrule,alertmanagerconfig,servicemonitor -kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy|otel|tempo' +kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy|otel|tempo|blackbox' kubectl -n longhorn-system get pod,svc ``` Grafana에서는 `CodePlace Overview` dashboard에서 `namespace`를 알림의 namespace로 맞춥니다. +공개 URL 장애는 `CodePlace Public Endpoints` dashboard에서 먼저 봅니다. 로그 수집이나 로그 기반 확인이 필요하면 `CodePlace Logs` dashboard를 함께 봅니다. PVC/volume 문제가 의심되면 `CodePlace Storage` dashboard에서 Longhorn 상태를 먼저 확인합니다. @@ -32,6 +33,8 @@ Status -> Targets -> backend Status -> Targets -> postgres Status -> Targets -> redis Status -> Targets -> longhorn +Status -> Targets -> codeplace-public-http-dev +Status -> Targets -> codeplace-public-http-prod ``` Alertmanager 확인: @@ -78,6 +81,20 @@ Tempo에서 trace를 선택한 뒤 같은 `trace_id`를 Loki에서 조회합니 {namespace="", container=~"celery-worker|celery-beat"} | json | trace_id="" ``` +공개 URL synthetic probe 확인: + +```sh +kubectl -n monitoring get pod,svc,probe | grep -E 'blackbox|codeplace-public' +kubectl -n monitoring logs deploy/blackbox-exporter --tail=100 +``` + +```promql +probe_success{probe_type="public-http"} +probe_http_status_code{probe_type="public-http"} +probe_duration_seconds{probe_type="public-http"} +(probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) / 86400 +``` + Longhorn 상태 확인: ```sh @@ -174,6 +191,33 @@ Grafana Explore: - backend 5xx도 같이 증가하면 app/backend 문제입니다. - ingress 5xx만 증가하면 Traefik route, Service endpoint, upstream connection 문제를 의심합니다. +### PublicEndpointDown / PublicEndpointTLSCritical + +증상: 공개 HTTPS endpoint synthetic probe가 실패하거나 인증서 만료가 3일 이내입니다. + +확인: + +```promql +probe_success{probe_type="public-http"} +probe_http_status_code{probe_type="public-http"} +probe_http_duration_seconds{probe_type="public-http"} +(probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) / 86400 +``` + +```sh +kubectl -n monitoring get pod,svc,probe | grep -E 'blackbox|codeplace-public' +kubectl -n monitoring logs deploy/blackbox-exporter --tail=200 +kubectl -n get ingress,svc,endpoints frontend +kubectl -n kube-system logs deploy/traefik --tail=200 +``` + +판단: + +- `probe_success=0`이고 ingress/backend 지표는 정상이면 DNS, TLS, Traefik public route, frontend nginx 기본 경로 문제를 먼저 봅니다. +- `probe_http_status_code`가 0이면 DNS/TCP/TLS 연결 실패 가능성이 큽니다. +- 5xx이면 Traefik/frontend/backend 로그와 같은 시간대를 대조합니다. +- TLS 만료 알림은 Traefik ACME resolver와 인증서 저장소를 확인합니다. + ### JudgeAllUnavailable 증상: 사용 가능한 judge-server가 0개입니다. @@ -365,6 +409,24 @@ sum by (namespace) (codeplace_waiting_queue_length{namespace=""}) backend latency와 DB/Redis readiness, CPU/memory를 같이 봅니다. +### PublicEndpointDown / PublicEndpointLatencyHigh / PublicEndpointTLSExpiringSoon / BlackboxExporterUnavailable + +확인: + +```promql +probe_success{probe_type="public-http"} +probe_duration_seconds{probe_type="public-http"} +probe_http_duration_seconds{probe_type="public-http"} +(probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) / 86400 +kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condition="true"} +``` + +판단: + +- dev endpoint down은 배포 테스트 영향으로 P1입니다. prod endpoint down은 P0입니다. +- latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. +- `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. + ### JudgeWaitingQueueBacklog 확인: diff --git a/kubernetes/monitoring/blackbox-exporter.yaml b/kubernetes/monitoring/blackbox-exporter.yaml new file mode 100644 index 00000000..c013525c --- /dev/null +++ b/kubernetes/monitoring/blackbox-exporter.yaml @@ -0,0 +1,103 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: blackbox-exporter-config +data: + blackbox.yml: | + modules: + http_2xx: + prober: http + timeout: 10s + http: + method: GET + preferred_ip_protocol: ip4 + valid_http_versions: + - HTTP/1.1 + - HTTP/2.0 + valid_status_codes: + - 200 + fail_if_not_ssl: true + tls_config: + insecure_skip_verify: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: blackbox-exporter + labels: + app: blackbox-exporter +spec: + replicas: 1 + selector: + matchLabels: + app: blackbox-exporter + template: + metadata: + labels: + app: blackbox-exporter + spec: + containers: + - name: blackbox-exporter + image: quay.io/prometheus/blackbox-exporter:v0.27.0 + args: + - --config.file=/etc/blackbox-exporter/blackbox.yml + ports: + - name: http + containerPort: 9115 + readinessProbe: + httpGet: + path: /-/ready + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /-/healthy + port: http + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: config + mountPath: /etc/blackbox-exporter + readOnly: true + volumes: + - name: config + configMap: + name: blackbox-exporter-config +--- +apiVersion: v1 +kind: Service +metadata: + name: blackbox-exporter + labels: + app: blackbox-exporter +spec: + selector: + app: blackbox-exporter + ports: + - name: http + port: 9115 + targetPort: http +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: blackbox-exporter + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: blackbox-exporter + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml new file mode 100644 index 00000000..c26d3aae --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml @@ -0,0 +1,209 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-public-endpoints + labels: + grafana_dashboard: "1" +data: + codeplace-public-endpoints.json: | + { + "uid": "codeplace-public-endpoints", + "title": "CodePlace Public Endpoints", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "synthetic", "blackbox"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "Prod Endpoint Up", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "probe_success{job=\"codeplace-public-http-prod\", environment=\"prod\"}", "legendFormat": "{{instance}}"} + ] + }, + { + "type": "stat", + "title": "Dev Endpoint Up", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "probe_success{job=\"codeplace-public-http-dev\", environment=\"dev\"}", "legendFormat": "{{instance}}"} + ] + }, + { + "type": "stat", + "title": "HTTP Status", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 200} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "probe_http_status_code{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "stat", + "title": "TLS Days Left", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "d", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 3}, + {"color": "green", "value": 14} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "(probe_ssl_earliest_cert_expiry{probe_type=\"public-http\"} - time()) / 86400", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "stat", + "title": "Probe Duration", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 2} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "probe_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "stat", + "title": "Blackbox Exporter Ready", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"blackbox-exporter-.*\"})", "legendFormat": "ready"} + ] + }, + { + "type": "timeseries", + "title": "Endpoint Availability", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}}, + "targets": [ + {"expr": "probe_success{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "timeseries", + "title": "Endpoint Duration", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "probe_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "timeseries", + "title": "TLS Days Left", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "d", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 14}, + {"color": "green", "value": 30} + ] + } + } + }, + "targets": [ + {"expr": "(probe_ssl_earliest_cert_expiry{probe_type=\"public-http\"} - time()) / 86400", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "timeseries", + "title": "HTTP Phase Duration", + "gridPos": {"x": 0, "y": 12, "w": 24, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "probe_http_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{phase}}"} + ] + } + ] + } diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index 1bfe4d4c..1503d1f1 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -4,6 +4,7 @@ prometheus: scrapeInterval: 15s serviceMonitorSelectorNilUsesHelmValues: false podMonitorSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false ruleSelectorNilUsesHelmValues: false alertmanager: diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index 183850b7..c91440ff 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -5,12 +5,15 @@ namespace: monitoring resources: - ./alertmanager-config.yaml - ./backend-service-monitor.yaml + - ./blackbox-exporter.yaml - ./datastore-pod-monitors.yaml - ./grafana-dashboard-codeplace.yaml - ./grafana-dashboard-logs.yaml + - ./grafana-dashboard-public-endpoints.yaml - ./grafana-dashboard-storage.yaml - ./grafana-dashboard-traces.yaml - ./longhorn-service-monitor.yaml - ./otel-collector.yaml - ./prometheus-rules.yaml + - ./public-endpoint-probes.yaml - ./tempo.yaml diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index c9d1e2b0..5470c901 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -47,6 +47,26 @@ spec: annotations: summary: "Ingress 5xx rate is high in prod" description: "Traefik 5xx responses for code-place-prod are above 5% for 2 minutes." + - alert: PublicEndpointDown + expr: probe_success{job="codeplace-public-http-prod", environment="prod"} == 0 + for: 1m + labels: + severity: critical + priority: P0 + namespace: code-place-prod + annotations: + summary: "Public frontend endpoint is down in prod" + description: "The synthetic HTTPS probe for code.pusan.ac.kr has failed for 1 minute." + - alert: PublicEndpointTLSCritical + expr: (probe_ssl_earliest_cert_expiry{probe_type="public-http", environment="prod"} - time()) < 259200 + for: 15m + labels: + severity: critical + priority: P0 + namespace: code-place-prod + annotations: + summary: "Public endpoint TLS certificate expires within 3 days" + description: "The production CodePlace public HTTPS certificate expires within 3 days." - alert: JudgeAllUnavailable expr: (sum by (namespace) (codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) == 0) or (max by (namespace) (up{job="backend", namespace=~"code-place-(dev|prod)"}) unless on (namespace) codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) for: 1m @@ -174,6 +194,44 @@ spec: annotations: summary: "Backend API p95 latency is high" description: "Backend API p95 latency is above 2 seconds for 5 minutes." + - alert: PublicEndpointDown + expr: probe_success{job="codeplace-public-http-dev", environment="dev"} == 0 + for: 2m + labels: + severity: warning + priority: P1 + namespace: code-place-dev + annotations: + summary: "Public frontend endpoint is down in dev" + description: "The synthetic HTTPS probe for k3s.code-place-dev.site has failed for 2 minutes." + - alert: PublicEndpointLatencyHigh + expr: probe_duration_seconds{probe_type="public-http"} > 2 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Public frontend endpoint latency is high" + description: "A synthetic HTTPS probe has taken more than 2 seconds for 5 minutes." + - alert: PublicEndpointTLSExpiringSoon + expr: (probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) < 1209600 + for: 1h + labels: + severity: warning + priority: P1 + annotations: + summary: "Public endpoint TLS certificate expires within 14 days" + description: "A CodePlace public HTTPS certificate expires within 14 days." + - alert: BlackboxExporterUnavailable + expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"blackbox-exporter-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"blackbox-exporter-.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Blackbox exporter is not ready" + description: "The synthetic public endpoint probe exporter has not been ready for 2 minutes." - alert: JudgeWaitingQueueBacklog expr: codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"} > 5 for: 3m diff --git a/kubernetes/monitoring/public-endpoint-probes.yaml b/kubernetes/monitoring/public-endpoint-probes.yaml new file mode 100644 index 00000000..f94b45e9 --- /dev/null +++ b/kubernetes/monitoring/public-endpoint-probes.yaml @@ -0,0 +1,45 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-public-dev-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-public-http-dev + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://k3s.code-place-dev.site/ + labels: + environment: dev + namespace: code-place-dev + service: frontend + probe_type: public-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-public-prod-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-public-http-prod + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://code.pusan.ac.kr/ + labels: + environment: prod + namespace: code-place-prod + service: frontend + probe_type: public-http From 07b3887d9f0819886ea988313b5bfd8a15083368 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 15:08:40 +0900 Subject: [PATCH 033/147] =?UTF-8?q?vLLM=20AI=20=EC=B6=94=EB=A1=A0=20?= =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 29 ++- .../infra/observability-runbook/_index.md | 69 +++++- kubernetes/base/vllm/README.md | 21 ++ .../grafana-dashboard-ai-inference.yaml | 212 ++++++++++++++++++ .../grafana-dashboard-codeplace.yaml | 8 +- .../monitoring/grafana-dashboard-logs.yaml | 4 +- kubernetes/monitoring/kustomization.yaml | 2 + kubernetes/monitoring/prometheus-rules.yaml | 48 +++- .../monitoring/vllm-service-monitor.yaml | 19 ++ 9 files changed, 396 insertions(+), 16 deletions(-) create mode 100644 kubernetes/monitoring/grafana-dashboard-ai-inference.yaml create mode 100644 kubernetes/monitoring/vllm-service-monitor.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 6de7a7b1..dcbe49ab 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -15,6 +15,7 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 - **User Path:** synthetic public HTTPS probe, Traefik/Ingress, frontend nginx, backend Django API. - **Submission/Judge Path:** submission 생성, Celery `judge_task`, Redis `waiting_queue`, judge-server heartbeat, judge-server `/judge` 호출, 결과 저장. +- **AI Hint Path:** backend AI hint API, vLLM OpenAI-compatible server, vLLM HF cache PVC. - **Async Jobs:** celery-worker, celery-beat, scheduled tasks. - **Data Stores:** PostgreSQL/CNPG, Redis Sentinel. - **Platform/Storage:** backend/frontend/celery/judge Pod, node, PVC, Longhorn volume/node/disk. @@ -42,6 +43,17 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_celery_task_last_runtime_seconds{task_name,status}` +prod AI 힌트용 vLLM은 OpenAI-compatible server의 `/metrics`를 ServiceMonitor로 scrape합니다. vLLM production metrics 기준으로 다음 항목을 봅니다. + +- `vllm:num_requests_running` +- `vllm:num_requests_waiting` +- `vllm:kv_cache_usage_perc` +- `vllm:e2e_request_latency_seconds` +- `vllm:request_queue_time_seconds` +- `vllm:time_to_first_token_seconds` +- `vllm:prompt_tokens` +- `vllm:generation_tokens` + `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. ### Logs @@ -136,6 +148,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `prometheus-rules.yaml`: P0/P1 fast alert rules. - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. +- `grafana-dashboard-ai-inference.yaml`: prod vLLM readiness, queue/cache/latency/token throughput, HF cache PVC dashboard. - `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. @@ -146,6 +159,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `grafana-dashboard-traces.yaml`: Collector/Tempo readiness, span ingest/export, Tempo PVC usage dashboard. - `longhorn-service-monitor.yaml`: Longhorn manager `/metrics` scrape, interval 30s. - `grafana-dashboard-storage.yaml`: Longhorn volume/node/disk 상태와 storage capacity dashboard. +- `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. - `validate.sh`: monitoring YAML, Grafana dashboard JSON, kustomize render, 선택적 promtool/Helm render 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. @@ -174,6 +188,7 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `LonghornManagerDown`: Longhorn manager 전부 not ready 2분 지속. - `LonghornVolumeFaulted`: Longhorn volume faulted 1분 지속. - `LonghornVolumeReadOnly`: Longhorn volume filesystem read-only 1분 지속. +- `VLLMUnavailable`: prod vLLM `/metrics` scrape 실패 2분 지속. ### P1 @@ -209,6 +224,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `LonghornDiskNotReady`: Longhorn disk not ready 5분 지속. - `LonghornNodeStorageHigh`: Longhorn node storage 사용/예약률 85% 초과 10분. - `LonghornDiskUsageHigh`: Longhorn disk 사용/예약률 85% 초과 10분. +- `VLLMWaitingQueueHigh`: prod vLLM waiting request 5 초과 5분 지속. +- `VLLMKVCacheHigh`: prod vLLM KV cache 사용률 90% 초과 10분 지속. +- `VLLMRequestLatencyHigh`: prod vLLM p95 e2e latency 60초 초과 10분 지속. - `CodePlaceNodePressure`: node memory/disk/PID pressure 5분 지속. ## 5. 운영 확인 절차 @@ -224,17 +242,17 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - prod: `kubectl apply -k kubernetes/overlays/prod` 5. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` -6. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. -7. Prometheus target에서 `backend`, `longhorn`, `blackbox-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. +6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. 8. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. 9. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, judge/celery log panel을 확인합니다. 11. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 12. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. 13. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. -14. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -15. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. -16. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +14. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC panel이 비어 있지 않은지 확인합니다. +15. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +16. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. +17. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. @@ -251,6 +269,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. - Backend/Celery Sentry release는 `SENTRY_RELEASE` env가 있으면 사용합니다. 이미지 태그와 동일한 commit SHA를 넣어 Grafana/Loki/Sentry 간 배포 기준을 맞추는 것을 권장합니다. - Tempo tracing은 monitoring namespace에 배포하지만, prod app trace export는 dev 확인 후 별도 승격합니다. +- vLLM은 prod overlay에만 포함되어 있으므로 vLLM ServiceMonitor와 AI Inference dashboard도 `code-place-prod`를 기준으로 조회합니다. dev에 vLLM을 올리면 별도 dev ServiceMonitor/Probe 또는 namespace selector 확장이 필요합니다. ## 6. 검증 diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 21ba5834..f7ec9cf1 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -33,6 +33,7 @@ Status -> Targets -> backend Status -> Targets -> postgres Status -> Targets -> redis Status -> Targets -> longhorn +Status -> Targets -> vllm Status -> Targets -> codeplace-public-http-dev Status -> Targets -> codeplace-public-http-prod ``` @@ -53,7 +54,7 @@ Loki 로그 확인: {namespace="kube-system", app_kubernetes_io_name="traefik"} | json {namespace="", app="frontend"} | json {namespace="", app="backend"} | json -{namespace="", container=~"backend|celery-worker|judge-server"} +{namespace="", container=~"backend|celery-worker|judge-server|vllm"} ``` Loki/Alloy 상태 확인: @@ -109,6 +110,21 @@ longhorn_volume_file_system_read_only (longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1) ``` +vLLM 상태 확인: + +```sh +kubectl -n code-place-prod get pod,svc,pvc | grep vllm +kubectl -n monitoring get servicemonitor vllm -o yaml +``` + +```promql +up{job="vllm", namespace="code-place-prod"} +{__name__="vllm:num_requests_running", namespace="code-place-prod"} +{__name__="vllm:num_requests_waiting", namespace="code-place-prod"} +{__name__="vllm:kv_cache_usage_perc", namespace="code-place-prod"} +histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) +``` + ## P0 ### BackendTargetDown @@ -366,6 +382,30 @@ longhorn_volume_capacity_bytes - volume faulted는 PVC-backed workload의 storage availability 문제입니다. affected `pvc_namespace`/`pvc`를 찾아 해당 workload 상태와 Longhorn replica 상태를 확인합니다. - read-only volume은 쓰기 실패로 바로 이어질 수 있습니다. Loki/Tempo/PostgreSQL/Redis PVC가 대상이면 해당 관측/데이터 경로 장애로 취급합니다. +### VLLMUnavailable + +증상: prod vLLM `/metrics` scrape가 2분 이상 실패합니다. AI 힌트 기능이 실패하거나 장시간 지연될 수 있습니다. + +확인: + +```sh +kubectl -n code-place-prod get pod,svc,endpoints,pvc | grep vllm +kubectl -n code-place-prod describe pod -l app=vllm +kubectl -n code-place-prod logs deploy/vllm --tail=200 +kubectl -n monitoring get servicemonitor vllm -o yaml +``` + +```promql +up{job="vllm", namespace="code-place-prod"} +kube_pod_status_ready{namespace="code-place-prod", pod=~"vllm-.*", condition="true"} +``` + +판단: + +- Pod가 pending이면 GPU node label, `runtimeClassName: nvidia`, GPU resource, Longhorn HF cache PVC attach 상태를 봅니다. +- Pod는 ready인데 scrape만 실패하면 Service port/path, vLLM `/metrics`, NetworkPolicy 여부를 확인합니다. +- backend AI hint API error가 같이 증가하면 사용자 영향이 있습니다. + ## P1 ### OTelCollectorUnavailable / TempoUnavailable / TempoPVCAlmostFull @@ -566,6 +606,33 @@ longhorn_disk_status{condition="ready"} - storage usage가 85%를 넘으면 Loki/Tempo retention 축소, noisy 로그/trace 감소, Longhorn disk/PVC 증설 중 하나를 결정합니다. 클라우드 object storage가 없는 현재 조건에서는 Longhorn 여유 용량이 로그/트레이스 retention의 상한입니다. +### VLLMWaitingQueueHigh / VLLMKVCacheHigh / VLLMRequestLatencyHigh + +확인: + +```promql +{__name__="vllm:num_requests_running", namespace="code-place-prod"} +{__name__="vllm:num_requests_waiting", namespace="code-place-prod"} +{__name__="vllm:kv_cache_usage_perc", namespace="code-place-prod"} +histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) +histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:request_queue_time_seconds_bucket", namespace="code-place-prod"}[10m]))) +histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:time_to_first_token_seconds_bucket", namespace="code-place-prod"}[10m]))) +``` + +```sh +kubectl -n code-place-prod top pod -l app=vllm +kubectl -n code-place-prod describe pod -l app=vllm +kubectl -n code-place-prod logs deploy/vllm --tail=200 +kubectl -n code-place-prod describe pvc vllm-hf-cache +``` + +판단: + +- waiting queue와 queue p95가 같이 오르면 vLLM scheduling capacity 부족입니다. GPU saturation, `max-num-seqs`, KV cache pressure를 봅니다. +- KV cache가 90%를 넘으면 max model length, 동시 요청 수, prompt 길이, GPU memory utilization을 함께 봅니다. +- e2e latency만 증가하면 prompt/generation token 증가, backend streaming read timeout, vLLM 로그의 model/engine warning을 확인합니다. + 현재 vLLM은 prod 전용입니다. dev에서 AI hint를 별도 검증하려면 dev vLLM 배포와 ServiceMonitor 확장이 먼저 필요합니다. + ### LokiPVCAlmostFull 확인: diff --git a/kubernetes/base/vllm/README.md b/kubernetes/base/vllm/README.md index 6b066ed0..c323b551 100644 --- a/kubernetes/base/vllm/README.md +++ b/kubernetes/base/vllm/README.md @@ -52,3 +52,24 @@ kubectl describe pod -n code-place-prod -l app=vllm - Longhorn replica 수는 YAML 에서 고정하지 않고, 필요하면 Longhorn UI 에서 직접 조정하는 전제를 둡니다. - `max-num-seqs=60` 은 처리량 위주 값이라, 메모리 압박이나 OOM 이 보이면 가장 먼저 낮춰야 합니다. - 더 보수적으로 운영하려면 이미지 tag 대신 digest 로 pin 하는 것이 가장 안전합니다. + +## Monitoring + +vLLM OpenAI-compatible server는 `/metrics`에서 Prometheus metrics를 제공합니다. +CodePlace monitoring kustomization은 prod vLLM Service를 `vllm-service-monitor.yaml`로 scrape합니다. + +```bash +kubectl apply -k kubernetes/monitoring +kubectl -n monitoring get servicemonitor vllm +``` + +Grafana에서는 `CodePlace AI Inference` dashboard에서 다음 상태를 확인합니다. + +- vLLM scrape up / Pod ready. +- running/waiting request count. +- KV cache usage. +- p95 e2e latency, queue latency, time-to-first-token. +- prompt/generation token throughput. +- `vllm-hf-cache` PVC usage. + +알림은 vLLM scrape 실패를 P0로 처리하고, waiting queue 증가, KV cache 90% 초과, p95 latency 60초 초과는 P1로 처리합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml new file mode 100644 index 00000000..263a24d9 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml @@ -0,0 +1,212 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-ai-inference + labels: + grafana_dashboard: "1" +data: + codeplace-ai-inference.json: | + { + "uid": "codeplace-ai-inference", + "title": "CodePlace AI Inference", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "ai", "vllm"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "vLLM Scrape Up", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "up{job=\"vllm\", namespace=\"code-place-prod\"}", "legendFormat": "vllm"} + ] + }, + { + "type": "stat", + "title": "vLLM Pod Ready", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"code-place-prod\", condition=\"true\", pod=~\"vllm-.*\"})", "legendFormat": "ready"} + ] + }, + { + "type": "stat", + "title": "Running Requests", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max({__name__=\"vllm:num_requests_running\", namespace=\"code-place-prod\"})", "legendFormat": "running"} + ] + }, + { + "type": "stat", + "title": "Waiting Requests", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max({__name__=\"vllm:num_requests_waiting\", namespace=\"code-place-prod\"})", "legendFormat": "waiting"} + ] + }, + { + "type": "stat", + "title": "KV Cache Usage", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.9} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max({__name__=\"vllm:kv_cache_usage_perc\", namespace=\"code-place-prod\"})", "legendFormat": "kv-cache"} + ] + }, + { + "type": "stat", + "title": "HF Cache PVC", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} + ] + }, + { + "type": "timeseries", + "title": "vLLM Requests", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "{__name__=\"vllm:num_requests_running\", namespace=\"code-place-prod\"}", "legendFormat": "running"}, + {"expr": "{__name__=\"vllm:num_requests_waiting\", namespace=\"code-place-prod\"}", "legendFormat": "waiting"} + ] + }, + { + "type": "timeseries", + "title": "vLLM Request Success Rate", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (finished_reason) (rate({__name__=\"vllm:request_success_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "{{finished_reason}}"} + ] + }, + { + "type": "timeseries", + "title": "vLLM p95 Latency", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"vllm:e2e_request_latency_seconds_bucket\", namespace=\"code-place-prod\"}[10m])))", "legendFormat": "e2e p95"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"vllm:request_queue_time_seconds_bucket\", namespace=\"code-place-prod\"}[10m])))", "legendFormat": "queue p95"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"vllm:time_to_first_token_seconds_bucket\", namespace=\"code-place-prod\"}[10m])))", "legendFormat": "ttft p95"} + ] + }, + { + "type": "timeseries", + "title": "vLLM Token Throughput", + "gridPos": {"x": 0, "y": 12, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum(rate({__name__=\"vllm:prompt_tokens_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "prompt tokens/s"}, + {"expr": "sum(rate({__name__=\"vllm:generation_tokens_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "generation tokens/s"} + ] + }, + { + "type": "timeseries", + "title": "vLLM Pod Resources", + "gridPos": {"x": 8, "y": 12, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"code-place-prod\", pod=~\"vllm-.*\", container=\"vllm\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, + {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"code-place-prod\", pod=~\"vllm-.*\", container=\"vllm\"})", "legendFormat": "{{pod}} memory bytes"} + ] + }, + { + "type": "timeseries", + "title": "vLLM HF Cache PVC", + "gridPos": {"x": 16, "y": 12, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index a6dc577f..652a1ede 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -283,7 +283,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min by (namespace, pod) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"})", "legendFormat": "{{namespace}} {{pod}}"} + {"expr": "min by (namespace, pod) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}}"} ] }, { @@ -292,7 +292,7 @@ data: "gridPos": {"x": 6, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} ] }, { @@ -301,7 +301,7 @@ data: "gridPos": {"x": 12, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "cores"}}, "targets": [ - {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} ] }, { @@ -310,7 +310,7 @@ data: "gridPos": {"x": 18, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ - {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} ] }, { diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index 0d86af4b..ddd96498 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -198,7 +198,7 @@ data: }, { "type": "logs", - "title": "Judge and Celery Logs", + "title": "Judge, Celery, and vLLM Logs", "gridPos": {"x": 14, "y": 4, "w": 10, "h": 10}, "datasource": "Loki", "options": { @@ -208,7 +208,7 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=~\"$namespace\", container=~\"celery-worker|celery-beat|judge-server\"}"} + {"expr": "{namespace=~\"$namespace\", container=~\"celery-worker|celery-beat|judge-server|vllm\"}"} ] }, { diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index c91440ff..f2cedb66 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -8,6 +8,7 @@ resources: - ./blackbox-exporter.yaml - ./datastore-pod-monitors.yaml - ./grafana-dashboard-codeplace.yaml + - ./grafana-dashboard-ai-inference.yaml - ./grafana-dashboard-logs.yaml - ./grafana-dashboard-public-endpoints.yaml - ./grafana-dashboard-storage.yaml @@ -17,3 +18,4 @@ resources: - ./prometheus-rules.yaml - ./public-endpoint-probes.yaml - ./tempo.yaml + - ./vllm-service-monitor.yaml diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 5470c901..2966d0c6 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -182,6 +182,16 @@ spec: annotations: summary: "Longhorn volume is read-only" description: "A Longhorn volume filesystem is read-only for 1 minute. Writes from the attached workload may fail." + - alert: VLLMUnavailable + expr: (up{job="vllm", namespace="code-place-prod"} == 0) or absent(up{job="vllm", namespace="code-place-prod"}) + for: 2m + labels: + severity: critical + priority: P0 + namespace: code-place-prod + annotations: + summary: "vLLM metrics target is unavailable" + description: "Prometheus cannot scrape the vLLM /metrics endpoint for 2 minutes. AI hint serving may be unavailable." - name: codeplace.p1 interval: 30s rules: @@ -286,7 +296,7 @@ spec: summary: "Celery task runtime is high" description: "Celery task p95 runtime has been above 120 seconds for 10 minutes." - alert: PodCrashLooping - expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*"}[5m])) > 0 + expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|vllm-.*"}[5m])) > 0 for: 5m labels: severity: warning @@ -295,7 +305,7 @@ spec: summary: "CodePlace pod is restarting" description: "A CodePlace application pod has restarted within the last 5 minutes." - alert: CodePlacePodNotReady - expr: max by (namespace, pod) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}) == 0 + expr: max by (namespace, pod) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}) == 0 for: 5m labels: severity: warning @@ -304,7 +314,7 @@ spec: summary: "CodePlace pod is not ready" description: "A CodePlace application or datastore pod has not been ready for 5 minutes." - alert: CodePlaceContainerCPUHigh - expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}), 0.001) > 0.8 + expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 0.001) > 0.8 for: 10m labels: severity: warning @@ -313,7 +323,7 @@ spec: summary: "CodePlace container CPU usage is high" description: "A CodePlace container has used more than 80% of its CPU limit for 10 minutes." - alert: CodePlaceContainerMemoryHigh - expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="memory", unit="byte", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*"}), 1) > 0.85 + expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="memory", unit="byte", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 1) > 0.85 for: 10m labels: severity: warning @@ -467,6 +477,36 @@ spec: annotations: summary: "Longhorn disk usage is high" description: "A Longhorn disk has used or reserved more than 85% of capacity for 10 minutes." + - alert: VLLMWaitingQueueHigh + expr: max({__name__="vllm:num_requests_waiting", namespace="code-place-prod"}) > 5 + for: 5m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "vLLM request waiting queue is high" + description: "More than 5 vLLM requests have been waiting for 5 minutes. AI hint latency may be elevated." + - alert: VLLMKVCacheHigh + expr: max({__name__="vllm:kv_cache_usage_perc", namespace="code-place-prod"}) > 0.9 + for: 10m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "vLLM KV cache usage is high" + description: "vLLM KV cache usage has been above 90% for 10 minutes." + - alert: VLLMRequestLatencyHigh + expr: histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) > 60 + for: 10m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "vLLM request latency is high" + description: "vLLM p95 end-to-end request latency has been above 60 seconds for 10 minutes." - alert: CodePlaceNodePressure expr: kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure", status="true"} == 1 for: 5m diff --git a/kubernetes/monitoring/vllm-service-monitor.yaml b/kubernetes/monitoring/vllm-service-monitor.yaml new file mode 100644 index 00000000..034c580b --- /dev/null +++ b/kubernetes/monitoring/vllm-service-monitor.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: vllm + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - code-place-prod + selector: + matchLabels: + app: vllm + jobLabel: app + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s From 7548395c637694d63c43aed0ce9c30663530d945 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 15:13:45 +0900 Subject: [PATCH 034/147] =?UTF-8?q?hub-auth=20=EA=B4=80=EC=B8=A1=EC=84=B1?= =?UTF-8?q?=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 15 ++++-- .../infra/observability-runbook/_index.md | 9 ++-- kubernetes/base/hub-auth/deployment.yaml | 17 +++++++ .../grafana-dashboard-codeplace.yaml | 8 ++-- .../monitoring/grafana-dashboard-logs.yaml | 4 +- .../grafana-dashboard-public-endpoints.yaml | 8 ++-- kubernetes/monitoring/prometheus-rules.yaml | 20 ++++---- .../monitoring/public-endpoint-probes.yaml | 46 +++++++++++++++++++ 8 files changed, 99 insertions(+), 28 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index dcbe49ab..c1beed22 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -13,7 +13,7 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 관측 대상은 운영자가 장애 발생 후 1분 안에 알림을 받고, 5분 안에 원인 후보를 좁힐 수 있도록 다음 경로로 나눕니다. -- **User Path:** synthetic public HTTPS probe, Traefik/Ingress, frontend nginx, backend Django API. +- **User Path:** synthetic public HTTPS probe, Traefik/Ingress, frontend nginx, hub-auth OAuth helper, backend Django API. - **Submission/Judge Path:** submission 생성, Celery `judge_task`, Redis `waiting_queue`, judge-server heartbeat, judge-server `/judge` 호출, 결과 저장. - **AI Hint Path:** backend AI hint API, vLLM OpenAI-compatible server, vLLM HF cache PVC. - **Async Jobs:** celery-worker, celery-beat, scheduled tasks. @@ -27,7 +27,12 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니다. 이 엔드포인트는 ServiceMonitor가 cluster 내부에서 scrape하며 외부 Ingress에는 연결하지 않습니다. -공개 사용자 경로는 Blackbox exporter 기반 synthetic probe로 별도 확인합니다. 이 probe는 `https://k3s.code-place-dev.site/`, `https://code.pusan.ac.kr/`를 30초 간격으로 호출해 DNS, Traefik TLS termination, frontend nginx, 기본 라우팅을 함께 검증합니다. +공개 사용자 경로는 Blackbox exporter 기반 synthetic probe로 별도 확인합니다. 이 probe는 frontend와 hub-auth 공개 URL을 30초 간격으로 호출해 DNS, Traefik TLS termination, frontend nginx, OAuth helper 기본 라우팅을 함께 검증합니다. + +- dev frontend: `https://k3s.code-place-dev.site/` +- prod frontend: `https://code.pusan.ac.kr/` +- dev hub-auth: `https://hub-auth-dev.code-place-dev.site/` +- prod hub-auth: `https://hub-auth.code-place-dev.site/` 추가된 CodePlace custom metrics는 다음과 같습니다. @@ -176,7 +181,7 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `BackendTargetDown`: backend scrape 실패 1분 지속. - `Api5xxSpike`: backend 5xx 비율 5% 초과 2분 지속. - `Ingress5xxSpike`: Traefik 5xx 비율 5% 초과 2분 지속. -- `PublicEndpointDown`: prod 공개 HTTPS endpoint probe 실패 1분 지속. +- `PublicEndpointDown`: prod 공개 HTTPS endpoint probe 실패 1분 지속. frontend와 hub-auth를 포함합니다. - `PublicEndpointTLSCritical`: prod 공개 HTTPS 인증서 만료 3일 이내. - `JudgeAllUnavailable`: 사용 가능한 judge-server 0개 1분 지속. - `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. @@ -195,7 +200,7 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `ApiLatencyHigh`: p95 latency 2초 초과 5분 지속. -- `PublicEndpointDown`: dev 공개 HTTPS endpoint probe 실패 2분 지속. +- `PublicEndpointDown`: dev 공개 HTTPS endpoint probe 실패 2분 지속. frontend와 hub-auth를 포함합니다. - `PublicEndpointLatencyHigh`: 공개 HTTPS probe duration 2초 초과 5분 지속. - `PublicEndpointTLSExpiringSoon`: 공개 HTTPS 인증서 만료 14일 이내. - `BlackboxExporterUnavailable`: synthetic probe exporter Pod not ready 2분 지속. @@ -257,7 +262,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 운영 적용 전제는 다음과 같습니다. - `/metrics`는 cluster 내부 scrape 전용이며 외부 Ingress에 노출하지 않습니다. -- 공개 URL synthetic probe는 monitoring namespace 내부에서 외부 도메인으로 나가는 HTTPS 요청입니다. DNS, outbound network, TLS, Traefik/frontend 경로를 함께 검증하지만 브라우저 JS runtime error를 대체하지는 않습니다. +- 공개 URL synthetic probe는 monitoring namespace 내부에서 외부 도메인으로 나가는 HTTPS 요청입니다. DNS, outbound network, TLS, Traefik/frontend/hub-auth 경로를 함께 검증하지만 브라우저 JS runtime error를 대체하지는 않습니다. - Discord webhook URL은 Git에 저장하지 않습니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. - OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지하고 dev overlay에서 먼저 활성화합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index f7ec9cf1..3ee72573 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -54,7 +54,7 @@ Loki 로그 확인: {namespace="kube-system", app_kubernetes_io_name="traefik"} | json {namespace="", app="frontend"} | json {namespace="", app="backend"} | json -{namespace="", container=~"backend|celery-worker|judge-server|vllm"} +{namespace="", container=~"backend|hub-auth|celery-worker|judge-server|vllm"} ``` Loki/Alloy 상태 확인: @@ -96,6 +96,8 @@ probe_duration_seconds{probe_type="public-http"} (probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) / 86400 ``` +probe 대상은 frontend와 hub-auth를 모두 포함합니다. `service` label로 어느 endpoint인지 구분합니다. + Longhorn 상태 확인: ```sh @@ -221,15 +223,16 @@ probe_http_duration_seconds{probe_type="public-http"} ``` ```sh -kubectl -n monitoring get pod,svc,probe | grep -E 'blackbox|codeplace-public' +kubectl -n monitoring get pod,svc,probe | grep -E 'blackbox|codeplace-public|hub-auth' kubectl -n monitoring logs deploy/blackbox-exporter --tail=200 -kubectl -n get ingress,svc,endpoints frontend +kubectl -n get ingress,svc,endpoints frontend hub-auth kubectl -n kube-system logs deploy/traefik --tail=200 ``` 판단: - `probe_success=0`이고 ingress/backend 지표는 정상이면 DNS, TLS, Traefik public route, frontend nginx 기본 경로 문제를 먼저 봅니다. +- `service="hub-auth"` probe가 실패하면 OAuth helper Pod, Service, Ingress, GitHub OAuth secret 설정을 우선 확인합니다. - `probe_http_status_code`가 0이면 DNS/TCP/TLS 연결 실패 가능성이 큽니다. - 5xx이면 Traefik/frontend/backend 로그와 같은 시간대를 대조합니다. - TLS 만료 알림은 Traefik ACME resolver와 인증서 저장소를 확인합니다. diff --git a/kubernetes/base/hub-auth/deployment.yaml b/kubernetes/base/hub-auth/deployment.yaml index ea9725d4..62516cc1 100644 --- a/kubernetes/base/hub-auth/deployment.yaml +++ b/kubernetes/base/hub-auth/deployment.yaml @@ -27,6 +27,23 @@ spec: imagePullPolicy: Always ports: - containerPort: 80 + name: http + startupProbe: + tcpSocket: + port: http + periodSeconds: 5 + failureThreshold: 12 + readinessProbe: + tcpSocket: + port: http + periodSeconds: 10 + failureThreshold: 3 + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 3 resources: requests: memory: "128Mi" diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 652a1ede..d6533f9d 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -283,7 +283,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min by (namespace, pod) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}}"} + {"expr": "min by (namespace, pod) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}}"} ] }, { @@ -292,7 +292,7 @@ data: "gridPos": {"x": 6, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} ] }, { @@ -301,7 +301,7 @@ data: "gridPos": {"x": 12, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "cores"}}, "targets": [ - {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} ] }, { @@ -310,7 +310,7 @@ data: "gridPos": {"x": 18, "y": 29, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ - {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} ] }, { diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index ddd96498..500b6f9e 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -198,7 +198,7 @@ data: }, { "type": "logs", - "title": "Judge, Celery, and vLLM Logs", + "title": "Auth, Judge, Celery, and vLLM Logs", "gridPos": {"x": 14, "y": 4, "w": 10, "h": 10}, "datasource": "Loki", "options": { @@ -208,7 +208,7 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=~\"$namespace\", container=~\"celery-worker|celery-beat|judge-server|vllm\"}"} + {"expr": "{namespace=~\"$namespace\", container=~\"hub-auth|celery-worker|celery-beat|judge-server|vllm\"}"} ] }, { diff --git a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml index c26d3aae..57e24229 100644 --- a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml +++ b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml @@ -21,7 +21,7 @@ data: "panels": [ { "type": "stat", - "title": "Prod Endpoint Up", + "title": "Prod Endpoints Up", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -38,12 +38,12 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "probe_success{job=\"codeplace-public-http-prod\", environment=\"prod\"}", "legendFormat": "{{instance}}"} + {"expr": "probe_success{environment=\"prod\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{instance}}"} ] }, { "type": "stat", - "title": "Dev Endpoint Up", + "title": "Dev Endpoints Up", "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -60,7 +60,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "probe_success{job=\"codeplace-public-http-dev\", environment=\"dev\"}", "legendFormat": "{{instance}}"} + {"expr": "probe_success{environment=\"dev\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{instance}}"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 2966d0c6..87a6fc6b 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -48,15 +48,15 @@ spec: summary: "Ingress 5xx rate is high in prod" description: "Traefik 5xx responses for code-place-prod are above 5% for 2 minutes." - alert: PublicEndpointDown - expr: probe_success{job="codeplace-public-http-prod", environment="prod"} == 0 + expr: probe_success{environment="prod", probe_type="public-http"} == 0 for: 1m labels: severity: critical priority: P0 namespace: code-place-prod annotations: - summary: "Public frontend endpoint is down in prod" - description: "The synthetic HTTPS probe for code.pusan.ac.kr has failed for 1 minute." + summary: "Public endpoint is down in prod" + description: "A production synthetic HTTPS probe has failed for 1 minute." - alert: PublicEndpointTLSCritical expr: (probe_ssl_earliest_cert_expiry{probe_type="public-http", environment="prod"} - time()) < 259200 for: 15m @@ -205,15 +205,15 @@ spec: summary: "Backend API p95 latency is high" description: "Backend API p95 latency is above 2 seconds for 5 minutes." - alert: PublicEndpointDown - expr: probe_success{job="codeplace-public-http-dev", environment="dev"} == 0 + expr: probe_success{environment="dev", probe_type="public-http"} == 0 for: 2m labels: severity: warning priority: P1 namespace: code-place-dev annotations: - summary: "Public frontend endpoint is down in dev" - description: "The synthetic HTTPS probe for k3s.code-place-dev.site has failed for 2 minutes." + summary: "Public endpoint is down in dev" + description: "A dev synthetic HTTPS probe has failed for 2 minutes." - alert: PublicEndpointLatencyHigh expr: probe_duration_seconds{probe_type="public-http"} > 2 for: 5m @@ -296,7 +296,7 @@ spec: summary: "Celery task runtime is high" description: "Celery task p95 runtime has been above 120 seconds for 10 minutes." - alert: PodCrashLooping - expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|vllm-.*"}[5m])) > 0 + expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|vllm-.*"}[5m])) > 0 for: 5m labels: severity: warning @@ -305,7 +305,7 @@ spec: summary: "CodePlace pod is restarting" description: "A CodePlace application pod has restarted within the last 5 minutes." - alert: CodePlacePodNotReady - expr: max by (namespace, pod) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}) == 0 + expr: max by (namespace, pod) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}) == 0 for: 5m labels: severity: warning @@ -314,7 +314,7 @@ spec: summary: "CodePlace pod is not ready" description: "A CodePlace application or datastore pod has not been ready for 5 minutes." - alert: CodePlaceContainerCPUHigh - expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 0.001) > 0.8 + expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 0.001) > 0.8 for: 10m labels: severity: warning @@ -323,7 +323,7 @@ spec: summary: "CodePlace container CPU usage is high" description: "A CodePlace container has used more than 80% of its CPU limit for 10 minutes." - alert: CodePlaceContainerMemoryHigh - expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="memory", unit="byte", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 1) > 0.85 + expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="memory", unit="byte", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 1) > 0.85 for: 10m labels: severity: warning diff --git a/kubernetes/monitoring/public-endpoint-probes.yaml b/kubernetes/monitoring/public-endpoint-probes.yaml index f94b45e9..38addfef 100644 --- a/kubernetes/monitoring/public-endpoint-probes.yaml +++ b/kubernetes/monitoring/public-endpoint-probes.yaml @@ -43,3 +43,49 @@ spec: namespace: code-place-prod service: frontend probe_type: public-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-hub-auth-dev-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-hub-auth-http-dev + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://hub-auth-dev.code-place-dev.site/ + labels: + environment: dev + namespace: code-place-dev + service: hub-auth + probe_type: public-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-hub-auth-prod-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-hub-auth-http-prod + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://hub-auth.code-place-dev.site/ + labels: + environment: prod + namespace: code-place-prod + service: hub-auth + probe_type: public-http From f23601166374f2874dc97d128b1eb90779007d73 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 15:20:01 +0900 Subject: [PATCH 035/147] =?UTF-8?q?GPU=20=EA=B4=80=EC=B8=A1=EC=84=B1=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 21 +++- .../infra/observability-runbook/_index.md | 71 ++++++++++++ kubernetes/base/vllm/README.md | 5 +- kubernetes/monitoring/dcgm-exporter.yaml | 94 +++++++++++++++ .../grafana-dashboard-ai-inference.yaml | 109 ++++++++++++++++++ kubernetes/monitoring/kustomization.yaml | 1 + kubernetes/monitoring/prometheus-rules.yaml | 50 ++++++++ 7 files changed, 348 insertions(+), 3 deletions(-) create mode 100644 kubernetes/monitoring/dcgm-exporter.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index c1beed22..08a2a5d0 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -15,7 +15,7 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 - **User Path:** synthetic public HTTPS probe, Traefik/Ingress, frontend nginx, hub-auth OAuth helper, backend Django API. - **Submission/Judge Path:** submission 생성, Celery `judge_task`, Redis `waiting_queue`, judge-server heartbeat, judge-server `/judge` 호출, 결과 저장. -- **AI Hint Path:** backend AI hint API, vLLM OpenAI-compatible server, vLLM HF cache PVC. +- **AI Hint Path:** backend AI hint API, vLLM OpenAI-compatible server, NVIDIA GPU/DCGM, vLLM HF cache PVC. - **Async Jobs:** celery-worker, celery-beat, scheduled tasks. - **Data Stores:** PostgreSQL/CNPG, Redis Sentinel. - **Platform/Storage:** backend/frontend/celery/judge Pod, node, PVC, Longhorn volume/node/disk. @@ -59,6 +59,16 @@ prod AI 힌트용 vLLM은 OpenAI-compatible server의 `/metrics`를 ServiceMonit - `vllm:prompt_tokens` - `vllm:generation_tokens` +vLLM이 사용하는 GPU node에는 NVIDIA DCGM exporter를 DaemonSet으로 배포합니다. 현재는 vLLM node label인 `workload.code-place.ai/vllm=true`가 붙은 node만 대상으로 하며, Prometheus는 DCGM exporter의 `:9400/metrics`를 scrape합니다. + +- `DCGM_FI_DEV_GPU_UTIL` +- `DCGM_FI_DEV_FB_USED` +- `DCGM_FI_DEV_FB_FREE` +- `DCGM_FI_DEV_GPU_TEMP` +- `DCGM_FI_DEV_XID_ERRORS` +- `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` +- `DCGM_FI_PROF_DRAM_ACTIVE` + `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. ### Logs @@ -165,6 +175,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `longhorn-service-monitor.yaml`: Longhorn manager `/metrics` scrape, interval 30s. - `grafana-dashboard-storage.yaml`: Longhorn volume/node/disk 상태와 storage capacity dashboard. - `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. +- `dcgm-exporter.yaml`: vLLM GPU node용 NVIDIA DCGM exporter DaemonSet/Service/ServiceMonitor. - `validate.sh`: monitoring YAML, Grafana dashboard JSON, kustomize render, 선택적 promtool/Helm render 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. @@ -194,6 +205,7 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `LonghornVolumeFaulted`: Longhorn volume faulted 1분 지속. - `LonghornVolumeReadOnly`: Longhorn volume filesystem read-only 1분 지속. - `VLLMUnavailable`: prod vLLM `/metrics` scrape 실패 2분 지속. +- `GPUXIDError`: NVIDIA GPU XID error 1분 지속. ### P1 @@ -232,6 +244,10 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `VLLMWaitingQueueHigh`: prod vLLM waiting request 5 초과 5분 지속. - `VLLMKVCacheHigh`: prod vLLM KV cache 사용률 90% 초과 10분 지속. - `VLLMRequestLatencyHigh`: prod vLLM p95 e2e latency 60초 초과 10분 지속. +- `DCGMExporterUnavailable`: DCGM exporter DaemonSet unavailable 2분 지속. +- `GPUUtilizationHigh`: GPU utilization 95% 초과 15분 지속. +- `GPUMemoryHigh`: GPU framebuffer memory usage 90% 초과 10분 지속. +- `GPUTemperatureHigh`: GPU temperature 85C 초과 10분 지속. - `CodePlaceNodePressure`: node memory/disk/PID pressure 5분 지속. ## 5. 운영 확인 절차 @@ -254,7 +270,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 11. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 12. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. 13. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. -14. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC panel이 비어 있지 않은지 확인합니다. +14. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. 15. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. 16. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. 17. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. @@ -275,6 +291,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Backend/Celery Sentry release는 `SENTRY_RELEASE` env가 있으면 사용합니다. 이미지 태그와 동일한 commit SHA를 넣어 Grafana/Loki/Sentry 간 배포 기준을 맞추는 것을 권장합니다. - Tempo tracing은 monitoring namespace에 배포하지만, prod app trace export는 dev 확인 후 별도 승격합니다. - vLLM은 prod overlay에만 포함되어 있으므로 vLLM ServiceMonitor와 AI Inference dashboard도 `code-place-prod`를 기준으로 조회합니다. dev에 vLLM을 올리면 별도 dev ServiceMonitor/Probe 또는 namespace selector 확장이 필요합니다. +- DCGM exporter는 `workload.code-place.ai/vllm=true` node label을 기준으로 배포합니다. GPU node가 늘어나거나 vLLM 외 GPU workload를 운영하면 selector와 dashboard scope를 확장해야 합니다. ## 6. 검증 diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 3ee72573..2fb63811 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -34,6 +34,7 @@ Status -> Targets -> postgres Status -> Targets -> redis Status -> Targets -> longhorn Status -> Targets -> vllm +Status -> Targets -> dcgm-exporter Status -> Targets -> codeplace-public-http-dev Status -> Targets -> codeplace-public-http-prod ``` @@ -127,6 +128,21 @@ up{job="vllm", namespace="code-place-prod"} histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) ``` +GPU 상태 확인: + +```sh +kubectl -n monitoring get ds,svc,servicemonitor dcgm-exporter +kubectl -n monitoring logs daemonset/dcgm-exporter --tail=100 +kubectl -n code-place-prod get pod -l app=vllm -o wide +``` + +```promql +DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"} +DCGM_FI_DEV_FB_USED{namespace="monitoring"} / clamp_min(DCGM_FI_DEV_FB_USED{namespace="monitoring"} + DCGM_FI_DEV_FB_FREE{namespace="monitoring"}, 1) +DCGM_FI_DEV_GPU_TEMP{namespace="monitoring"} +DCGM_FI_DEV_XID_ERRORS{namespace="monitoring"} +``` + ## P0 ### BackendTargetDown @@ -409,6 +425,31 @@ kube_pod_status_ready{namespace="code-place-prod", pod=~"vllm-.*", condition="tr - Pod는 ready인데 scrape만 실패하면 Service port/path, vLLM `/metrics`, NetworkPolicy 여부를 확인합니다. - backend AI hint API error가 같이 증가하면 사용자 영향이 있습니다. +### GPUXIDError + +증상: NVIDIA DCGM이 GPU XID error를 1분 이상 보고합니다. vLLM이 ready여도 GPU driver, ECC, PCIe, memory, thermal 문제로 추론 장애가 날 수 있습니다. + +확인: + +```sh +kubectl -n monitoring logs daemonset/dcgm-exporter --tail=200 +kubectl -n code-place-prod get pod -l app=vllm -o wide +kubectl -n code-place-prod logs deploy/vllm --tail=200 +kubectl describe node -l workload.code-place.ai/vllm=true +``` + +```promql +DCGM_FI_DEV_XID_ERRORS{namespace="monitoring"} +DCGM_FI_DEV_GPU_TEMP{namespace="monitoring"} +DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"} +``` + +판단: + +- XID가 증가하고 vLLM latency/5xx가 같이 증가하면 AI hint 사용자 영향으로 봅니다. +- 같은 node에서 temperature나 utilization도 높으면 cooling, sustained load, `max-num-seqs`, KV cache 설정을 함께 낮춰 봅니다. +- XID가 반복되면 Pod 재시작만으로 닫지 말고 node driver/runtime 상태와 GPU hardware 상태를 확인합니다. + ## P1 ### OTelCollectorUnavailable / TempoUnavailable / TempoPVCAlmostFull @@ -636,6 +677,36 @@ kubectl -n code-place-prod describe pvc vllm-hf-cache - e2e latency만 증가하면 prompt/generation token 증가, backend streaming read timeout, vLLM 로그의 model/engine warning을 확인합니다. 현재 vLLM은 prod 전용입니다. dev에서 AI hint를 별도 검증하려면 dev vLLM 배포와 ServiceMonitor 확장이 먼저 필요합니다. +### DCGMExporterUnavailable / GPUUtilizationHigh / GPUMemoryHigh / GPUTemperatureHigh + +확인: + +```sh +kubectl -n monitoring get daemonset,pod,svc,servicemonitor dcgm-exporter -o wide +kubectl -n monitoring describe daemonset dcgm-exporter +kubectl -n monitoring logs daemonset/dcgm-exporter --tail=200 +kubectl get node -l workload.code-place.ai/vllm=true +``` + +PromQL: + +```promql +kube_daemonset_status_number_available{namespace="monitoring", daemonset="dcgm-exporter"} +kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"} +DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"} +DCGM_FI_DEV_FB_USED{namespace="monitoring"} / clamp_min(DCGM_FI_DEV_FB_USED{namespace="monitoring"} + DCGM_FI_DEV_FB_FREE{namespace="monitoring"}, 1) +DCGM_FI_DEV_GPU_TEMP{namespace="monitoring"} +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace="monitoring"} +DCGM_FI_PROF_DRAM_ACTIVE{namespace="monitoring"} +``` + +판단: + +- exporter가 unavailable이면 GPU 상태를 볼 수 없습니다. vLLM node label, `runtimeClassName: nvidia`, NVIDIA device plugin/runtime, pod scheduling event를 확인합니다. +- GPU utilization만 높고 memory 여유가 있으면 request concurrency와 vLLM scheduling capacity 문제일 가능성이 큽니다. +- framebuffer memory usage가 높으면 KV cache pressure, `max-num-seqs`, `max-model-len`, model load 상태를 우선 봅니다. +- temperature가 높으면 AI latency보다 node cooling/thermal throttling을 먼저 의심합니다. + ### LokiPVCAlmostFull 확인: diff --git a/kubernetes/base/vllm/README.md b/kubernetes/base/vllm/README.md index c323b551..4d0fdf2f 100644 --- a/kubernetes/base/vllm/README.md +++ b/kubernetes/base/vllm/README.md @@ -57,10 +57,12 @@ kubectl describe pod -n code-place-prod -l app=vllm vLLM OpenAI-compatible server는 `/metrics`에서 Prometheus metrics를 제공합니다. CodePlace monitoring kustomization은 prod vLLM Service를 `vllm-service-monitor.yaml`로 scrape합니다. +같은 monitoring kustomization은 `dcgm-exporter.yaml`도 배포해 `workload.code-place.ai/vllm=true` node의 NVIDIA GPU metric을 수집합니다. ```bash kubectl apply -k kubernetes/monitoring kubectl -n monitoring get servicemonitor vllm +kubectl -n monitoring get daemonset,servicemonitor dcgm-exporter ``` Grafana에서는 `CodePlace AI Inference` dashboard에서 다음 상태를 확인합니다. @@ -71,5 +73,6 @@ Grafana에서는 `CodePlace AI Inference` dashboard에서 다음 상태를 확 - p95 e2e latency, queue latency, time-to-first-token. - prompt/generation token throughput. - `vllm-hf-cache` PVC usage. +- GPU utilization, framebuffer memory usage, temperature, XID error. -알림은 vLLM scrape 실패를 P0로 처리하고, waiting queue 증가, KV cache 90% 초과, p95 latency 60초 초과는 P1로 처리합니다. +알림은 vLLM scrape 실패와 GPU XID error를 P0로 처리합니다. waiting queue 증가, KV cache 90% 초과, p95 latency 60초 초과, DCGM exporter unavailable, GPU utilization 95% 초과, GPU framebuffer memory 90% 초과, GPU temperature 85C 초과는 P1로 처리합니다. diff --git a/kubernetes/monitoring/dcgm-exporter.yaml b/kubernetes/monitoring/dcgm-exporter.yaml new file mode 100644 index 00000000..a574bd9a --- /dev/null +++ b/kubernetes/monitoring/dcgm-exporter.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dcgm-exporter + labels: + app.kubernetes.io/name: dcgm-exporter + app.kubernetes.io/version: "4.8.2" +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: dcgm-exporter + template: + metadata: + labels: + app.kubernetes.io/name: dcgm-exporter + app.kubernetes.io/version: "4.8.2" + spec: + automountServiceAccountToken: false + runtimeClassName: nvidia + nodeSelector: + workload.code-place.ai/vllm: "true" + containers: + - name: dcgm-exporter + image: nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless + env: + - name: DCGM_EXPORTER_LISTEN + value: ":9400" + - name: DCGM_EXPORTER_KUBERNETES + value: "true" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: metrics + containerPort: 9400 + securityContext: + runAsNonRoot: false + runAsUser: 0 + allowPrivilegeEscalation: false + capabilities: + add: + - SYS_ADMIN + drop: + - ALL + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + volumeMounts: + - name: pod-gpu-resources + mountPath: /var/lib/kubelet/pod-resources + readOnly: true + volumes: + - name: pod-gpu-resources + hostPath: + path: /var/lib/kubelet/pod-resources +--- +apiVersion: v1 +kind: Service +metadata: + name: dcgm-exporter + labels: + app.kubernetes.io/name: dcgm-exporter + app.kubernetes.io/version: "4.8.2" +spec: + selector: + app.kubernetes.io/name: dcgm-exporter + ports: + - name: metrics + port: 9400 + targetPort: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: dcgm-exporter + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: dcgm-exporter + jobLabel: app.kubernetes.io/name + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml index 263a24d9..c18787b7 100644 --- a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml +++ b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml @@ -207,6 +207,115 @@ data: "targets": [ {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} ] + }, + { + "type": "timeseries", + "title": "GPU Utilization", + "gridPos": {"x": 0, "y": 20, "w": 6, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 80}, + {"color": "red", "value": 95} + ] + } + } + }, + "targets": [ + {"expr": "DCGM_FI_DEV_GPU_UTIL{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}"} + ] + }, + { + "type": "timeseries", + "title": "GPU FB Memory Usage", + "gridPos": {"x": 6, "y": 20, "w": 6, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.8}, + {"color": "red", "value": 0.9} + ] + } + } + }, + "targets": [ + {"expr": "DCGM_FI_DEV_FB_USED{namespace=\"monitoring\"} / clamp_min(DCGM_FI_DEV_FB_USED{namespace=\"monitoring\"} + DCGM_FI_DEV_FB_FREE{namespace=\"monitoring\"}, 1)", "legendFormat": "{{Hostname}} gpu{{gpu}}"} + ] + }, + { + "type": "timeseries", + "title": "GPU Temperature", + "gridPos": {"x": 12, "y": 20, "w": 6, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 75}, + {"color": "red", "value": 85} + ] + } + } + }, + "targets": [ + {"expr": "DCGM_FI_DEV_GPU_TEMP{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}"} + ] + }, + { + "type": "timeseries", + "title": "GPU XID Errors", + "gridPos": {"x": 18, "y": 20, "w": 6, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "targets": [ + {"expr": "DCGM_FI_DEV_XID_ERRORS{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}"} + ] + }, + { + "type": "timeseries", + "title": "GPU Tensor / DRAM Activity", + "gridPos": {"x": 0, "y": 28, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "targets": [ + {"expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}} tensor"}, + {"expr": "DCGM_FI_PROF_DRAM_ACTIVE{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}} dram"} + ] + }, + { + "type": "timeseries", + "title": "GPU FB Memory Bytes", + "gridPos": {"x": 12, "y": 28, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "DCGM_FI_DEV_FB_USED{namespace=\"monitoring\"} * 1024 * 1024", "legendFormat": "{{Hostname}} gpu{{gpu}} used"}, + {"expr": "DCGM_FI_DEV_FB_FREE{namespace=\"monitoring\"} * 1024 * 1024", "legendFormat": "{{Hostname}} gpu{{gpu}} free"} + ] } ] } diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index f2cedb66..fc25304f 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -6,6 +6,7 @@ resources: - ./alertmanager-config.yaml - ./backend-service-monitor.yaml - ./blackbox-exporter.yaml + - ./dcgm-exporter.yaml - ./datastore-pod-monitors.yaml - ./grafana-dashboard-codeplace.yaml - ./grafana-dashboard-ai-inference.yaml diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 87a6fc6b..bf587f17 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -192,6 +192,16 @@ spec: annotations: summary: "vLLM metrics target is unavailable" description: "Prometheus cannot scrape the vLLM /metrics endpoint for 2 minutes. AI hint serving may be unavailable." + - alert: GPUXIDError + expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_XID_ERRORS{namespace="monitoring"}) > 0 + for: 1m + labels: + severity: critical + priority: P0 + namespace: monitoring + annotations: + summary: "GPU driver reported an XID error" + description: "DCGM reports a non-zero NVIDIA XID error for 1 minute. vLLM GPU health and node driver state should be checked." - name: codeplace.p1 interval: 30s rules: @@ -507,6 +517,46 @@ spec: annotations: summary: "vLLM request latency is high" description: "vLLM p95 end-to-end request latency has been above 60 seconds for 10 minutes." + - alert: DCGMExporterUnavailable + expr: (kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"} == 0) or (kube_daemonset_status_number_available{namespace="monitoring", daemonset="dcgm-exporter"} < kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"}) or absent(kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "DCGM exporter is not running on the GPU node" + description: "The NVIDIA DCGM exporter DaemonSet is unavailable for 2 minutes. GPU utilization, memory, temperature, and XID metrics may be missing." + - alert: GPUUtilizationHigh + expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"}) > 95 + for: 15m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "GPU utilization is high" + description: "DCGM reports GPU utilization above 95% for 15 minutes. vLLM AI hint latency may rise under sustained load." + - alert: GPUMemoryHigh + expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_FB_USED{namespace="monitoring"} / clamp_min(DCGM_FI_DEV_FB_USED{namespace="monitoring"} + DCGM_FI_DEV_FB_FREE{namespace="monitoring"}, 1)) > 0.9 + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "GPU framebuffer memory usage is high" + description: "DCGM reports GPU framebuffer memory usage above 90% for 10 minutes. vLLM KV cache and max sequence settings should be checked." + - alert: GPUTemperatureHigh + expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_GPU_TEMP{namespace="monitoring"}) > 85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "GPU temperature is high" + description: "DCGM reports GPU temperature above 85C for 10 minutes. Node cooling and sustained vLLM load should be checked." - alert: CodePlaceNodePressure expr: kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure", status="true"} == 1 for: 5m From 8a6b4bb5040c4393e63039ae6159112235c18189 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 15:28:24 +0900 Subject: [PATCH 036/147] =?UTF-8?q?=ED=94=84=EB=A1=A0=ED=8A=B8=EC=97=94?= =?UTF-8?q?=EB=93=9C=20=EC=98=A4=EB=A5=98=20=EA=B4=80=EC=B8=A1=EC=84=B1=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/oj/urls.py | 2 + backend/utils/json_logging.py | 7 ++ backend/utils/observability_metrics.py | 5 + backend/utils/tests.py | 53 +++++++++ backend/utils/views.py | 68 +++++++++++ .../observability-improvement-plan/_index.md | 22 +++- .../infra/observability-runbook/_index.md | 27 ++++- frontend/config/sentry.js | 8 +- frontend/src/pages/admin/index.js | 2 + frontend/src/pages/oj/index.js | 2 + frontend/src/utils/client_error_reporter.js | 108 ++++++++++++++++++ frontend/src/utils/sentry.js | 7 +- .../grafana-dashboard-codeplace.yaml | 21 ++++ .../monitoring/grafana-dashboard-logs.yaml | 15 +++ kubernetes/monitoring/prometheus-rules.yaml | 9 ++ 15 files changed, 340 insertions(+), 16 deletions(-) create mode 100644 frontend/src/utils/client_error_reporter.js diff --git a/backend/oj/urls.py b/backend/oj/urls.py index 7f25e9c2..1808967a 100644 --- a/backend/oj/urls.py +++ b/backend/oj/urls.py @@ -1,11 +1,13 @@ from django.urls import include, re_path from utils.observability_metrics import register_codeplace_metrics +from utils.views import ClientErrorReportAPIView register_codeplace_metrics() urlpatterns = [ re_path(r"^", include("django_prometheus.urls")), + re_path(r"^api/client_error/?$", ClientErrorReportAPIView.as_view(), name="client_error_report_api"), re_path(r"^api/", include("account.urls.oj")), re_path(r"^api/admin/", include("account.urls.admin")), re_path(r"^api/", include("announcement.urls.oj")), diff --git a/backend/utils/json_logging.py b/backend/utils/json_logging.py index 62b213ad..ffa4f5b9 100644 --- a/backend/utils/json_logging.py +++ b/backend/utils/json_logging.py @@ -51,6 +51,13 @@ def format(self, record): "task_status", "submission_id", "retry_reason", + "frontend_surface", + "frontend_error_type", + "frontend_message", + "frontend_route", + "frontend_release", + "frontend_component", + "frontend_info", ): value = getattr(record, key, None) if value is not None: diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 4835ab94..6294e0c0 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -46,6 +46,11 @@ ["method", "endpoint"], buckets=(0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10), ) +FRONTEND_ERROR_TOTAL = Counter( + "codeplace_frontend_error_total", + "Total frontend runtime errors reported by browser clients.", + ["surface", "error_type"], +) class CodePlaceCollector: diff --git a/backend/utils/tests.py b/backend/utils/tests.py index f0c23377..70e6b589 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -201,6 +201,59 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): self.assertIn("codeplace_judge_server_available", body) +class ClientErrorReportAPITest(SimpleTestCase): + + def test_client_error_report_records_metric_and_log(self): + labels = MagicMock() + with patch("utils.views.FRONTEND_ERROR_TOTAL.labels", return_value=labels) as metric_labels, \ + patch("utils.views.frontend_error_logger") as frontend_error_logger: + response = self.client.post( + "/api/client_error", + data=json.dumps({ + "surface": "oj", + "error_type": "vue", + "message": "boom\nwith newline", + "route": "/problem/1", + "release": "20260619-test", + "component": "ProblemView", + "info": "render", + }), + content_type="application/json", + HTTP_X_REQUEST_ID="request-frontend-1", + ) + + self.assertEqual(response.status_code, 200) + metric_labels.assert_called_once_with(surface="oj", error_type="vue") + labels.inc.assert_called_once() + extra = frontend_error_logger.warning.call_args.kwargs["extra"] + self.assertEqual(extra["request_id"], "request-frontend-1") + self.assertEqual(extra["frontend_surface"], "oj") + self.assertEqual(extra["frontend_error_type"], "vue") + self.assertEqual(extra["frontend_message"], "boom with newline") + self.assertEqual(extra["frontend_route"], "/problem/1") + self.assertEqual(extra["frontend_release"], "20260619-test") + + def test_client_error_report_bounds_metric_labels(self): + labels = MagicMock() + with patch("utils.views.FRONTEND_ERROR_TOTAL.labels", return_value=labels) as metric_labels, \ + patch("utils.views.frontend_error_logger") as frontend_error_logger: + response = self.client.post( + "/api/client_error", + data=json.dumps({ + "surface": "attacker-controlled", + "error_type": "TypeError: high cardinality", + "message": "token=abc123 " + ("x" * 500), + }), + content_type="application/json", + ) + + self.assertEqual(response.status_code, 200) + metric_labels.assert_called_once_with(surface="unknown", error_type="unknown") + extra = frontend_error_logger.warning.call_args.kwargs["extra"] + self.assertTrue(extra["frontend_message"].startswith("token=[REDACTED]")) + self.assertLessEqual(len(extra["frontend_message"]), 300) + + class ObservabilityTracingTest(SimpleTestCase): def test_get_tracer_returns_noop_tracer_when_opentelemetry_is_unavailable(self): diff --git a/backend/utils/views.py b/backend/utils/views.py index 18621079..ca7c5c25 100644 --- a/backend/utils/views.py +++ b/backend/utils/views.py @@ -1,12 +1,47 @@ import os +import re from django.conf import settings from account.serializers import FileUploadForm from profile.serializers import ImageUploadForm from utils.shortcuts import rand_str from utils.api import CSRFExemptAPIView +from utils.observability_metrics import FRONTEND_ERROR_TOTAL import logging logger = logging.getLogger(__name__) +frontend_error_logger = logging.getLogger("frontend.error") + +FRONTEND_ERROR_TYPES = { + "vue", + "window_error", + "unhandled_rejection", +} +FRONTEND_SURFACES = { + "oj", + "admin", + "unknown", +} +MAX_FRONTEND_FIELD_LENGTH = 300 +FRONTEND_SECRET_PATTERNS = ( + re.compile(r"(?i)(authorization|cookie)\s*[:=]\s*[^,\s]+"), + re.compile(r"(?i)(password|token|secret|code)=([^&\s]+)"), +) + + +def _clean_frontend_field(value, default=""): + if value is None: + return default + value = str(value).replace("\r", " ").replace("\n", " ").strip() + if not value: + return default + for pattern in FRONTEND_SECRET_PATTERNS: + value = pattern.sub(lambda match: f"{match.group(1)}=[REDACTED]", value) + return value[:MAX_FRONTEND_FIELD_LENGTH] + + +def _clean_choice(value, allowed, default): + value = _clean_frontend_field(value, default) + return value if value in allowed else default class SimditorImageUploadAPIView(CSRFExemptAPIView): @@ -58,3 +93,36 @@ def post(self, request): "file_path": f"{settings.UPLOAD_PREFIX}/{file_name}", "file_name": file.name }) + + +class ClientErrorReportAPIView(CSRFExemptAPIView): + + def post(self, request): + surface = _clean_choice(request.data.get("surface"), FRONTEND_SURFACES, "unknown") + error_type = _clean_choice(request.data.get("error_type"), FRONTEND_ERROR_TYPES, "unknown") + frontend_message = _clean_frontend_field(request.data.get("message"), "unknown") + frontend_route = _clean_frontend_field(request.data.get("route"), "") + frontend_release = _clean_frontend_field(request.data.get("release"), "") + frontend_component = _clean_frontend_field(request.data.get("component"), "") + frontend_info = _clean_frontend_field(request.data.get("info"), "") + request_id = _clean_frontend_field( + request.META.get("HTTP_X_REQUEST_ID") or request.data.get("request_id"), + "", + ) + + FRONTEND_ERROR_TOTAL.labels(surface=surface, error_type=error_type).inc() + frontend_error_logger.warning( + "Frontend runtime error reported", + extra={ + "request_id": request_id or None, + "frontend_surface": surface, + "frontend_error_type": error_type, + "frontend_message": frontend_message, + "frontend_route": frontend_route, + "frontend_release": frontend_release, + "frontend_component": frontend_component, + "frontend_info": frontend_info, + "remote_addr": request.META.get("REMOTE_ADDR"), + }, + ) + return self.success({"reported": True}) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 08a2a5d0..b7f2ecd0 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -19,7 +19,7 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 - **Async Jobs:** celery-worker, celery-beat, scheduled tasks. - **Data Stores:** PostgreSQL/CNPG, Redis Sentinel. - **Platform/Storage:** backend/frontend/celery/judge Pod, node, PVC, Longhorn volume/node/disk. -- **Client/Error:** frontend runtime error, backend exception, request ID 기반 로그. +- **Client/Error:** frontend runtime error, backend exception, request ID 기반 로그, Sentry. ## 2. 구현 내용 @@ -47,6 +47,7 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_celery_task_count{task_name,status}` - `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_celery_task_last_runtime_seconds{task_name,status}` +- `codeplace_frontend_error_total{surface,error_type}` prod AI 힌트용 vLLM은 OpenAI-compatible server의 `/metrics`를 ServiceMonitor로 scrape합니다. vLLM production metrics 기준으로 다음 항목을 봅니다. @@ -75,6 +76,11 @@ vLLM이 사용하는 GPU node에는 NVIDIA DCGM exporter를 DaemonSet으로 배 backend는 `X-Request-ID`를 수용하고, 없으면 request ID를 생성한 뒤 응답 header로 반환합니다. Frontend axios 요청은 `X-Request-ID`를 생성해 backend로 전달하고, backend가 반환한 request ID를 axios response/error와 Sentry request context에 보존합니다. API 요청에서 Celery task를 enqueue할 때 request ID를 Celery header로 전파하고, worker 실행 중 같은 request ID를 JSON log context에 복원합니다. +frontend runtime error는 두 경로로 수집합니다. + +- Sentry가 build-time `SENTRY_DSN`으로 활성화된 경우 Sentry event로 전송합니다. +- Sentry 설정이 빠져도 Vue error, `window.error`, `unhandledrejection`은 `/api/client_error`로 sanitized report를 보내고, backend가 `frontend.error` JSON log와 `codeplace_frontend_error_total{surface,error_type}` metric을 남깁니다. + Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설정합니다. JSON 로그 필드는 다음을 기준으로 합니다. - `timestamp` @@ -94,6 +100,11 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 - `task_status` - `trace_id` - `span_id` +- `frontend_surface` +- `frontend_error_type` +- `frontend_message` +- `frontend_route` +- `frontend_release` 민감 정보는 로그에 싣지 않습니다. token, password, cookie, OAuth secret, authorization header, 제출 source code는 저장 대상이 아닙니다. OpenTelemetry가 활성화된 요청/작업 로그에는 현재 span의 `trace_id`, `span_id`를 추가해 Tempo trace와 Loki JSON log를 연결합니다. @@ -215,6 +226,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PublicEndpointDown`: dev 공개 HTTPS endpoint probe 실패 2분 지속. frontend와 hub-auth를 포함합니다. - `PublicEndpointLatencyHigh`: 공개 HTTPS probe duration 2초 초과 5분 지속. - `PublicEndpointTLSExpiringSoon`: 공개 HTTPS 인증서 만료 14일 이내. +- `FrontendRuntimeErrorsSpike`: browser runtime error report 10개 초과/5분, 5분 지속. - `BlackboxExporterUnavailable`: synthetic probe exporter Pod not ready 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. @@ -265,8 +277,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. 8. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -9. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. -10. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, judge/celery log panel을 확인합니다. +9. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. +10. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 11. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 12. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. 13. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. @@ -286,8 +298,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - PostgreSQL은 CNPG instance exporter와 PodMonitor로 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor로 `redis_*` 지표를 수집합니다. - Sentry backend SDK는 기본 PII 자동 전송을 비활성화하고 전송 직전 `authorization`, `cookie`, `password`, `token`, `secret`, 제출 source code 계열 필드를 redaction합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. -- Frontend는 axios 요청마다 `X-Request-ID`를 전파하고 마지막 request ID를 `window.__CODEPLACE_LAST_REQUEST_ID__`와 Sentry request context에 저장합니다. -- Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. +- Frontend는 axios 요청마다 `X-Request-ID`를 전파하고 마지막 request ID를 `window.__CODEPLACE_LAST_REQUEST_ID__`와 Sentry request context 및 `/api/client_error` report에 저장합니다. +- Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. DSN이 없으면 Sentry는 비활성화되지만 `/api/client_error` 기반 metric/log는 계속 동작합니다. - Backend/Celery Sentry release는 `SENTRY_RELEASE` env가 있으면 사용합니다. 이미지 태그와 동일한 commit SHA를 넣어 Grafana/Loki/Sentry 간 배포 기준을 맞추는 것을 권장합니다. - Tempo tracing은 monitoring namespace에 배포하지만, prod app trace export는 dev 확인 후 별도 승격합니다. - vLLM은 prod overlay에만 포함되어 있으므로 vLLM ServiceMonitor와 AI Inference dashboard도 `code-place-prod`를 기준으로 조회합니다. dev에 vLLM을 올리면 별도 dev ServiceMonitor/Probe 또는 namespace selector 확장이 필요합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 2fb63811..d5b37845 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -55,6 +55,7 @@ Loki 로그 확인: {namespace="kube-system", app_kubernetes_io_name="traefik"} | json {namespace="", app="frontend"} | json {namespace="", app="backend"} | json +{namespace="", app="backend"} | json | logger="frontend.error" {namespace="", container=~"backend|hub-auth|celery-worker|judge-server|vllm"} ``` @@ -194,8 +195,9 @@ Grafana Explore: - 5xx와 latency가 같이 오르면 backend/DB/Redis 병목 가능성이 큽니다. - 5xx만 증가하면 최근 배포, exception, 외부 dependency를 우선 확인합니다. - Sentry event가 있으면 `environment`, `release`, `request_id` tag를 기준으로 같은 시간대 Loki 로그와 대조합니다. +- Sentry event가 없더라도 `codeplace_frontend_error_total`과 `frontend.error` backend JSON log를 확인합니다. - 사용자 브라우저에서 재현 중이면 `window.__CODEPLACE_LAST_REQUEST_ID__` 값을 확인해 같은 request ID로 Loki/backend JSON log를 조회합니다. -- Frontend Sentry 설정은 image build 시점에 결정됩니다. `SENTRY_DSN`, `USE_SENTRY`, `SENTRY_ENVIRONMENT`, `APP_VERSION` 변경 후에는 frontend image rebuild가 필요합니다. +- Frontend Sentry 설정은 image build 시점에 결정됩니다. `SENTRY_DSN`, `USE_SENTRY`, `SENTRY_ENVIRONMENT`, `APP_VERSION` 변경 후에는 frontend image rebuild가 필요합니다. DSN이 빠져 있으면 Sentry는 꺼지지만 `/api/client_error` metric/log 수집은 계속 동작해야 합니다. ### Ingress5xxSpike @@ -511,6 +513,29 @@ kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condi - latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. - `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. +### FrontendRuntimeErrorsSpike + +확인: + +```promql +sum by (namespace, surface, error_type) (increase(codeplace_frontend_error_total{namespace=""}[10m])) +sum by (namespace, surface, error_type) (rate(codeplace_frontend_error_total{namespace=""}[5m])) +``` + +Grafana Explore: + +```logql +{namespace="", app="backend"} | json | logger="frontend.error" +{namespace="", app="backend"} | json | logger="frontend.error" | request_id="" +``` + +판단: + +- API 5xx/latency는 정상인데 frontend error만 증가하면 최근 frontend image, JS bundle, browser compatibility, feature flag를 먼저 봅니다. +- `surface="admin"`만 증가하면 admin bundle 또는 관리자 화면 배포 영향으로 좁힙니다. +- `error_type="unhandled_rejection"`이 많으면 API response handling, auth/session 만료 처리, network rejection을 봅니다. +- Sentry가 켜진 환경이면 같은 `release`, `environment`, `request_id`로 Sentry event와 Loki `frontend.error` 로그를 대조합니다. + ### JudgeWaitingQueueBacklog 확인: diff --git a/frontend/config/sentry.js b/frontend/config/sentry.js index b3519192..fedbab04 100644 --- a/frontend/config/sentry.js +++ b/frontend/config/sentry.js @@ -1,14 +1,12 @@ "use strict" function isSentryEnabled(nodeEnv = process.env.NODE_ENV) { - return (process.env.USE_SENTRY || (nodeEnv === "production" ? "1" : "0")) === "1" + const enabled = (process.env.USE_SENTRY || (nodeEnv === "production" ? "1" : "0")) === "1" + return enabled && Boolean(getSentryDsn()) } function getSentryDsn() { - return ( - process.env.SENTRY_DSN || - "https://143814aaa2d6e0b4550b2e5effefe90d@o4511586463776768.ingest.us.sentry.io/4511586483634176" - ) + return process.env.SENTRY_DSN || "" } function getSentryEnvironment(nodeEnv = process.env.NODE_ENV) { diff --git a/frontend/src/pages/admin/index.js b/frontend/src/pages/admin/index.js index 5d2df0a5..84d48e9a 100644 --- a/frontend/src/pages/admin/index.js +++ b/frontend/src/pages/admin/index.js @@ -25,6 +25,7 @@ import "./style.less" import ECharts from "vue-echarts" import "@/utils/echarts" import { initSentry } from "@/utils/sentry" +import { initClientErrorReporter } from "@/utils/client_error_reporter" const legacyPanelTag = "Panel" const legacySaveTag = "Save" @@ -68,5 +69,6 @@ Vue.prototype.$success = (msg) => { } initSentry(Vue) +initClientErrorReporter(Vue, { surface: "admin", router }) new Vue(Vue.util.extend({ router, store, i18n }, App)).$mount("#app") diff --git a/frontend/src/pages/oj/index.js b/frontend/src/pages/oj/index.js index 76b96a2e..8d822638 100644 --- a/frontend/src/pages/oj/index.js +++ b/frontend/src/pages/oj/index.js @@ -24,6 +24,7 @@ import filters from "@/utils/filters.js" import ECharts from "vue-echarts" import "@/utils/echarts" import { initSentry } from "@/utils/sentry" +import { initClientErrorReporter } from "@/utils/client_error_reporter" const legacyPanelTag = "Panel" @@ -59,5 +60,6 @@ Vue.prototype.$info = (s) => Vue.prototype.$Message.info(s) Vue.prototype.$success = (s) => Vue.prototype.$Message.success(s) initSentry(Vue) +initClientErrorReporter(Vue, { surface: "oj", router }) new Vue(Vue.util.extend({ router, store, i18n }, App)).$mount("#app") diff --git a/frontend/src/utils/client_error_reporter.js b/frontend/src/utils/client_error_reporter.js new file mode 100644 index 00000000..62591f91 --- /dev/null +++ b/frontend/src/utils/client_error_reporter.js @@ -0,0 +1,108 @@ +const CLIENT_ERROR_ENDPOINT = "/api/client_error" +const MAX_MESSAGE_LENGTH = 300 + +function cleanField(value) { + if (value === undefined || value === null) { + return "" + } + return String(value).replace(/[\r\n]+/g, " ").trim().slice(0, MAX_MESSAGE_LENGTH) +} + +function currentRequestId() { + return cleanField(window.__CODEPLACE_LAST_REQUEST_ID__) +} + +function currentRoute(router) { + if (router && router.currentRoute && router.currentRoute.path) { + return router.currentRoute.path + } + return window.location.pathname +} + +function reportClientError(payload) { + try { + const requestId = currentRequestId() + const body = JSON.stringify({ + ...payload, + request_id: requestId, + release: process.env.VERSION, + }) + const headers = { + "Content-Type": "application/json", + } + if (requestId) { + headers["X-Request-ID"] = requestId + } + + if (navigator.sendBeacon) { + const blob = new Blob([body], { type: "application/json" }) + if (navigator.sendBeacon(CLIENT_ERROR_ENDPOINT, blob)) { + return + } + } + + window.fetch(CLIENT_ERROR_ENDPOINT, { + method: "POST", + headers, + body, + credentials: "same-origin", + keepalive: true, + }).catch(() => {}) + } catch (error) { + console.error(error) + } +} + +function errorMessage(error) { + if (!error) { + return "unknown" + } + if (error.message) { + return error.message + } + return error.reason || error.toString() +} + +export function initClientErrorReporter(Vue, { surface = "unknown", router } = {}) { + const previousErrorHandler = Vue.config.errorHandler + + Vue.config.errorHandler = (error, vm, info) => { + reportClientError({ + surface, + error_type: "vue", + message: cleanField(errorMessage(error)), + route: cleanField(currentRoute(router)), + component: cleanField(vm && vm.$options && (vm.$options.name || vm.$options._componentTag)), + info: cleanField(info), + }) + + if (previousErrorHandler) { + previousErrorHandler.call(Vue.config, error, vm, info) + } else { + console.error(error) + } + } + + if (window.__CODEPLACE_CLIENT_ERROR_REPORTER_READY__) { + return + } + window.__CODEPLACE_CLIENT_ERROR_REPORTER_READY__ = true + + window.addEventListener("error", (event) => { + reportClientError({ + surface, + error_type: "window_error", + message: cleanField(errorMessage(event.error) || event.message), + route: cleanField(currentRoute(router)), + }) + }) + + window.addEventListener("unhandledrejection", (event) => { + reportClientError({ + surface, + error_type: "unhandled_rejection", + message: cleanField(errorMessage(event.reason)), + route: cleanField(currentRoute(router)), + }) + }) +} diff --git a/frontend/src/utils/sentry.js b/frontend/src/utils/sentry.js index 4a135b2a..a38ff528 100644 --- a/frontend/src/utils/sentry.js +++ b/frontend/src/utils/sentry.js @@ -1,8 +1,5 @@ import { init, setContext } from "@sentry/vue" -const SENTRY_DSN = - "https://143814aaa2d6e0b4550b2e5effefe90d@o4511586463776768.ingest.us.sentry.io/4511586483634176" - const SENSITIVE_EXACT_FIELDS = [ "authorization", "cookie", @@ -70,13 +67,13 @@ const options = { } export function initSentry(Vue) { - if (process.env.USE_SENTRY !== "1") { + if (process.env.USE_SENTRY !== "1" || !process.env.SENTRY_DSN) { return } init({ Vue, - dsn: process.env.SENTRY_DSN || SENTRY_DSN, + dsn: process.env.SENTRY_DSN, ...options, }) diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index d6533f9d..120020c9 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -470,6 +470,27 @@ data: {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} used"}, {"expr": "max by (namespace, pod) (redis_memory_max_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} maxmemory"} ] + }, + { + "type": "timeseries", + "title": "Frontend Runtime Errors", + "gridPos": {"x": 0, "y": 51, "w": 12, "h": 6}, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.02}, + {"color": "red", "value": 0.1} + ] + } + } + }, + "targets": [ + {"expr": "sum by (namespace, surface, error_type) (rate(codeplace_frontend_error_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{surface}} {{error_type}}"} + ] } ] } diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index 500b6f9e..bd71d69c 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -241,6 +241,21 @@ data: "targets": [ {"expr": "{namespace=~\"$namespace\"} | json | request_id=~\"$request_id\""} ] + }, + { + "type": "logs", + "title": "Frontend Runtime Error Reports", + "gridPos": {"x": 0, "y": 31, "w": 24, "h": 8}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\", app=\"backend\"} | json | logger=\"frontend.error\""} + ] } ] } diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index bf587f17..3c695c65 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -242,6 +242,15 @@ spec: annotations: summary: "Public endpoint TLS certificate expires within 14 days" description: "A CodePlace public HTTPS certificate expires within 14 days." + - alert: FrontendRuntimeErrorsSpike + expr: sum by (namespace, surface) (increase(codeplace_frontend_error_total{namespace=~"code-place-(dev|prod)"}[5m])) > 10 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Frontend runtime errors increased" + description: "Browser clients reported more than 10 frontend runtime errors in 5 minutes. Check recent deploys, Sentry events, and frontend.error logs." - alert: BlackboxExporterUnavailable expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"blackbox-exporter-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"blackbox-exporter-.*"}) for: 2m From 5778e179b05b280271cef90dbcc8d699ab71255e Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 15:35:17 +0900 Subject: [PATCH 037/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?=EC=8A=A4=ED=83=9D=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 20 +- .../infra/observability-runbook/_index.md | 70 ++++++ .../grafana-dashboard-monitoring-stack.yaml | 228 ++++++++++++++++++ kubernetes/monitoring/kustomization.yaml | 1 + kubernetes/monitoring/prometheus-rules.yaml | 60 +++++ 5 files changed, 373 insertions(+), 6 deletions(-) create mode 100644 kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index b7f2ecd0..b3e22334 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -175,6 +175,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. - `grafana-dashboard-ai-inference.yaml`: prod vLLM readiness, queue/cache/latency/token throughput, HF cache PVC dashboard. +- `grafana-dashboard-monitoring-stack.yaml`: Prometheus, Alertmanager, Grafana, Prometheus Operator readiness와 rule/notification 상태 dashboard. - `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. @@ -217,6 +218,8 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `LonghornVolumeReadOnly`: Longhorn volume filesystem read-only 1분 지속. - `VLLMUnavailable`: prod vLLM `/metrics` scrape 실패 2분 지속. - `GPUXIDError`: NVIDIA GPU XID error 1분 지속. +- `PrometheusUnavailable`: Prometheus Pod not ready 1분 지속. +- `AlertmanagerUnavailable`: Alertmanager Pod not ready 1분 지속. ### P1 @@ -228,6 +231,10 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PublicEndpointTLSExpiringSoon`: 공개 HTTPS 인증서 만료 14일 이내. - `FrontendRuntimeErrorsSpike`: browser runtime error report 10개 초과/5분, 5분 지속. - `BlackboxExporterUnavailable`: synthetic probe exporter Pod not ready 2분 지속. +- `GrafanaUnavailable`: Grafana Pod not ready 2분 지속. +- `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. +- `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. +- `PrometheusAlertmanagerDiscoveryFailed`: Prometheus가 Alertmanager target을 0개 discovery 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. - `CeleryBeatDown`: beat Pod not ready 2분 지속. @@ -280,12 +287,13 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 9. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 11. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. -12. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. -13. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. -14. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. -15. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -16. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. -17. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +12. Grafana의 `CodePlace Monitoring Stack` dashboard에서 Prometheus, Alertmanager, Grafana, operator readiness, target scrape failure, rule evaluation failure, notification failure panel을 확인합니다. +13. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. +14. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. +15. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. +16. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +17. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. +18. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index d5b37845..3a28568f 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -24,6 +24,7 @@ kubectl -n longhorn-system get pod,svc Grafana에서는 `CodePlace Overview` dashboard에서 `namespace`를 알림의 namespace로 맞춥니다. 공개 URL 장애는 `CodePlace Public Endpoints` dashboard에서 먼저 봅니다. 로그 수집이나 로그 기반 확인이 필요하면 `CodePlace Logs` dashboard를 함께 봅니다. +Prometheus, Alertmanager, Grafana 자체 이상은 `CodePlace Monitoring Stack` dashboard에서 봅니다. PVC/volume 문제가 의심되면 `CodePlace Storage` dashboard에서 Longhorn 상태를 먼저 확인합니다. Prometheus target 확인: @@ -48,6 +49,21 @@ Status -> Configuration -> p0-discord / p1-discord Email fallback을 켠 환경에서는 `Status -> Configuration`에서 `p0-email` / `p1-email` receiver도 확인합니다. +Monitoring stack 확인: + +```sh +kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|operator' +kubectl -n monitoring get prometheusrule codeplace-fast-alerts -o yaml +kubectl -n monitoring get alertmanagerconfig codeplace-alert-routing -o yaml +``` + +```promql +kube_pod_status_ready{namespace="monitoring", pod=~"prometheus-kube-prometheus-stack-prometheus-.*|alertmanager-kube-prometheus-stack-alertmanager-.*|kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} +prometheus_notifications_alertmanagers_discovered +increase(prometheus_rule_evaluation_failures_total[5m]) +sum by (job, namespace) (up == 0) +``` + Loki 로그 확인: ```logql @@ -377,6 +393,32 @@ kubectl -n monitoring describe pvc -l app.kubernetes.io/name=loki - readiness만 실패하면 Loki config, retention, disk full, compactor error를 확인합니다. - Loki가 내려가도 앱 요청 처리는 계속되지만 로그 수집/조회가 끊기므로 빠르게 복구합니다. +### PrometheusUnavailable / AlertmanagerUnavailable + +증상: Prometheus 또는 Alertmanager Pod가 1분 이상 ready가 아닙니다. + +확인: + +```sh +kubectl -n monitoring get pod,svc | grep -E 'prometheus|alertmanager' +kubectl -n monitoring describe pod prometheus-kube-prometheus-stack-prometheus-0 +kubectl -n monitoring describe pod alertmanager-kube-prometheus-stack-alertmanager-0 +kubectl -n monitoring logs prometheus-kube-prometheus-stack-prometheus-0 -c prometheus --tail=200 +kubectl -n monitoring logs alertmanager-kube-prometheus-stack-alertmanager-0 -c alertmanager --tail=200 +``` + +```promql +kube_pod_status_ready{namespace="monitoring", pod=~"prometheus-kube-prometheus-stack-prometheus-.*|alertmanager-kube-prometheus-stack-alertmanager-.*", condition="true"} +prometheus_notifications_alertmanagers_discovered +prometheus_rule_evaluation_failures_total +``` + +판단: + +- Prometheus가 down이면 metric 수집과 rule evaluation이 멈춥니다. Alertmanager가 살아 있어도 새 알림은 생성되지 않습니다. +- Alertmanager가 down이면 firing alert가 있어도 Discord/email 전달이 실패합니다. P0 수신 지연 원인으로 먼저 봅니다. +- PVC, config reload, rule syntax, resource pressure, operator reconciliation event를 함께 확인합니다. + ### LonghornManagerDown / LonghornVolumeFaulted / LonghornVolumeReadOnly 증상: Longhorn manager가 전부 ready가 아니거나, volume이 faulted/read-only 상태입니다. @@ -513,6 +555,34 @@ kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condi - latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. - `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. +### GrafanaUnavailable / PrometheusOperatorUnavailable / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed + +확인: + +```sh +kubectl -n monitoring get pod | grep -E 'grafana|operator|prometheus|alertmanager' +kubectl -n monitoring describe pod -l app.kubernetes.io/name=grafana +kubectl -n monitoring describe pod -l app.kubernetes.io/name=prometheus-operator +kubectl -n monitoring logs deploy/kube-prometheus-stack-operator --tail=200 +kubectl -n monitoring get prometheusrule,servicemonitor,podmonitor,alertmanagerconfig +``` + +PromQL: + +```promql +kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} +sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) +prometheus_notifications_alertmanagers_discovered +sum by (job, namespace) (up == 0) +``` + +판단: + +- Grafana만 down이면 알림은 계속 갈 수 있지만 dashboard/Explore 기반 원인 분석이 막힙니다. +- Operator가 down이면 새 ServiceMonitor/PodMonitor/PrometheusRule/AlertmanagerConfig 변경이 Prometheus/Alertmanager에 반영되지 않을 수 있습니다. +- rule evaluation failure가 있으면 최근 추가한 PromQL expression과 label cardinality, metric 존재 여부를 먼저 봅니다. +- Alertmanager discovery가 0이면 AlertmanagerConfig가 맞아도 Prometheus가 알림을 보낼 대상이 없습니다. Prometheus generated config와 Alertmanager Service/endpoints를 확인합니다. + ### FrontendRuntimeErrorsSpike 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml new file mode 100644 index 00000000..f04adbb1 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -0,0 +1,228 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-monitoring-stack + labels: + grafana_dashboard: "1" +data: + codeplace-monitoring-stack.json: | + { + "uid": "codeplace-monitoring-stack", + "title": "CodePlace Monitoring Stack", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "monitoring"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "Prometheus Ready", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\"}) or vector(0)", "legendFormat": "prometheus"} + ] + }, + { + "type": "stat", + "title": "Alertmanager Ready", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"alertmanager-kube-prometheus-stack-alertmanager-.*\"}) or vector(0)", "legendFormat": "alertmanager"} + ] + }, + { + "type": "stat", + "title": "Grafana Ready", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kube-prometheus-stack-grafana-.*|grafana-.*\"}) or vector(0)", "legendFormat": "grafana"} + ] + }, + { + "type": "stat", + "title": "Prometheus Operator Ready", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kube-prometheus-stack-operator-.*\"}) or vector(0)", "legendFormat": "operator"} + ] + }, + { + "type": "stat", + "title": "Alertmanager Discovered", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(prometheus_notifications_alertmanagers_discovered) or vector(0)", "legendFormat": "targets"} + ] + }, + { + "type": "stat", + "title": "Rule Eval Failures", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "failures"} + ] + }, + { + "type": "timeseries", + "title": "Monitoring Pod Restarts", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=\"monitoring\", pod=~\"prometheus-.*|alertmanager-.*|kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*\"}[1h]))", "legendFormat": "{{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Prometheus Target Scrape Failures", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (job, namespace) (up == 0) or vector(0)", "legendFormat": "{{namespace}} {{job}}"} + ] + }, + { + "type": "timeseries", + "title": "Prometheus TSDB Head Series", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "prometheus_tsdb_head_series", "legendFormat": "{{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Rule Evaluation Failures", + "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "{{rule_group}}"} + ] + }, + { + "type": "timeseries", + "title": "Alertmanager Notifications", + "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (integration) (rate(alertmanager_notifications_total[5m]))", "legendFormat": "{{integration}} sent"}, + {"expr": "sum by (integration) (rate(alertmanager_notifications_failed_total[5m])) or vector(0)", "legendFormat": "{{integration}} failed"} + ] + }, + { + "type": "timeseries", + "title": "Prometheus Resource Usage", + "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\", container=\"prometheus\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, + {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\", container=\"prometheus\"})", "legendFormat": "{{pod}} memory bytes"} + ] + }, + { + "type": "timeseries", + "title": "Grafana Resource Usage", + "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*|grafana-.*\", container!=\"\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, + {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*|grafana-.*\", container!=\"\"})", "legendFormat": "{{pod}} memory bytes"} + ] + } + ] + } diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index fc25304f..2f70c051 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -11,6 +11,7 @@ resources: - ./grafana-dashboard-codeplace.yaml - ./grafana-dashboard-ai-inference.yaml - ./grafana-dashboard-logs.yaml + - ./grafana-dashboard-monitoring-stack.yaml - ./grafana-dashboard-public-endpoints.yaml - ./grafana-dashboard-storage.yaml - ./grafana-dashboard-traces.yaml diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 3c695c65..33af6098 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -202,6 +202,26 @@ spec: annotations: summary: "GPU driver reported an XID error" description: "DCGM reports a non-zero NVIDIA XID error for 1 minute. vLLM GPU health and node driver state should be checked." + - alert: PrometheusUnavailable + expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) + for: 1m + labels: + severity: critical + priority: P0 + namespace: monitoring + annotations: + summary: "Prometheus is not ready" + description: "The kube-prometheus-stack Prometheus pod has not been ready for 1 minute. Metric collection, rule evaluation, and alert delivery may be unavailable." + - alert: AlertmanagerUnavailable + expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) + for: 1m + labels: + severity: critical + priority: P0 + namespace: monitoring + annotations: + summary: "Alertmanager is not ready" + description: "The kube-prometheus-stack Alertmanager pod has not been ready for 1 minute. Fast alert delivery may be unavailable." - name: codeplace.p1 interval: 30s rules: @@ -261,6 +281,46 @@ spec: annotations: summary: "Blackbox exporter is not ready" description: "The synthetic public endpoint probe exporter has not been ready for 2 minutes." + - alert: GrafanaUnavailable + expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Grafana is not ready" + description: "No Grafana pod in the monitoring namespace has been ready for 2 minutes. Dashboards and Explore may be unavailable." + - alert: PrometheusOperatorUnavailable + expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-operator-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-operator-.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Prometheus Operator is not ready" + description: "The kube-prometheus-stack operator has not been ready for 2 minutes. ServiceMonitor, PodMonitor, PrometheusRule, and AlertmanagerConfig reconciliation may be stalled." + - alert: PrometheusRuleEvaluationFailures + expr: sum(increase(prometheus_rule_evaluation_failures_total[5m])) > 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Prometheus rule evaluation is failing" + description: "Prometheus has reported rule evaluation failures for 5 minutes. Alert and recording rule expressions should be checked." + - alert: PrometheusAlertmanagerDiscoveryFailed + expr: (max(prometheus_notifications_alertmanagers_discovered) < 1) or absent(prometheus_notifications_alertmanagers_discovered) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Prometheus has not discovered Alertmanager" + description: "Prometheus has discovered zero Alertmanager targets for 2 minutes. Alert delivery may be broken even if alert rules are firing." - alert: JudgeWaitingQueueBacklog expr: codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"} > 5 for: 3m From a1c1e36cca29fb4abafa5eebe66d9da0d6b8b341 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 15:54:31 +0900 Subject: [PATCH 038/147] =?UTF-8?q?=EC=BF=A0=EB=B2=84=EB=84=A4=ED=8B=B0?= =?UTF-8?q?=EC=8A=A4=20=EC=9D=B4=EB=B2=A4=ED=8A=B8=20=EA=B4=80=EC=B8=A1?= =?UTF-8?q?=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 24 +- .../infra/observability-runbook/_index.md | 65 +++++- .../grafana-dashboard-kubernetes-events.yaml | 221 ++++++++++++++++++ .../monitoring/kubernetes-event-exporter.yaml | 140 +++++++++++ kubernetes/monitoring/kustomization.yaml | 2 + kubernetes/monitoring/logs/README.md | 2 + kubernetes/monitoring/prometheus-rules.yaml | 46 ++++ 7 files changed, 492 insertions(+), 8 deletions(-) create mode 100644 kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml create mode 100644 kubernetes/monitoring/kubernetes-event-exporter.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index b3e22334..32c4f1e3 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -175,8 +175,10 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. - `grafana-dashboard-ai-inference.yaml`: prod vLLM readiness, queue/cache/latency/token throughput, HF cache PVC dashboard. +- `grafana-dashboard-kubernetes-events.yaml`: image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable 상태와 Kubernetes Warning event log dashboard. - `grafana-dashboard-monitoring-stack.yaml`: Prometheus, Alertmanager, Grafana, Prometheus Operator readiness와 rule/notification 상태 dashboard. - `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. +- `kubernetes-event-exporter.yaml`: Kubernetes Warning event를 stdout JSON으로 export해서 Alloy/Loki가 수집하도록 하는 exporter, ServiceMonitor, 최소 event read RBAC. - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. @@ -231,6 +233,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PublicEndpointTLSExpiringSoon`: 공개 HTTPS 인증서 만료 14일 이내. - `FrontendRuntimeErrorsSpike`: browser runtime error report 10개 초과/5분, 5분 지속. - `BlackboxExporterUnavailable`: synthetic probe exporter Pod not ready 2분 지속. +- `KubernetesEventExporterUnavailable`: Kubernetes event exporter Pod not ready 2분 지속. - `GrafanaUnavailable`: Grafana Pod not ready 2분 지속. - `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. - `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. @@ -243,6 +246,10 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `CeleryTaskRuntimeHigh`: task p95 runtime 120초 초과 10분 지속. - `PodCrashLooping`: 주요 Pod restart 증가 5분 지속. - `CodePlacePodNotReady`: 주요 앱/DB/Redis Pod not ready 5분 지속. +- `KubernetesPodImagePullBackOff`: image pull 실패 2분 지속. +- `KubernetesPodCrashLoopBackOff`: CrashLoopBackOff 5분 지속. +- `KubernetesPodOOMKilled`: OOMKilled 종료가 최근 10분 내 발생하고 2분 지속. +- `KubernetesPodUnschedulable`: Pending 또는 Unschedulable 상태 10분 지속. - `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. - `CodePlaceContainerMemoryHigh`: 주요 컨테이너 memory limit 사용률 85% 초과 10분 지속. - `PostgresCollectorError`: CNPG PostgreSQL metrics collection error 5분 지속. @@ -282,18 +289,20 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - prod: `kubectl apply -k kubernetes/overlays/prod` 5. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` -6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. -8. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -9. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. -10. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. -11. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. +6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. +7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. +9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. +10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. +11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. 12. Grafana의 `CodePlace Monitoring Stack` dashboard에서 Prometheus, Alertmanager, Grafana, operator readiness, target scrape failure, rule evaluation failure, notification failure panel을 확인합니다. 13. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. 14. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. 15. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. 16. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -17. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. -18. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +17. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"}` 쿼리가 Kubernetes Warning event를 반환하는지 확인합니다. +18. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. +19. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. @@ -304,6 +313,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지하고 dev overlay에서 먼저 활성화합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. +- Kubernetes event는 Kubernetes Event Exporter가 Warning event만 stdout JSON으로 내보내고, Alloy가 일반 Pod log와 같은 경로로 Loki에 적재합니다. - PostgreSQL은 CNPG instance exporter와 PodMonitor로 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor로 `redis_*` 지표를 수집합니다. - Sentry backend SDK는 기본 PII 자동 전송을 비활성화하고 전송 직전 `authorization`, `cookie`, `password`, `token`, `secret`, 제출 source code 계열 필드를 redaction합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. - Frontend는 axios 요청마다 `X-Request-ID`를 전파하고 마지막 request ID를 `window.__CODEPLACE_LAST_REQUEST_ID__`와 Sentry request context 및 `/api/client_error` report에 저장합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 3a28568f..ac6c07de 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -17,13 +17,14 @@ weight: 6 bash kubernetes/monitoring/validate.sh kubectl get ns kubectl -n monitoring get prometheusrule,alertmanagerconfig,servicemonitor -kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy|otel|tempo|blackbox' +kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy|otel|tempo|blackbox|event-exporter' kubectl -n longhorn-system get pod,svc ``` Grafana에서는 `CodePlace Overview` dashboard에서 `namespace`를 알림의 namespace로 맞춥니다. 공개 URL 장애는 `CodePlace Public Endpoints` dashboard에서 먼저 봅니다. 로그 수집이나 로그 기반 확인이 필요하면 `CodePlace Logs` dashboard를 함께 봅니다. +Pod 배포/스케줄링/이미지 pull 실패는 `CodePlace Kubernetes Events` dashboard에서 Kubernetes Warning event와 kube-state-metrics 상태를 같이 봅니다. Prometheus, Alertmanager, Grafana 자체 이상은 `CodePlace Monitoring Stack` dashboard에서 봅니다. PVC/volume 문제가 의심되면 `CodePlace Storage` dashboard에서 Longhorn 상태를 먼저 확인합니다. @@ -36,6 +37,7 @@ Status -> Targets -> redis Status -> Targets -> longhorn Status -> Targets -> vllm Status -> Targets -> dcgm-exporter +Status -> Targets -> kubernetes-event-exporter Status -> Targets -> codeplace-public-http-dev Status -> Targets -> codeplace-public-http-prod ``` @@ -68,6 +70,8 @@ Loki 로그 확인: ```logql {namespace=""} +{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} +{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} |~ "FailedScheduling|ImagePullBackOff|ErrImagePull|BackOff|OOMKilling|Unhealthy|FailedMount|FailedAttachVolume" {namespace="kube-system", app_kubernetes_io_name="traefik"} | json {namespace="", app="frontend"} | json {namespace="", app="backend"} | json @@ -673,10 +677,69 @@ Grafana Explore: ```logql {namespace="", pod=""} +{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} |= "" ``` 이미지 pull, secret mount, readiness, resource limit, node scheduling을 확인합니다. +### KubernetesEventExporterUnavailable + +확인: + +```sh +kubectl -n monitoring get pod,svc,servicemonitor | grep event-exporter +kubectl -n monitoring describe pod -l app.kubernetes.io/name=kubernetes-event-exporter +kubectl -n monitoring logs deploy/kubernetes-event-exporter --tail=200 +``` + +PromQL: + +```promql +kube_pod_status_ready{namespace="monitoring", pod=~"kubernetes-event-exporter-.*", condition="true"} +up{job="kubernetes-event-exporter", namespace="monitoring"} +``` + +판단: + +- exporter가 down이면 `CodePlace Kubernetes Events` dashboard의 Warning event 로그가 비어 있을 수 있습니다. +- kube-state-metrics 기반 `ImagePullBackOff`, `CrashLoopBackOff`, `OOMKilled`, `Pending` 알림은 exporter와 별개로 계속 동작합니다. +- event exporter 로그에 throttling 또는 discarded event가 보이면 `maxEventAgeSeconds`, API server 부하, 이벤트 폭주 원인을 확인합니다. + +### KubernetesPodImagePullBackOff / KubernetesPodCrashLoopBackOff / KubernetesPodOOMKilled / KubernetesPodUnschedulable + +확인: + +```sh +kubectl -n get pod +kubectl -n describe pod +kubectl -n logs --previous --tail=200 +kubectl describe node +``` + +PromQL: + +```promql +kube_pod_container_status_waiting_reason{namespace="", reason=~"ImagePullBackOff|ErrImagePull|CrashLoopBackOff|CreateContainerConfigError|CreateContainerError"} +max_over_time(kube_pod_container_status_last_terminated_reason{namespace="", reason="OOMKilled"}[10m]) +kube_pod_status_unschedulable{namespace=""} +kube_pod_status_phase{namespace="", phase="Pending"} +``` + +Grafana Explore: + +```logql +{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} |= "" +{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} |~ "FailedScheduling|ImagePullBackOff|ErrImagePull|BackOff|OOMKilling|Unhealthy|FailedMount|FailedAttachVolume" +{namespace="", pod=""} +``` + +판단: + +- `ImagePullBackOff`/`ErrImagePull`은 image tag, registry 접근, image pull secret, node DNS/egress를 먼저 봅니다. +- `CrashLoopBackOff`는 `--previous` 로그와 직전 Warning event를 같이 봅니다. readiness 실패만으로 재시작되는지, 프로세스가 종료되는지 구분합니다. +- `OOMKilled`는 memory limit, 최근 트래픽, 캐시/큐 증가, JVM/Python heap 설정을 확인합니다. +- `Pending`/`Unschedulable`은 node capacity, taint/toleration, nodeSelector/affinity, PVC binding, Longhorn volume attach event를 확인합니다. + ### CodePlaceContainerCPUHigh / CodePlaceContainerMemoryHigh 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml b/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml new file mode 100644 index 00000000..8dec3196 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml @@ -0,0 +1,221 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-kubernetes-events + labels: + grafana_dashboard: "1" +data: + codeplace-kubernetes-events.json: | + { + "uid": "codeplace-kubernetes-events", + "title": "CodePlace Kubernetes Events", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "kubernetes", "events"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "namespace", + "type": "custom", + "query": "code-place-dev,code-place-prod,monitoring,longhorn-system,kube-system", + "multi": true, + "includeAll": true, + "allValue": "code-place-dev|code-place-prod|monitoring|longhorn-system|kube-system", + "current": { + "text": "code-place-dev", + "value": "code-place-dev" + } + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Event Exporter Ready", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kubernetes-event-exporter-.*\"}) or vector(0)", "legendFormat": "ready"} + ] + }, + { + "type": "stat", + "title": "Image Pull BackOff", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=~\"ImagePullBackOff|ErrImagePull\"}) or vector(0)", "legendFormat": "pods"} + ] + }, + { + "type": "stat", + "title": "CrashLoop BackOff", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=\"CrashLoopBackOff\"}) or vector(0)", "legendFormat": "pods"} + ] + }, + { + "type": "stat", + "title": "OOMKilled", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\", reason=\"OOMKilled\"}[10m])) or vector(0)", "legendFormat": "containers"} + ] + }, + { + "type": "stat", + "title": "Pending Pods", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)", "legendFormat": "pods"} + ] + }, + { + "type": "stat", + "title": "Unschedulable Pods", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_unschedulable{namespace=~\"$namespace\"}) or vector(0)", "legendFormat": "pods"} + ] + }, + { + "type": "timeseries", + "title": "Container Waiting Reasons", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (namespace, pod, container, reason) (kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=~\"CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError\"})", "legendFormat": "{{namespace}} {{pod}} {{container}} {{reason}}"} + ] + }, + { + "type": "timeseries", + "title": "Recent OOMKilled Containers", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "max by (namespace, pod, container) (max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\", reason=\"OOMKilled\"}[10m]))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + ] + }, + { + "type": "logs", + "title": "Kubernetes Warning Events", + "gridPos": {"x": 0, "y": 12, "w": 24, "h": 9}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=\"monitoring\", app_kubernetes_io_name=\"kubernetes-event-exporter\"}"} + ] + }, + { + "type": "logs", + "title": "Scheduling, Pull, Restart, and Probe Events", + "gridPos": {"x": 0, "y": 21, "w": 24, "h": 9}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=\"monitoring\", app_kubernetes_io_name=\"kubernetes-event-exporter\"} |~ \"FailedScheduling|ImagePullBackOff|ErrImagePull|BackOff|OOMKilling|Unhealthy|FailedMount|FailedAttachVolume\""} + ] + } + ] + } diff --git a/kubernetes/monitoring/kubernetes-event-exporter.yaml b/kubernetes/monitoring/kubernetes-event-exporter.yaml new file mode 100644 index 00000000..17addff2 --- /dev/null +++ b/kubernetes/monitoring/kubernetes-event-exporter.yaml @@ -0,0 +1,140 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-event-exporter +subjects: + - kind: ServiceAccount + name: kubernetes-event-exporter + namespace: monitoring +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubernetes-event-exporter-config + labels: + app.kubernetes.io/name: kubernetes-event-exporter +data: + config.yaml: | + logLevel: warn + logFormat: json + maxEventAgeSeconds: 60 + metricsNamePrefix: event_exporter_ + route: + routes: + - drop: + - type: "Normal" + match: + - namespace: "code-place-dev|code-place-prod|monitoring|longhorn-system|kube-system" + receiver: "stdout" + receivers: + - name: "stdout" + stdout: + deDot: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubernetes-event-exporter + template: + metadata: + labels: + app.kubernetes.io/name: kubernetes-event-exporter + spec: + serviceAccountName: kubernetes-event-exporter + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: event-exporter + image: ghcr.io/resmoio/kubernetes-event-exporter:v1.7 + imagePullPolicy: IfNotPresent + args: + - -conf=/data/config.yaml + ports: + - name: metrics + containerPort: 2112 + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: config + mountPath: /data + readOnly: true + volumes: + - name: config + configMap: + name: kubernetes-event-exporter-config +--- +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +spec: + selector: + app.kubernetes.io/name: kubernetes-event-exporter + ports: + - name: metrics + port: 2112 + targetPort: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kubernetes-event-exporter + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: kubernetes-event-exporter + jobLabel: app.kubernetes.io/name + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index 2f70c051..5712e168 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -11,10 +11,12 @@ resources: - ./grafana-dashboard-codeplace.yaml - ./grafana-dashboard-ai-inference.yaml - ./grafana-dashboard-logs.yaml + - ./grafana-dashboard-kubernetes-events.yaml - ./grafana-dashboard-monitoring-stack.yaml - ./grafana-dashboard-public-endpoints.yaml - ./grafana-dashboard-storage.yaml - ./grafana-dashboard-traces.yaml + - ./kubernetes-event-exporter.yaml - ./longhorn-service-monitor.yaml - ./otel-collector.yaml - ./prometheus-rules.yaml diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 9eec1bba..8d24b1c5 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -4,6 +4,7 @@ This directory contains Helm values for the Kubernetes logs stack. - Loki: monolithic mode, filesystem storage, Longhorn PVC. - Alloy: DaemonSet log collector for `code-place-dev`, `code-place-prod`, `monitoring`, and Traefik pods in `kube-system`. +- Kubernetes Event Exporter: Warning events are written to stdout as JSON and collected by Alloy from the `monitoring` namespace. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. - Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. @@ -67,6 +68,7 @@ Grafana should show a `Loki` datasource. Useful Explore queries: {namespace="code-place-prod", app="backend"} | json {namespace="code-place-dev", container=~"backend|celery-worker|judge-server"} {namespace="kube-system", app_kubernetes_io_name="traefik"} | json | DownstreamStatus >= 400 +{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} ``` Grafana should also show: diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 33af6098..016cdd2c 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -281,6 +281,16 @@ spec: annotations: summary: "Blackbox exporter is not ready" description: "The synthetic public endpoint probe exporter has not been ready for 2 minutes." + - alert: KubernetesEventExporterUnavailable + expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kubernetes-event-exporter-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kubernetes-event-exporter-.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Kubernetes event exporter is not ready" + description: "Kubernetes Warning events may not be available in Loki for deployment and scheduling failure triage." - alert: GrafanaUnavailable expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*"}) for: 2m @@ -392,6 +402,42 @@ spec: annotations: summary: "CodePlace pod is not ready" description: "A CodePlace application or datastore pod has not been ready for 5 minutes." + - alert: KubernetesPodImagePullBackOff + expr: max by (namespace, pod, container, reason) (kube_pod_container_status_waiting_reason{namespace=~"code-place-(dev|prod)|monitoring", reason=~"ImagePullBackOff|ErrImagePull"}) == 1 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "A Kubernetes pod cannot pull its image" + description: "A pod has been waiting with ImagePullBackOff or ErrImagePull for 2 minutes. Check image tag, registry access, and image pull secret." + - alert: KubernetesPodCrashLoopBackOff + expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{namespace=~"code-place-(dev|prod)|monitoring", reason="CrashLoopBackOff"}) == 1 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "A Kubernetes pod is in CrashLoopBackOff" + description: "A pod has been waiting with CrashLoopBackOff for 5 minutes. Check the previous container logs and Kubernetes Warning events." + - alert: KubernetesPodOOMKilled + expr: max by (namespace, pod, container) (max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~"code-place-(dev|prod)|monitoring", reason="OOMKilled"}[10m])) == 1 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "A Kubernetes container was OOMKilled" + description: "A container was terminated by OOMKilled within the last 10 minutes. Check memory limits, recent traffic, and heap/cache growth." + - alert: KubernetesPodUnschedulable + expr: (max by (namespace, pod) (kube_pod_status_unschedulable{namespace=~"code-place-(dev|prod)|monitoring"}) == 1) or (max by (namespace, pod) (kube_pod_status_phase{namespace=~"code-place-(dev|prod)|monitoring", phase="Pending"}) == 1) + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "A Kubernetes pod is pending or unschedulable" + description: "A pod has been pending or unschedulable for 10 minutes. Check node capacity, taints, affinity, PVC binding, and FailedScheduling events." - alert: CodePlaceContainerCPUHigh expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 0.001) > 0.8 for: 10m From b56560cf279241d5e232d11422ba801af9a8c158 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 15:57:23 +0900 Subject: [PATCH 039/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=20=ED=8C=8C=EC=9D=B4?= =?UTF-8?q?=ED=94=84=EB=9D=BC=EC=9D=B8=20=EC=8B=A4=ED=8C=A8=20=EC=95=8C?= =?UTF-8?q?=EB=A6=BC=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 3 ++ .../infra/observability-runbook/_index.md | 11 +++++-- .../grafana-dashboard-monitoring-stack.yaml | 26 ++++++++++++++-- .../monitoring/grafana-dashboard-traces.yaml | 12 ++++---- kubernetes/monitoring/prometheus-rules.yaml | 30 +++++++++++++++++++ 5 files changed, 72 insertions(+), 10 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 32c4f1e3..6c91f9c6 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -238,6 +238,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. - `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. - `PrometheusAlertmanagerDiscoveryFailed`: Prometheus가 Alertmanager target을 0개 discovery 2분 지속. +- `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. - `CeleryBeatDown`: beat Pod not ready 2분 지속. @@ -260,6 +261,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `AlloyDaemonSetUnavailable`: Alloy log collector가 모든 node에서 available하지 않음 5분 지속. - `LokiGatewayUnavailable`: Loki gateway Pod not ready 2분 지속. - `OTelCollectorUnavailable`: OpenTelemetry Collector Pod not ready 2분 지속. +- `OTelCollectorRefusedSpans`: Collector span 수신 거부 발생 2분 지속. +- `OTelCollectorExportFailures`: Collector span export 실패 발생 2분 지속. - `TempoUnavailable`: Tempo Pod not ready 2분 지속. - `TempoPVCAlmostFull`: Tempo PVC 사용률 85% 초과 10분. - `LonghornVolumeDegraded`: Longhorn volume degraded 5분 지속. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index ac6c07de..fc8a2328 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -63,6 +63,7 @@ kubectl -n monitoring get alertmanagerconfig codeplace-alert-routing -o yaml kube_pod_status_ready{namespace="monitoring", pod=~"prometheus-kube-prometheus-stack-prometheus-.*|alertmanager-kube-prometheus-stack-alertmanager-.*|kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} prometheus_notifications_alertmanagers_discovered increase(prometheus_rule_evaluation_failures_total[5m]) +increase(alertmanager_notifications_failed_total[5m]) sum by (job, namespace) (up == 0) ``` @@ -500,7 +501,7 @@ DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"} ## P1 -### OTelCollectorUnavailable / TempoUnavailable / TempoPVCAlmostFull +### OTelCollectorUnavailable / OTelCollectorRefusedSpans / OTelCollectorExportFailures / TempoUnavailable / TempoPVCAlmostFull 확인: @@ -520,12 +521,15 @@ kube_pod_status_ready{namespace="monitoring", pod=~"otel-collector-.*|tempo-.*", max(kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) / max(kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) sum(rate(otelcol_receiver_accepted_spans_total{namespace="monitoring"}[5m])) +sum(rate(otelcol_receiver_refused_spans_total{namespace="monitoring"}[5m])) sum(rate(otelcol_exporter_send_failed_spans_total{namespace="monitoring"}[5m])) ``` 판단: - Collector가 down이면 dev backend/celery trace export가 실패합니다. 앱 요청 처리는 계속되지만 trace 관측이 끊깁니다. +- refused span이 증가하면 Collector receiver limit, malformed OTLP payload, backend/celery exporter 설정, collector 로그를 먼저 봅니다. +- export failure가 증가하면 Tempo readiness, `tempo.monitoring.svc.cluster.local:4317` 경로, Collector exporter queue/retry 로그를 봅니다. - Tempo가 down이면 Collector가 span export 실패를 기록합니다. - Tempo PVC가 85%를 넘으면 retention 단축, noisy trace sampling 조정, PVC 증설 중 하나를 먼저 결정합니다. - prod는 기본 `OTEL_ENABLED=0`입니다. prod trace가 없다고 해서 장애는 아니며, dev에서 Collector/Tempo를 먼저 확인한 뒤 prod 승격 여부를 결정합니다. @@ -559,7 +563,7 @@ kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condi - latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. - `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. -### GrafanaUnavailable / PrometheusOperatorUnavailable / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed +### GrafanaUnavailable / PrometheusOperatorUnavailable / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures 확인: @@ -568,6 +572,7 @@ kubectl -n monitoring get pod | grep -E 'grafana|operator|prometheus|alertmanage kubectl -n monitoring describe pod -l app.kubernetes.io/name=grafana kubectl -n monitoring describe pod -l app.kubernetes.io/name=prometheus-operator kubectl -n monitoring logs deploy/kube-prometheus-stack-operator --tail=200 +kubectl -n monitoring logs alertmanager-kube-prometheus-stack-alertmanager-0 -c alertmanager --tail=200 kubectl -n monitoring get prometheusrule,servicemonitor,podmonitor,alertmanagerconfig ``` @@ -577,6 +582,7 @@ PromQL: kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) prometheus_notifications_alertmanagers_discovered +sum by (integration) (increase(alertmanager_notifications_failed_total[5m])) sum by (job, namespace) (up == 0) ``` @@ -586,6 +592,7 @@ sum by (job, namespace) (up == 0) - Operator가 down이면 새 ServiceMonitor/PodMonitor/PrometheusRule/AlertmanagerConfig 변경이 Prometheus/Alertmanager에 반영되지 않을 수 있습니다. - rule evaluation failure가 있으면 최근 추가한 PromQL expression과 label cardinality, metric 존재 여부를 먼저 봅니다. - Alertmanager discovery가 0이면 AlertmanagerConfig가 맞아도 Prometheus가 알림을 보낼 대상이 없습니다. Prometheus generated config와 Alertmanager Service/endpoints를 확인합니다. +- notification failure가 있으면 webhook Secret 값, Alertmanager generated config, Discord endpoint 응답, cluster outbound network, rate limit을 확인합니다. 로컬 `curl` 성공만으로 Alertmanager Pod의 egress 성공을 보장하지 않습니다. ### FrontendRuntimeErrorsSpike diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index f04adbb1..af4c53db 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -131,8 +131,8 @@ data: }, { "type": "stat", - "title": "Rule Eval Failures", - "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "title": "Rule Fail", + "gridPos": {"x": 20, "y": 0, "w": 2, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -151,6 +151,28 @@ data: {"expr": "sum(increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "failures"} ] }, + { + "type": "stat", + "title": "Notify Fail", + "gridPos": {"x": 22, "y": 0, "w": 2, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(increase(alertmanager_notifications_failed_total[5m])) or vector(0)", "legendFormat": "failures"} + ] + }, { "type": "timeseries", "title": "Monitoring Pod Restarts", diff --git a/kubernetes/monitoring/grafana-dashboard-traces.yaml b/kubernetes/monitoring/grafana-dashboard-traces.yaml index 20363275..40e59040 100644 --- a/kubernetes/monitoring/grafana-dashboard-traces.yaml +++ b/kubernetes/monitoring/grafana-dashboard-traces.yaml @@ -38,7 +38,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"tempo-.*\"})", "legendFormat": "tempo"} + {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"tempo-.*\"}) or vector(0)", "legendFormat": "tempo"} ] }, { @@ -60,7 +60,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"otel-collector-.*\"})", "legendFormat": "otel-collector"} + {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"otel-collector-.*\"}) or vector(0)", "legendFormat": "otel-collector"} ] }, { @@ -71,7 +71,7 @@ data: "fieldConfig": {"defaults": {"unit": "ops"}}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "spans/s"} + {"expr": "sum(rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "spans/s"} ] }, { @@ -94,7 +94,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(increase(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "failed"} + {"expr": "sum(increase(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "failed"} ] }, { @@ -138,8 +138,8 @@ data: "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "ops"}}, "targets": [ - {"expr": "sum(rate(otelcol_receiver_refused_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "refused"}, - {"expr": "sum(rate(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "failed"} + {"expr": "sum(rate(otelcol_receiver_refused_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "refused"}, + {"expr": "sum(rate(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "failed"} ] } ] diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 016cdd2c..84b5e3ac 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -331,6 +331,16 @@ spec: annotations: summary: "Prometheus has not discovered Alertmanager" description: "Prometheus has discovered zero Alertmanager targets for 2 minutes. Alert delivery may be broken even if alert rules are firing." + - alert: AlertmanagerNotificationFailures + expr: sum(increase(alertmanager_notifications_failed_total[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Alertmanager notification delivery is failing" + description: "Alertmanager has failed to send one or more notifications in the last 5 minutes. Check Discord webhook secret, receiver config, outbound network, and Alertmanager logs." - alert: JudgeWaitingQueueBacklog expr: codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"} > 5 for: 3m @@ -532,6 +542,26 @@ spec: annotations: summary: "OpenTelemetry Collector is not ready" description: "The OpenTelemetry Collector has not been ready for 2 minutes. Enabled services cannot export traces." + - alert: OTelCollectorRefusedSpans + expr: sum(increase(otelcol_receiver_refused_spans_total{namespace="monitoring"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "OpenTelemetry Collector is refusing spans" + description: "The OpenTelemetry Collector has refused incoming spans in the last 5 minutes. Check collector limits, receiver errors, and backend/celery OTLP client logs." + - alert: OTelCollectorExportFailures + expr: sum(increase(otelcol_exporter_send_failed_spans_total{namespace="monitoring"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "OpenTelemetry Collector cannot export spans" + description: "The OpenTelemetry Collector has failed to export spans to Tempo in the last 5 minutes. Check Tempo readiness, collector exporter logs, and network path." - alert: TempoUnavailable expr: (max by (namespace, pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) for: 2m From e4eff0cd4bbcb2e1fe749c8294a456cea0d5ad40 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:02:55 +0900 Subject: [PATCH 040/147] =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EC=8A=A4?= =?UTF-8?q?=ED=86=A0=EC=96=B4=20=EB=B3=91=EB=AA=A9=20=EA=B4=80=EC=B8=A1?= =?UTF-8?q?=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_metrics.py | 115 ++++++++++++++++++ backend/utils/tests.py | 46 ++++++- .../observability-improvement-plan/_index.md | 14 ++- .../infra/observability-runbook/_index.md | 20 ++- .../grafana-dashboard-codeplace.yaml | 33 ++++- kubernetes/monitoring/prometheus-rules.yaml | 45 +++++++ 6 files changed, 269 insertions(+), 4 deletions(-) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 6294e0c0..1ce9dcc5 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -1,6 +1,7 @@ import logging from datetime import timedelta +from django.db import connection from django.db.models import Count from django.utils import timezone from prometheus_client import Counter, Histogram, REGISTRY @@ -99,6 +100,34 @@ def describe(self): "Latest observed Celery task runtime grouped by task name and status.", labels=["task_name", "status"], ) + yield GaugeMetricFamily( + "codeplace_postgres_connections", + "Current PostgreSQL connections for the active database.", + ) + yield GaugeMetricFamily( + "codeplace_postgres_max_connections", + "Configured PostgreSQL max_connections.", + ) + yield GaugeMetricFamily( + "codeplace_postgres_long_transactions", + "Current PostgreSQL transactions older than five minutes.", + ) + yield GaugeMetricFamily( + "codeplace_postgres_lock_waits", + "Current PostgreSQL locks waiting to be granted.", + ) + yield GaugeMetricFamily( + "codeplace_redis_connected_clients", + "Current Redis connected clients from INFO clients.", + ) + yield GaugeMetricFamily( + "codeplace_redis_max_clients", + "Configured Redis maxclients from CONFIG GET maxclients.", + ) + yield GaugeMetricFamily( + "codeplace_redis_rejected_connections_total", + "Total Redis rejected connections from INFO stats.", + ) def collect(self): yield from self._submission_status_count() @@ -106,6 +135,8 @@ def collect(self): yield from self._judge_server_metrics() yield self._judge_duration_histogram() yield from self._celery_task_metrics() + yield from self._postgres_metrics() + yield from self._redis_health_metrics() def _submission_status_count(self): metric = GaugeMetricFamily( @@ -253,6 +284,90 @@ def _celery_task_metrics(self): yield runtime_metric yield last_runtime_metric + def _postgres_metrics(self): + connections_metric = GaugeMetricFamily( + "codeplace_postgres_connections", + "Current PostgreSQL connections for the active database.", + ) + max_connections_metric = GaugeMetricFamily( + "codeplace_postgres_max_connections", + "Configured PostgreSQL max_connections.", + ) + long_transactions_metric = GaugeMetricFamily( + "codeplace_postgres_long_transactions", + "Current PostgreSQL transactions older than five minutes.", + ) + lock_waits_metric = GaugeMetricFamily( + "codeplace_postgres_lock_waits", + "Current PostgreSQL locks waiting to be granted.", + ) + try: + with connection.cursor() as cursor: + cursor.execute("SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()") + connections = cursor.fetchone()[0] + cursor.execute("SELECT setting::float FROM pg_settings WHERE name = 'max_connections'") + max_connections = cursor.fetchone()[0] + cursor.execute( + """ + SELECT count(*) + FROM pg_stat_activity + WHERE datname = current_database() + AND xact_start IS NOT NULL + AND now() - xact_start > interval '5 minutes' + AND state <> 'idle' + """ + ) + long_transactions = cursor.fetchone()[0] + cursor.execute("SELECT count(*) FROM pg_locks WHERE NOT granted") + lock_waits = cursor.fetchone()[0] + except Exception as e: + logger.warning("Failed to collect PostgreSQL health metrics: %s", e) + connections = max_connections = long_transactions = lock_waits = 0 + + connections_metric.add_metric([], float(connections or 0)) + max_connections_metric.add_metric([], float(max_connections or 0)) + long_transactions_metric.add_metric([], float(long_transactions or 0)) + lock_waits_metric.add_metric([], float(lock_waits or 0)) + yield connections_metric + yield max_connections_metric + yield long_transactions_metric + yield lock_waits_metric + + def _redis_health_metrics(self): + connected_clients_metric = GaugeMetricFamily( + "codeplace_redis_connected_clients", + "Current Redis connected clients from INFO clients.", + ) + max_clients_metric = GaugeMetricFamily( + "codeplace_redis_max_clients", + "Configured Redis maxclients from CONFIG GET maxclients.", + ) + rejected_connections_metric = GaugeMetricFamily( + "codeplace_redis_rejected_connections_total", + "Total Redis rejected connections from INFO stats.", + ) + try: + client = cache.client.get_client(write=False) + info = client.info() + connected_clients = float(info.get("connected_clients") or 0) + rejected_connections = float(info.get("rejected_connections") or 0) + max_clients = 0 + try: + raw_max_clients = client.config_get("maxclients") + max_clients = float(raw_max_clients.get("maxclients") or 0) + except Exception as e: + logger.warning("Failed to collect Redis maxclients metric: %s", e) + except Exception as e: + logger.warning("Failed to collect Redis health metrics: %s", e) + connected_clients = max_clients = rejected_connections = 0 + + connected_clients_metric.add_metric([], connected_clients) + max_clients_metric.add_metric([], max_clients) + rejected_connections_metric.add_metric([], rejected_connections) + yield connected_clients_metric + yield max_clients_metric + yield rejected_connections_metric + @staticmethod def _redis_hash(key): values = cache.hgetall(key) or {} diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 70e6b589..9a6ca4e1 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -102,6 +102,48 @@ def test_celery_task_metrics_are_collected_from_redis(self): last_runtime = samples["codeplace_celery_task_last_runtime_seconds"][0] self.assertEqual(last_runtime.value, 1.5) + def test_postgres_health_metrics_are_collected(self): + class FakeCursor: + def __init__(self): + self.values = iter([(12,), (100,), (2,), (1,)]) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, _query): + return None + + def fetchone(self): + return next(self.values) + + fake_connection = MagicMock() + fake_connection.cursor.return_value = FakeCursor() + with patch("utils.observability_metrics.connection", fake_connection): + samples = self._samples_by_metric(list(CodePlaceCollector()._postgres_metrics())) + + self.assertEqual(samples["codeplace_postgres_connections"][0].value, 12) + self.assertEqual(samples["codeplace_postgres_max_connections"][0].value, 100) + self.assertEqual(samples["codeplace_postgres_long_transactions"][0].value, 2) + self.assertEqual(samples["codeplace_postgres_lock_waits"][0].value, 1) + + def test_redis_health_metrics_are_collected(self): + redis_client = MagicMock() + redis_client.info.return_value = { + "connected_clients": 25, + "rejected_connections": 3, + } + redis_client.config_get.return_value = {"maxclients": "1000"} + + with patch("utils.observability_metrics.cache.client.get_client", return_value=redis_client): + samples = self._samples_by_metric(list(CodePlaceCollector()._redis_health_metrics())) + + self.assertEqual(samples["codeplace_redis_connected_clients"][0].value, 25) + self.assertEqual(samples["codeplace_redis_max_clients"][0].value, 1000) + self.assertEqual(samples["codeplace_redis_rejected_connections_total"][0].value, 3) + class CeleryObservabilityTest(SimpleTestCase): @@ -190,7 +232,9 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ patch.object(CodePlaceCollector, "_judge_server_metrics", return_value=[judge_metric]), \ patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric), \ - patch.object(CodePlaceCollector, "_celery_task_metrics", return_value=[]): + patch.object(CodePlaceCollector, "_celery_task_metrics", return_value=[]), \ + patch.object(CodePlaceCollector, "_postgres_metrics", return_value=[]), \ + patch.object(CodePlaceCollector, "_redis_health_metrics", return_value=[]): response = self.client.get("/metrics") self.assertEqual(response.status_code, 200) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 6c91f9c6..48781ba2 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -47,6 +47,13 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_celery_task_count{task_name,status}` - `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_celery_task_last_runtime_seconds{task_name,status}` +- `codeplace_postgres_connections` +- `codeplace_postgres_max_connections` +- `codeplace_postgres_long_transactions` +- `codeplace_postgres_lock_waits` +- `codeplace_redis_connected_clients` +- `codeplace_redis_max_clients` +- `codeplace_redis_rejected_connections_total` - `codeplace_frontend_error_total{surface,error_type}` prod AI 힌트용 vLLM은 OpenAI-compatible server의 `/metrics`를 ServiceMonitor로 scrape합니다. vLLM production metrics 기준으로 다음 항목을 봅니다. @@ -256,6 +263,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PostgresCollectorError`: CNPG PostgreSQL metrics collection error 5분 지속. - `PostgresHADegraded`: PostgreSQL instance가 3개 node에 분산되지 않음 10분 지속. - `RedisMemoryHigh`: Redis memory 사용률 85% 초과 10분 지속. +- `PostgresConnectionUsageHigh`: backend 기준 PostgreSQL connection/max_connections 80% 초과 10분 지속. +- `PostgresLongTransactions`: 5분 초과 transaction이 10분 지속. +- `PostgresLockWaits`: lock wait가 5분 지속. +- `RedisConnectedClientsHigh`: backend 기준 Redis connected_clients/maxclients 80% 초과 10분 지속. +- `RedisRejectedConnections`: Redis rejected connection 증가 2분 지속. - `PVCAlmostFull`: PVC 사용률 85% 초과 10분 지속. - `LokiPVCAlmostFull`: Loki Longhorn PVC 사용률 85% 초과 10분 지속. - `AlloyDaemonSetUnavailable`: Alloy log collector가 모든 node에서 available하지 않음 5분 지속. @@ -294,7 +306,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness panel을 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index fc8a2328..66026bdc 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -758,7 +758,7 @@ topk(10, container_memory_working_set_bytes{namespace="", container!= CPU/memory가 지속적으로 높으면 replica 조정, limit 조정, 최근 배포 변경을 확인합니다. -### PostgresCollectorError / PostgresHADegraded / RedisMemoryHigh +### PostgresCollectorError / PostgresHADegraded / RedisMemoryHigh / PostgresConnectionUsageHigh / PostgresLongTransactions / PostgresLockWaits / RedisConnectedClientsHigh / RedisRejectedConnections 확인: @@ -767,13 +767,31 @@ cnpg_collector_last_collection_error{namespace="", cluster="postgres" cnpg_collector_nodes_used{namespace="", cluster="postgres"} redis_memory_used_bytes{namespace=""} redis_memory_max_bytes{namespace=""} +codeplace_postgres_connections{namespace=""} +codeplace_postgres_max_connections{namespace=""} +codeplace_postgres_long_transactions{namespace=""} +codeplace_postgres_lock_waits{namespace=""} +codeplace_redis_connected_clients{namespace=""} +codeplace_redis_max_clients{namespace=""} +increase(codeplace_redis_rejected_connections_total{namespace=""}[5m]) +``` + +```sh +kubectl -n logs deploy/backend --tail=200 +kubectl -n get pod -l cnpg.io/cluster=postgres -o wide +kubectl -n get pod -l app=redis-replication -o wide ``` 판단: - `PostgresCollectorError`는 metrics query 실패이므로 CNPG exporter 로그와 PostgreSQL 권한/상태를 확인합니다. - `PostgresHADegraded`는 3개 instance가 서로 다른 node에 분산되지 않은 상태입니다. node 수, taint, Longhorn volume attach 상태를 확인합니다. +- `PostgresConnectionUsageHigh`는 backend connection leak, worker 동시성 증가, DB max_connections 부족, slow query 누적으로 구분합니다. API latency와 동시에 오르면 DB 병목 후보가 강합니다. +- `PostgresLongTransactions`는 transaction 범위가 긴 view/task, idle-in-transaction, migration/admin 작업을 확인합니다. +- `PostgresLockWaits`는 쓰기 contention이나 DDL/migration lock 가능성이 큽니다. 같은 시간대 backend error와 Celery runtime 증가를 같이 봅니다. - `RedisMemoryHigh`는 Redis `maxmemory`가 0보다 클 때만 동작합니다. `maxmemory=0`이면 Redis 자체 제한이 없으므로 container memory alert를 기준으로 보고, 제한을 둘지 별도로 결정합니다. +- `RedisConnectedClientsHigh`는 connection pool 설정, worker replica/concurrency, client leak를 확인합니다. +- `RedisRejectedConnections`는 Redis `maxclients`에 실제로 걸린 상태입니다. API/cache와 Celery queue가 동시에 영향을 받을 수 있습니다. - Redis memory가 높으면 eviction 정책, queue/backlog, cache key 증가를 확인합니다. ### PVCAlmostFull diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 120020c9..253fa46c 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -471,10 +471,41 @@ data: {"expr": "max by (namespace, pod) (redis_memory_max_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} maxmemory"} ] }, + { + "type": "timeseries", + "title": "PostgreSQL Connections", + "gridPos": {"x": 0, "y": 51, "w": 8, "h": 6}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "codeplace_postgres_connections{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} connections"}, + {"expr": "codeplace_postgres_max_connections{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} max"} + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL Long Tx and Lock Waits", + "gridPos": {"x": 8, "y": 51, "w": 8, "h": 6}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "codeplace_postgres_long_transactions{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} long tx"}, + {"expr": "codeplace_postgres_lock_waits{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} lock waits"} + ] + }, + { + "type": "timeseries", + "title": "Redis Clients and Rejections", + "gridPos": {"x": 16, "y": 51, "w": 8, "h": 6}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "codeplace_redis_connected_clients{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} clients"}, + {"expr": "codeplace_redis_max_clients{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} maxclients"}, + {"expr": "increase(codeplace_redis_rejected_connections_total{namespace=~\"$namespace\"}[5m])", "legendFormat": "{{namespace}} rejected/5m"} + ] + }, { "type": "timeseries", "title": "Frontend Runtime Errors", - "gridPos": {"x": 0, "y": 51, "w": 12, "h": 6}, + "gridPos": {"x": 0, "y": 57, "w": 12, "h": 6}, "fieldConfig": { "defaults": { "unit": "ops", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 84b5e3ac..d865f704 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -493,6 +493,51 @@ spec: annotations: summary: "Redis memory usage is high" description: "Redis exporter reports memory usage above 85% of configured maxmemory for 10 minutes." + - alert: PostgresConnectionUsageHigh + expr: codeplace_postgres_connections{namespace=~"code-place-(dev|prod)"} / clamp_min(codeplace_postgres_max_connections{namespace=~"code-place-(dev|prod)"}, 1) > 0.8 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL connection usage is high" + description: "Backend reports PostgreSQL connection usage above 80% of max_connections for 10 minutes." + - alert: PostgresLongTransactions + expr: codeplace_postgres_long_transactions{namespace=~"code-place-(dev|prod)"} > 0 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL long transactions detected" + description: "Backend reports at least one PostgreSQL transaction older than 5 minutes for 10 minutes." + - alert: PostgresLockWaits + expr: codeplace_postgres_lock_waits{namespace=~"code-place-(dev|prod)"} > 0 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL lock waits detected" + description: "Backend reports PostgreSQL locks waiting to be granted for 5 minutes." + - alert: RedisConnectedClientsHigh + expr: (codeplace_redis_connected_clients{namespace=~"code-place-(dev|prod)"} / clamp_min(codeplace_redis_max_clients{namespace=~"code-place-(dev|prod)"}, 1) > 0.8) and codeplace_redis_max_clients{namespace=~"code-place-(dev|prod)"} > 0 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "Redis connected clients usage is high" + description: "Backend reports Redis connected clients above 80% of maxclients for 10 minutes." + - alert: RedisRejectedConnections + expr: increase(codeplace_redis_rejected_connections_total{namespace=~"code-place-(dev|prod)"}[5m]) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "Redis rejected connections detected" + description: "Backend reports Redis rejected connections in the last 5 minutes. Check maxclients, connection pooling, and Redis resource pressure." - alert: PVCAlmostFull expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 for: 10m From bb8a8d2356ebbd1fcbfc253958d3609fc633775a Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:07:15 +0900 Subject: [PATCH 041/147] =?UTF-8?q?=EC=B1=84=EC=A0=90=20=EC=A4=91=EB=8B=A8?= =?UTF-8?q?=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_metrics.py | 36 ++++++++++++++++++- backend/utils/tests.py | 26 ++++++++++++++ .../observability-improvement-plan/_index.md | 5 ++- .../infra/observability-runbook/_index.md | 17 +++++++-- .../grafana-dashboard-codeplace.yaml | 24 ++++++++++++- kubernetes/monitoring/prometheus-rules.yaml | 18 ++++++++++ 6 files changed, 121 insertions(+), 5 deletions(-) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 1ce9dcc5..e6599995 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -2,7 +2,7 @@ from datetime import timedelta from django.db import connection -from django.db.models import Count +from django.db.models import Count, Min from django.utils import timezone from prometheus_client import Counter, Histogram, REGISTRY from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily @@ -62,6 +62,11 @@ def describe(self): "Current number of submissions grouped by judge status.", labels=["status"], ) + yield GaugeMetricFamily( + "codeplace_submission_oldest_age_seconds", + "Age of the oldest submission in each in-flight judge status.", + labels=["status"], + ) yield GaugeMetricFamily( "codeplace_waiting_queue_length", "Number of submissions waiting because no judge-server was available.", @@ -131,6 +136,7 @@ def describe(self): def collect(self): yield from self._submission_status_count() + yield self._submission_oldest_age_seconds() yield self._waiting_queue_length() yield from self._judge_server_metrics() yield self._judge_duration_histogram() @@ -165,6 +171,34 @@ def _waiting_queue_length(self): metric.add_metric([], 0) return metric + def _submission_oldest_age_seconds(self): + metric = GaugeMetricFamily( + "codeplace_submission_oldest_age_seconds", + "Age of the oldest submission in each in-flight judge status.", + labels=["status"], + ) + now = timezone.now() + status_map = { + JudgeStatus.PENDING: "pending", + JudgeStatus.JUDGING: "judging", + } + try: + oldest_by_status = { + row["result"]: row["oldest_create_time"] + for row in Submission.objects.filter( + result__in=status_map.keys(), + ).values("result").annotate(oldest_create_time=Min("create_time")) + } + except Exception as e: + logger.warning("Failed to collect oldest submission age metrics: %s", e) + oldest_by_status = {} + + for status_value, status_label in status_map.items(): + created_at = oldest_by_status.get(status_value) + age_seconds = max((now - created_at).total_seconds(), 0) if created_at else 0 + metric.add_metric([status_label], age_seconds) + return metric + def _judge_server_metrics(self): from conf.models import JudgeServer diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 9a6ca4e1..36034be5 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -64,6 +64,25 @@ def test_judge_server_metrics_mark_stale_enabled_server_unavailable(self): self.assertEqual(available.value, 0) self.assertGreaterEqual(heartbeat_age.value, 599) + def test_submission_oldest_age_metrics_are_collected_for_inflight_statuses(self): + now = timezone.now() + rows = [ + {"result": 6, "oldest_create_time": now - timedelta(minutes=7)}, + {"result": 7, "oldest_create_time": now - timedelta(minutes=3)}, + ] + queryset = MagicMock() + queryset.values.return_value.annotate.return_value = rows + + with patch("utils.observability_metrics.Submission.objects.filter", return_value=queryset): + samples = self._samples_by_metric([CodePlaceCollector()._submission_oldest_age_seconds()]) + + by_status = { + sample.labels["status"]: sample.value + for sample in samples["codeplace_submission_oldest_age_seconds"] + } + self.assertGreaterEqual(by_status["pending"], 419) + self.assertGreaterEqual(by_status["judging"], 179) + def test_celery_task_metrics_are_collected_from_redis(self): hashes = { celery_observability.CELERY_TASK_TOTAL_KEY: { @@ -216,6 +235,12 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): "Number of submissions waiting because no judge-server was available.", ) waiting_queue_metric.add_metric([], 0) + oldest_age_metric = GaugeMetricFamily( + "codeplace_submission_oldest_age_seconds", + "Age of the oldest submission in each in-flight judge status.", + labels=["status"], + ) + oldest_age_metric.add_metric(["pending"], 0) judge_metric = GaugeMetricFamily( "codeplace_judge_server_available", "Whether each judge-server is enabled and has a fresh heartbeat.", @@ -229,6 +254,7 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): judge_duration_metric.add_metric([], [("+Inf", 0)], 0) with patch.object(CodePlaceCollector, "_submission_status_count", return_value=[submission_metric]), \ + patch.object(CodePlaceCollector, "_submission_oldest_age_seconds", return_value=oldest_age_metric), \ patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ patch.object(CodePlaceCollector, "_judge_server_metrics", return_value=[judge_metric]), \ patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric), \ diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 48781ba2..b1e18839 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -39,6 +39,7 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_http_requests_total{method,endpoint,status_code}` - `codeplace_http_request_duration_seconds{method,endpoint}` - `codeplace_submission_status_count{status}` +- `codeplace_submission_oldest_age_seconds{status}` - `codeplace_submission_judge_duration_seconds` - `codeplace_waiting_queue_length` - `codeplace_judge_server_available{hostname}` @@ -247,6 +248,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PrometheusAlertmanagerDiscoveryFailed`: Prometheus가 Alertmanager target을 0개 discovery 2분 지속. - `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. +- `PendingSubmissionsStuck`: oldest pending submission age 300초 초과 3분 지속. +- `JudgingSubmissionsStuck`: oldest judging submission age 900초 초과 5분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. - `CeleryBeatDown`: beat Pod not ready 2분 지속. - `CeleryTaskFailures`: task failure 발생 2분 지속. @@ -306,7 +309,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, oldest in-flight submission age, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 66026bdc..7ca58b93 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -617,16 +617,29 @@ Grafana Explore: - `error_type="unhandled_rejection"`이 많으면 API response handling, auth/session 만료 처리, network rejection을 봅니다. - Sentry가 켜진 환경이면 같은 `release`, `environment`, `request_id`로 Sentry event와 Loki `frontend.error` 로그를 대조합니다. -### JudgeWaitingQueueBacklog +### JudgeWaitingQueueBacklog / PendingSubmissionsStuck / JudgingSubmissionsStuck 확인: ```promql codeplace_waiting_queue_length{namespace=""} +codeplace_submission_oldest_age_seconds{namespace=""} +codeplace_submission_status_count{namespace="", status=~"pending|judging"} sum by (namespace) (codeplace_judge_server_available{namespace=""}) ``` -queue 증가와 judge available 감소가 같이 보이면 judge-server 문제입니다. judge는 정상인데 queue만 증가하면 celery-worker 처리량을 봅니다. +```sh +kubectl -n logs deploy/backend --tail=200 +kubectl -n logs deploy/celery-worker --tail=200 +kubectl -n get pod -l app=judge-server +``` + +판단: + +- queue 증가와 judge available 감소가 같이 보이면 judge-server 문제입니다. +- pending age만 증가하면 submission 생성 이후 Celery enqueue/worker dispatch/Redis queue 경로를 봅니다. +- judging age가 증가하면 judge-server 실행, judge-server `/judge` 호출, result save path, worker 로그의 `judge.tasks.judge_task`를 봅니다. +- pending/judging count는 낮지만 oldest age만 높으면 특정 제출이 고립된 상태입니다. 사용자 영향은 제한적일 수 있으나 결과 저장/상태 전환 실패를 확인합니다. ### CeleryWorkerRestarting / CeleryBeatDown diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 253fa46c..6f3c51ca 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -177,13 +177,35 @@ data: { "type": "bargauge", "title": "Submission Status", - "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "gridPos": {"x": 16, "y": 4, "w": 4, "h": 8}, "fieldConfig": {"defaults": {"unit": "short"}}, "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ {"expr": "codeplace_submission_status_count{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{status}}"} ] }, + { + "type": "bargauge", + "title": "Oldest In-flight Submission", + "gridPos": {"x": 20, "y": 4, "w": 4, "h": 8}, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 300}, + {"color": "red", "value": 900} + ] + } + } + }, + "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace_submission_oldest_age_seconds{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{status}}"} + ] + }, { "type": "row", "title": "Async Jobs", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index d865f704..d0e72014 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -350,6 +350,24 @@ spec: annotations: summary: "Judge waiting queue is backing up" description: "More than 5 submissions are waiting because no judge-server was available." + - alert: PendingSubmissionsStuck + expr: codeplace_submission_oldest_age_seconds{namespace=~"code-place-(dev|prod)", status="pending"} > 300 + for: 3m + labels: + severity: warning + priority: P1 + annotations: + summary: "Pending submissions are stuck" + description: "The oldest pending submission has been pending for more than 5 minutes. Check Celery workers, Redis queue state, judge dispatch, and backend logs." + - alert: JudgingSubmissionsStuck + expr: codeplace_submission_oldest_age_seconds{namespace=~"code-place-(dev|prod)", status="judging"} > 900 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Judging submissions are stuck" + description: "The oldest judging submission has been judging for more than 15 minutes. Check judge-server execution, callback/result save path, and worker logs." - alert: CeleryWorkerRestarting expr: sum by (namespace) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"celery-worker-.*"}[15m])) > 3 labels: From cbdb1aa3cc0f7d53d1f86dd6f92e50075030472d Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:11:44 +0900 Subject: [PATCH 042/147] =?UTF-8?q?=EB=B0=B0=ED=8F=AC=20=EB=A1=A4=EC=95=84?= =?UTF-8?q?=EC=9B=83=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 4 +- .../infra/observability-runbook/_index.md | 26 ++++++++++++ .../grafana-dashboard-codeplace.yaml | 40 ++++++++++++------- kubernetes/monitoring/prometheus-rules.yaml | 18 +++++++++ 4 files changed, 72 insertions(+), 16 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index b1e18839..e3e962a0 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -261,6 +261,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `KubernetesPodCrashLoopBackOff`: CrashLoopBackOff 5분 지속. - `KubernetesPodOOMKilled`: OOMKilled 종료가 최근 10분 내 발생하고 2분 지속. - `KubernetesPodUnschedulable`: Pending 또는 Unschedulable 상태 10분 지속. +- `CodePlaceDeploymentUnavailable`: 주요 Deployment unavailable replica 5분 지속. +- `CodePlaceDeploymentRolloutStuck`: Deployment observed generation 미반영 10분 지속. - `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. - `CodePlaceContainerMemoryHigh`: 주요 컨테이너 memory limit 사용률 85% 초과 10분 지속. - `PostgresCollectorError`: CNPG PostgreSQL metrics collection error 5분 지속. @@ -309,7 +311,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, oldest in-flight submission age, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, oldest in-flight submission age, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 7ca58b93..988a1e21 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -760,6 +760,32 @@ Grafana Explore: - `OOMKilled`는 memory limit, 최근 트래픽, 캐시/큐 증가, JVM/Python heap 설정을 확인합니다. - `Pending`/`Unschedulable`은 node capacity, taint/toleration, nodeSelector/affinity, PVC binding, Longhorn volume attach event를 확인합니다. +### CodePlaceDeploymentUnavailable / CodePlaceDeploymentRolloutStuck + +확인: + +```sh +kubectl -n get deploy,rs,pod +kubectl -n rollout status deploy/ +kubectl -n describe deploy +kubectl -n get event --sort-by=.lastTimestamp | tail -50 +``` + +PromQL: + +```promql +kube_deployment_status_replicas_unavailable{namespace=""} +kube_deployment_status_observed_generation{namespace=""} +kube_deployment_metadata_generation{namespace=""} +kube_pod_container_status_waiting_reason{namespace=""} +``` + +판단: + +- unavailable replica가 늘면 readiness probe, image pull, crash loop, resource pressure, scheduling event를 먼저 봅니다. +- observed generation이 metadata generation을 따라가지 못하면 Deployment controller reconcile, ReplicaSet 생성 실패, admission/image pull/secret 문제를 확인합니다. +- 배포 직후 API 5xx/latency와 같이 발생하면 사용자 영향이 있는 rollout 장애로 봅니다. + ### CodePlaceContainerCPUHigh / CodePlaceContainerMemoryHigh 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 6f3c51ca..edcfb04c 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -338,7 +338,7 @@ data: { "type": "timeseries", "title": "PVC Usage", - "gridPos": {"x": 0, "y": 34, "w": 12, "h": 6}, + "gridPos": {"x": 0, "y": 34, "w": 8, "h": 6}, "fieldConfig": { "defaults": { "unit": "percentunit", @@ -359,23 +359,33 @@ data: { "type": "timeseries", "title": "Node Pressure", - "gridPos": {"x": 12, "y": 34, "w": 12, "h": 6}, + "gridPos": {"x": 8, "y": 34, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "kube_node_status_condition{condition=~\"MemoryPressure|DiskPressure|PIDPressure\", status=\"true\"}", "legendFormat": "{{node}} {{condition}}"} ] }, + { + "type": "timeseries", + "title": "Deployment Unavailable Replicas", + "gridPos": {"x": 16, "y": 34, "w": 8, "h": 6}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "max by (namespace, deployment) (kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} unavailable"}, + {"expr": "max by (namespace, deployment) (kube_deployment_status_observed_generation{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"} != on (namespace, deployment) kube_deployment_metadata_generation{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} generation mismatch"} + ] + }, { "type": "row", "title": "Data Stores", - "gridPos": {"x": 0, "y": 40, "w": 24, "h": 1}, + "gridPos": {"x": 0, "y": 46, "w": 24, "h": 1}, "collapsed": false, "panels": [] }, { "type": "stat", "title": "PostgreSQL Ready", - "gridPos": {"x": 0, "y": 41, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 47, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -396,7 +406,7 @@ data: { "type": "stat", "title": "Redis Ready", - "gridPos": {"x": 6, "y": 41, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 47, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -417,7 +427,7 @@ data: { "type": "stat", "title": "PostgreSQL Exporter Up", - "gridPos": {"x": 12, "y": 41, "w": 6, "h": 5}, + "gridPos": {"x": 12, "y": 47, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -438,7 +448,7 @@ data: { "type": "stat", "title": "Redis Exporter Up", - "gridPos": {"x": 18, "y": 41, "w": 6, "h": 5}, + "gridPos": {"x": 18, "y": 47, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -459,7 +469,7 @@ data: { "type": "timeseries", "title": "PostgreSQL Restarts", - "gridPos": {"x": 0, "y": 46, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 52, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -468,7 +478,7 @@ data: { "type": "timeseries", "title": "Redis Restarts", - "gridPos": {"x": 6, "y": 46, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 52, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -477,7 +487,7 @@ data: { "type": "timeseries", "title": "PostgreSQL WAL Bytes", - "gridPos": {"x": 12, "y": 46, "w": 6, "h": 5}, + "gridPos": {"x": 12, "y": 52, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "Bps"}}, "targets": [ {"expr": "sum by (namespace) (rate(cnpg_collector_wal_bytes{namespace=~\"$namespace\", cluster=\"postgres\"}[5m]))", "legendFormat": "{{namespace}}"} @@ -486,7 +496,7 @@ data: { "type": "timeseries", "title": "Redis Memory Usage", - "gridPos": {"x": 18, "y": 46, "w": 6, "h": 5}, + "gridPos": {"x": 18, "y": 52, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} used"}, @@ -496,7 +506,7 @@ data: { "type": "timeseries", "title": "PostgreSQL Connections", - "gridPos": {"x": 0, "y": 51, "w": 8, "h": 6}, + "gridPos": {"x": 0, "y": 57, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "codeplace_postgres_connections{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} connections"}, @@ -506,7 +516,7 @@ data: { "type": "timeseries", "title": "PostgreSQL Long Tx and Lock Waits", - "gridPos": {"x": 8, "y": 51, "w": 8, "h": 6}, + "gridPos": {"x": 8, "y": 57, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "codeplace_postgres_long_transactions{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} long tx"}, @@ -516,7 +526,7 @@ data: { "type": "timeseries", "title": "Redis Clients and Rejections", - "gridPos": {"x": 16, "y": 51, "w": 8, "h": 6}, + "gridPos": {"x": 16, "y": 57, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "codeplace_redis_connected_clients{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} clients"}, @@ -527,7 +537,7 @@ data: { "type": "timeseries", "title": "Frontend Runtime Errors", - "gridPos": {"x": 0, "y": 57, "w": 12, "h": 6}, + "gridPos": {"x": 0, "y": 63, "w": 12, "h": 6}, "fieldConfig": { "defaults": { "unit": "ops", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index d0e72014..9b308b37 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -466,6 +466,24 @@ spec: annotations: summary: "A Kubernetes pod is pending or unschedulable" description: "A pod has been pending or unschedulable for 10 minutes. Check node capacity, taints, affinity, PVC binding, and FailedScheduling events." + - alert: CodePlaceDeploymentUnavailable + expr: max by (namespace, deployment) (kube_deployment_status_replicas_unavailable{namespace=~"code-place-(dev|prod)", deployment=~"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm"}) > 0 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace deployment has unavailable replicas" + description: "A CodePlace deployment has had unavailable replicas for 5 minutes. Check rollout status, pod events, readiness probes, image pulls, and resource pressure." + - alert: CodePlaceDeploymentRolloutStuck + expr: max by (namespace, deployment) (kube_deployment_status_observed_generation{namespace=~"code-place-(dev|prod)", deployment=~"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm"} != on (namespace, deployment) kube_deployment_metadata_generation{namespace=~"code-place-(dev|prod)", deployment=~"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm"}) > 0 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace deployment rollout is stuck" + description: "A CodePlace deployment observed generation has not caught up with metadata generation for 10 minutes. Check deployment events, ReplicaSet state, and rollout history." - alert: CodePlaceContainerCPUHigh expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 0.001) > 0.8 for: 10m From 620d7c9d031ab2e3a900e66e4211e76b444931a2 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:15:08 +0900 Subject: [PATCH 043/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=ED=8C=8C=EC=9D=B4?= =?UTF-8?q?=ED=94=84=EB=9D=BC=EC=9D=B8=20=EC=9E=A5=EC=95=A0=20=EA=B4=80?= =?UTF-8?q?=EC=B8=A1=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 7 ++- .../infra/observability-runbook/_index.md | 17 ++++++- .../monitoring/grafana-dashboard-logs.yaml | 43 ++++++++++++++++ kubernetes/monitoring/prometheus-rules.yaml | 50 +++++++++++++++++++ 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index e3e962a0..5c337bc4 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -190,7 +190,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. -- `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, 최근 backend error, judge/celery log 조회 dashboard. +- `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, Loki ingest/error/canary, Alloy write backpressure, 최근 backend error, judge/celery log 조회 dashboard. - `otel-collector.yaml`: backend/celery OTLP trace를 받아 Tempo로 전송하는 OpenTelemetry Collector. - `tempo.yaml`: Tempo single binary, Longhorn PVC 기반 trace 저장소. - `grafana-dashboard-traces.yaml`: Collector/Tempo readiness, span ingest/export, Tempo PVC usage dashboard. @@ -277,6 +277,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `LokiPVCAlmostFull`: Loki Longhorn PVC 사용률 85% 초과 10분 지속. - `AlloyDaemonSetUnavailable`: Alloy log collector가 모든 node에서 available하지 않음 5분 지속. - `LokiGatewayUnavailable`: Loki gateway Pod not ready 2분 지속. +- `LokiIngestionStalled`: Loki log line 수신량 0이 10분 지속. +- `LokiRequestErrors`: Loki 5xx response 발생 5분 지속. +- `LokiCanaryMissingEntries`: Loki canary write/readback 누락 발생 5분 지속. +- `AlloyLokiWriteDrops`: Alloy가 Loki write 전 log entry drop 2분 지속. +- `AlloyLokiWriteRetries`: Alloy Loki write batch retry 5분 지속. - `OTelCollectorUnavailable`: OpenTelemetry Collector Pod not ready 2분 지속. - `OTelCollectorRefusedSpans`: Collector span 수신 거부 발생 2분 지속. - `OTelCollectorExportFailures`: Collector span export 실패 발생 2분 지속. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 988a1e21..c3f6d1fe 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -949,7 +949,7 @@ kubectl -n monitoring describe pvc - 당장 여유가 없으면 Longhorn PVC size를 증설합니다. - 반복되면 retention 기간을 줄이거나 prod 로그 저장 방식을 내부 S3 호환 object storage로 분리합니다. -### AlloyDaemonSetUnavailable / LokiGatewayUnavailable +### AlloyDaemonSetUnavailable / LokiGatewayUnavailable / LokiIngestionStalled / LokiRequestErrors / LokiCanaryMissingEntries / AlloyLokiWriteDrops / AlloyLokiWriteRetries 확인: @@ -959,12 +959,27 @@ kubectl -n monitoring describe daemonset alloy kubectl -n monitoring logs daemonset/alloy --tail=200 kubectl -n monitoring get pod -l app.kubernetes.io/component=gateway kubectl -n monitoring logs deploy/loki-gateway --tail=200 +kubectl -n monitoring logs loki-0 --tail=200 +``` + +PromQL: + +```promql +sum(rate(loki_distributor_lines_received_total{namespace="monitoring"}[5m])) +sum(rate(loki_request_duration_seconds_count{namespace="monitoring", status_code=~"5.."}[5m])) +sum(increase(loki_canary_missing_entries_total{namespace="monitoring"}[5m])) +sum(increase(loki_write_dropped_entries_total{namespace="monitoring"}[5m])) +sum(increase(loki_write_batch_retries_total{namespace="monitoring"}[5m])) ``` 판단: - Alloy가 특정 node에서만 비면 해당 node의 scheduling, taint, resource pressure를 봅니다. - Loki gateway가 비면 Alloy write와 Grafana query가 실패할 수 있습니다. +- Loki가 ready인데 ingest rate가 0이면 Alloy discovery/relabel, node별 container log path, gateway write path를 먼저 확인합니다. +- Loki 5xx가 있으면 gateway와 `loki-0` 로그를 같이 보고, PVC full/Longhorn degraded/compactor error를 확인합니다. +- Loki canary missing entry는 write 성공 후 readback 또는 storage/query path가 깨진 상태일 수 있습니다. +- Alloy write drop/retry가 있으면 Loki gateway 응답, network, Loki backpressure, Alloy resource limit을 확인합니다. ### CodePlaceNodePressure diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index bd71d69c..2946b09f 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -256,6 +256,49 @@ data: "targets": [ {"expr": "{namespace=~\"$namespace\", app=\"backend\"} | json | logger=\"frontend.error\""} ] + }, + { + "type": "timeseries", + "title": "Loki Ingestion Rate", + "gridPos": {"x": 0, "y": 39, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum(rate(loki_distributor_lines_received_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "lines received"}, + {"expr": "sum(rate(loki_distributor_bytes_received_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "bytes received"} + ] + }, + { + "type": "timeseries", + "title": "Loki Request Errors", + "gridPos": {"x": 8, "y": 39, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "reqps"}}, + "targets": [ + {"expr": "sum by (route, status_code) (rate(loki_request_duration_seconds_count{namespace=\"monitoring\", status_code=~\"5..\"}[5m])) or vector(0)", "legendFormat": "{{route}} {{status_code}}"}, + {"expr": "sum by (route, status_code) (rate(loki_request_duration_seconds_count{namespace=\"monitoring\", status_code=~\"4..\"}[5m])) or vector(0)", "legendFormat": "{{route}} {{status_code}}"} + ] + }, + { + "type": "timeseries", + "title": "Alloy Loki Write Backpressure", + "gridPos": {"x": 16, "y": 39, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum(rate(loki_write_batch_retries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "batch retries/s"}, + {"expr": "sum(rate(loki_write_dropped_entries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "dropped entries/s"} + ] + }, + { + "type": "timeseries", + "title": "Loki Canary Missing Entries", + "gridPos": {"x": 0, "y": 46, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum(increase(loki_canary_missing_entries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "missing entries"} + ] } ] } diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 9b308b37..2f0517a9 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -613,6 +613,56 @@ spec: annotations: summary: "Loki gateway is not ready" description: "The Loki gateway has not been ready for 2 minutes. Alloy writes and Grafana Loki queries may fail." + - alert: LokiIngestionStalled + expr: (sum(rate(loki_distributor_lines_received_total{namespace="monitoring"}[5m])) == 0) or absent(loki_distributor_lines_received_total{namespace="monitoring"}) + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki is receiving no log lines" + description: "Loki has received zero log lines for 10 minutes. Check Alloy discovery, Loki gateway writes, and Loki ingestion health." + - alert: LokiRequestErrors + expr: sum(rate(loki_request_duration_seconds_count{namespace="monitoring", status_code=~"5.."}[5m])) > 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki is returning 5xx responses" + description: "Loki has returned 5xx responses for 5 minutes. Log writes or Grafana log queries may be failing." + - alert: LokiCanaryMissingEntries + expr: sum(increase(loki_canary_missing_entries_total{namespace="monitoring"}[5m])) > 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki canary cannot read back written logs" + description: "Loki canary reports missing entries. Loki may be accepting writes while queries or storage are unhealthy." + - alert: AlloyLokiWriteDrops + expr: sum(increase(loki_write_dropped_entries_total{namespace="monitoring"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Alloy is dropping log entries" + description: "Alloy dropped log entries before writing to Loki. Check Alloy limits, Loki gateway errors, and backpressure." + - alert: AlloyLokiWriteRetries + expr: sum(increase(loki_write_batch_retries_total{namespace="monitoring"}[5m])) > 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Alloy is retrying Loki writes" + description: "Alloy has retried Loki write batches for 5 minutes. Loki writes may be slow or intermittently failing." - alert: OTelCollectorUnavailable expr: (max by (namespace, pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) for: 2m From fb3c03c3b817b1431ebb1e306feca75edf9cdec8 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:18:09 +0900 Subject: [PATCH 044/147] =?UTF-8?q?=EA=B7=B8=EB=9D=BC=ED=8C=8C=EB=82=98=20?= =?UTF-8?q?=EA=B3=B5=EA=B0=9C=20=EA=B2=BD=EB=A1=9C=20=EA=B4=80=EC=B8=A1?= =?UTF-8?q?=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 10 ++++---- .../infra/observability-runbook/_index.md | 5 ++-- kubernetes/monitoring/prometheus-rules.yaml | 10 ++++++++ .../monitoring/public-endpoint-probes.yaml | 23 +++++++++++++++++++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 5c337bc4..c53bf44e 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -27,12 +27,13 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니다. 이 엔드포인트는 ServiceMonitor가 cluster 내부에서 scrape하며 외부 Ingress에는 연결하지 않습니다. -공개 사용자 경로는 Blackbox exporter 기반 synthetic probe로 별도 확인합니다. 이 probe는 frontend와 hub-auth 공개 URL을 30초 간격으로 호출해 DNS, Traefik TLS termination, frontend nginx, OAuth helper 기본 라우팅을 함께 검증합니다. +공개 사용자 경로는 Blackbox exporter 기반 synthetic probe로 별도 확인합니다. 이 probe는 frontend, hub-auth, Grafana 공개 URL을 30초 간격으로 호출해 DNS, Traefik TLS termination, frontend nginx, OAuth helper, monitoring UI 기본 라우팅을 함께 검증합니다. - dev frontend: `https://k3s.code-place-dev.site/` - prod frontend: `https://code.pusan.ac.kr/` - dev hub-auth: `https://hub-auth-dev.code-place-dev.site/` - prod hub-auth: `https://hub-auth.code-place-dev.site/` +- Grafana: `https://monitoring.code-place-dev.site/` 추가된 CodePlace custom metrics는 다음과 같습니다. @@ -178,7 +179,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `backend-service-monitor.yaml`: backend `/metrics` scrape, interval 15s. - `blackbox-exporter.yaml`: public endpoint synthetic probe용 Blackbox exporter. -- `public-endpoint-probes.yaml`: dev/prod 공개 HTTPS URL Probe, interval 30s. +- `public-endpoint-probes.yaml`: dev/prod frontend, hub-auth, Grafana 공개 HTTPS URL Probe, interval 30s. - `prometheus-rules.yaml`: P0/P1 fast alert rules. - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. @@ -241,6 +242,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PublicEndpointTLSExpiringSoon`: 공개 HTTPS 인증서 만료 14일 이내. - `FrontendRuntimeErrorsSpike`: browser runtime error report 10개 초과/5분, 5분 지속. - `BlackboxExporterUnavailable`: synthetic probe exporter Pod not ready 2분 지속. +- `GrafanaPublicEndpointDown`: Grafana 공개 HTTPS endpoint probe 실패 2분 지속. - `KubernetesEventExporterUnavailable`: Kubernetes event exporter Pod not ready 2분 지속. - `GrafanaUnavailable`: Grafana Pod not ready 2분 지속. - `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. @@ -314,8 +316,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - prod: `kubectl apply -k kubernetes/overlays/prod` 5. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` -6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`가 healthy인지 확인합니다. -7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. +6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`, `codeplace-grafana-http`가 healthy인지 확인합니다. +7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev/Grafana availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. 8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, oldest in-flight submission age, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index c3f6d1fe..4a868f88 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -248,7 +248,7 @@ Grafana Explore: - backend 5xx도 같이 증가하면 app/backend 문제입니다. - ingress 5xx만 증가하면 Traefik route, Service endpoint, upstream connection 문제를 의심합니다. -### PublicEndpointDown / PublicEndpointTLSCritical +### PublicEndpointDown / GrafanaPublicEndpointDown / PublicEndpointTLSCritical 증상: 공개 HTTPS endpoint synthetic probe가 실패하거나 인증서 만료가 3일 이내입니다. @@ -262,7 +262,7 @@ probe_http_duration_seconds{probe_type="public-http"} ``` ```sh -kubectl -n monitoring get pod,svc,probe | grep -E 'blackbox|codeplace-public|hub-auth' +kubectl -n monitoring get pod,svc,probe | grep -E 'blackbox|codeplace-public|hub-auth|grafana' kubectl -n monitoring logs deploy/blackbox-exporter --tail=200 kubectl -n get ingress,svc,endpoints frontend hub-auth kubectl -n kube-system logs deploy/traefik --tail=200 @@ -272,6 +272,7 @@ kubectl -n kube-system logs deploy/traefik --tail=200 - `probe_success=0`이고 ingress/backend 지표는 정상이면 DNS, TLS, Traefik public route, frontend nginx 기본 경로 문제를 먼저 봅니다. - `service="hub-auth"` probe가 실패하면 OAuth helper Pod, Service, Ingress, GitHub OAuth secret 설정을 우선 확인합니다. +- `service="grafana"` probe가 실패하면 Grafana Pod readiness와 별개로 Grafana ingress, Traefik route, TLS resolver, public DNS를 확인합니다. - `probe_http_status_code`가 0이면 DNS/TCP/TLS 연결 실패 가능성이 큽니다. - 5xx이면 Traefik/frontend/backend 로그와 같은 시간대를 대조합니다. - TLS 만료 알림은 Traefik ACME resolver와 인증서 저장소를 확인합니다. diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 2f0517a9..76af3550 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -281,6 +281,16 @@ spec: annotations: summary: "Blackbox exporter is not ready" description: "The synthetic public endpoint probe exporter has not been ready for 2 minutes." + - alert: GrafanaPublicEndpointDown + expr: probe_success{environment="monitoring", service="grafana", probe_type="public-http"} == 0 + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Grafana public endpoint is down" + description: "The public Grafana monitoring endpoint synthetic HTTPS probe has failed for 2 minutes. Dashboards may be unavailable even if the Grafana pod is ready." - alert: KubernetesEventExporterUnavailable expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kubernetes-event-exporter-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kubernetes-event-exporter-.*"}) for: 2m diff --git a/kubernetes/monitoring/public-endpoint-probes.yaml b/kubernetes/monitoring/public-endpoint-probes.yaml index 38addfef..d46405b3 100644 --- a/kubernetes/monitoring/public-endpoint-probes.yaml +++ b/kubernetes/monitoring/public-endpoint-probes.yaml @@ -46,6 +46,29 @@ spec: --- apiVersion: monitoring.coreos.com/v1 kind: Probe +metadata: + name: codeplace-grafana-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-grafana-http + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://monitoring.code-place-dev.site/ + labels: + environment: monitoring + namespace: monitoring + service: grafana + probe_type: public-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe metadata: name: codeplace-hub-auth-dev-http labels: From f75b94198fc108fb1f3f5cfca211cd79f054f4cc Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:21:07 +0900 Subject: [PATCH 045/147] =?UTF-8?q?=EC=B1=84=EC=A0=90=20=EA=B2=BD=EB=A1=9C?= =?UTF-8?q?=20=EC=B6=94=EC=A0=81=20=EA=B5=AC=EA=B0=84=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/judge/dispatcher.py | 29 ++++++++++++------- .../observability-improvement-plan/_index.md | 2 ++ .../infra/observability-runbook/_index.md | 1 + 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/backend/judge/dispatcher.py b/backend/judge/dispatcher.py index 231fde01..e3b63fde 100644 --- a/backend/judge/dispatcher.py +++ b/backend/judge/dispatcher.py @@ -44,15 +44,20 @@ def __init__(self): self.server = None def __enter__(self) -> [JudgeServer, None]: - with transaction.atomic(): - servers = JudgeServer.objects.select_for_update().filter(is_disabled=False).order_by("task_number") - servers = [s for s in servers if s.status == "normal"] - for server in servers: - if server.task_number <= server.cpu_core * 2: - server.task_number = F("task_number") + 1 - server.save(update_fields=["task_number"]) - self.server = server - return server + with tracer.start_as_current_span("judge_server.select") as span: + with transaction.atomic(): + servers = JudgeServer.objects.select_for_update().filter(is_disabled=False).order_by("task_number") + servers = [s for s in servers if s.status == "normal"] + span.set_attribute("codeplace.judge_server.candidates", len(servers)) + for server in servers: + if server.task_number <= server.cpu_core * 2: + server.task_number = F("task_number") + 1 + server.save(update_fields=["task_number"]) + self.server = server + span.set_attribute("codeplace.judge_server.selected", True) + span.set_attribute("codeplace.judge_server.hostname", server.hostname) + return server + span.set_attribute("codeplace.judge_server.selected", False) return None def __exit__(self, exc_type, exc_val, exc_tb): @@ -211,7 +216,11 @@ def _judge(self, span): self.submission.result = error_test_case[0]["result"] else: self.submission.result = JudgeStatus.PARTIALLY_ACCEPTED - self.submission.save() + with tracer.start_as_current_span("submission.result_save") as save_span: + save_span.set_attribute("codeplace.submission_id", self.submission.id) + save_span.set_attribute("codeplace.problem_id", self.problem.id) + save_span.set_attribute("codeplace.judge.result", self.submission.result) + self.submission.save() span.set_attribute("codeplace.judge.result", self.submission.result) if self.contest_id: diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index c53bf44e..df22390c 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -158,7 +158,9 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있고, base/prod 기본 - `judge_task` - `submission.judge` +- `judge_server.select` - `judge_server.request` +- `submission.result_save` 기본 sampling 값은 `OTEL_TRACES_SAMPLER_ARG=0.05`입니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 4a868f88..c9b591ba 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -533,6 +533,7 @@ sum(rate(otelcol_exporter_send_failed_spans_total{namespace="monitoring"}[5m])) - export failure가 증가하면 Tempo readiness, `tempo.monitoring.svc.cluster.local:4317` 경로, Collector exporter queue/retry 로그를 봅니다. - Tempo가 down이면 Collector가 span export 실패를 기록합니다. - Tempo PVC가 85%를 넘으면 retention 단축, noisy trace sampling 조정, PVC 증설 중 하나를 먼저 결정합니다. +- 채점 병목은 Tempo에서 `judge_task`, `submission.judge`, `judge_server.select`, `judge_server.request`, `submission.result_save` span duration을 순서대로 봅니다. 선택 span이 길면 DB row lock/available judge 부족, request span이 길면 judge-server 실행/네트워크, result save span이 길면 DB lock/write 병목을 의심합니다. - prod는 기본 `OTEL_ENABLED=0`입니다. prod trace가 없다고 해서 장애는 아니며, dev에서 Collector/Tempo를 먼저 확인한 뒤 prod 승격 여부를 결정합니다. ### ApiLatencyHigh From c489eeadc70a5025b1857534f1d9ba1ad36094d2 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:25:01 +0900 Subject: [PATCH 046/147] =?UTF-8?q?=EC=9A=B4=EC=98=81=20=EC=9A=94=EC=95=BD?= =?UTF-8?q?=20=EA=B8=B0=EB=A1=9D=20=EA=B7=9C=EC=B9=99=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 16 ++++++++++++++-- .../monitoring/grafana-dashboard-codeplace.yaml | 10 +++++----- .../grafana-dashboard-public-endpoints.yaml | 6 +++--- kubernetes/monitoring/prometheus-rules.yaml | 17 +++++++++++++++++ kubernetes/monitoring/validate.sh | 4 ++++ 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index df22390c..746d7565 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -58,6 +58,18 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_redis_rejected_connections_total` - `codeplace_frontend_error_total{surface,error_type}` +운영 대시보드와 빠른 판단에는 다음 recording rule을 사용합니다. + +- `codeplace:api_request_rate5m` +- `codeplace:api_5xx_rate5m` +- `codeplace:api_5xx_ratio5m` +- `codeplace:api_p95_latency_seconds5m` +- `codeplace:public_endpoint_availability5m` +- `codeplace:judge_available_total` +- `codeplace:judge_waiting_queue_length` + +이 recording rule은 원본 metric을 대체하지 않습니다. 알림과 대시보드에서 반복되는 PromQL을 줄이고, user path와 judge path의 최근 상태를 한 화면에서 빠르게 읽기 위한 요약 계층입니다. + prod AI 힌트용 vLLM은 OpenAI-compatible server의 `/metrics`를 ServiceMonitor로 scrape합니다. vLLM production metrics 기준으로 다음 항목을 봅니다. - `vllm:num_requests_running` @@ -182,7 +194,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `backend-service-monitor.yaml`: backend `/metrics` scrape, interval 15s. - `blackbox-exporter.yaml`: public endpoint synthetic probe용 Blackbox exporter. - `public-endpoint-probes.yaml`: dev/prod frontend, hub-auth, Grafana 공개 HTTPS URL Probe, interval 30s. -- `prometheus-rules.yaml`: P0/P1 fast alert rules. +- `prometheus-rules.yaml`: P0/P1 fast alert rules와 운영 요약 recording rules. - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. - `grafana-dashboard-ai-inference.yaml`: prod vLLM readiness, queue/cache/latency/token throughput, HF cache PVC dashboard. @@ -320,7 +332,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`, `codeplace-grafana-http`가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev/Grafana availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, frontend runtime error, submission status, oldest in-flight submission age, waiting queue, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission status, oldest in-flight submission age, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index edcfb04c..39ca1ba6 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -65,7 +65,7 @@ data: "fieldConfig": {"defaults": {"unit": "reqps"}}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:api_request_rate5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} ] }, { @@ -87,7 +87,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "((sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\", status_code=~\"5..\"}[5m])) or (sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m])) * 0)) / clamp_min(sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m])), 0.001)", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:api_5xx_ratio5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} ] }, { @@ -109,7 +109,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:api_p95_latency_seconds5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} ] }, { @@ -131,7 +131,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:judge_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} ] }, { @@ -152,7 +152,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum by (namespace) (codeplace_judge_server_available{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:judge_available_total{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} ] }, { diff --git a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml index 57e24229..87519b57 100644 --- a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml +++ b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml @@ -38,7 +38,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "probe_success{environment=\"prod\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{instance}}"} + {"expr": "codeplace:public_endpoint_availability5m{environment=\"prod\"}", "legendFormat": "{{service}} {{instance}}"} ] }, { @@ -60,7 +60,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "probe_success{environment=\"dev\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{instance}}"} + {"expr": "codeplace:public_endpoint_availability5m{environment=\"dev\"}", "legendFormat": "{{service}} {{instance}}"} ] }, { @@ -160,7 +160,7 @@ data: "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}}, "targets": [ - {"expr": "probe_success{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + {"expr": "codeplace:public_endpoint_availability5m", "legendFormat": "{{environment}} {{service}} {{instance}}"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 76af3550..abd28d3d 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -6,6 +6,23 @@ metadata: release: kube-prometheus-stack spec: groups: + - name: codeplace.recording + interval: 15s + rules: + - record: codeplace:api_request_rate5m + expr: sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)"}[5m])) + - record: codeplace:api_5xx_rate5m + expr: sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)", status_code=~"5.."}[5m])) + - record: codeplace:api_5xx_ratio5m + expr: codeplace:api_5xx_rate5m / clamp_min(codeplace:api_request_rate5m, 0.001) + - record: codeplace:api_p95_latency_seconds5m + expr: histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~"code-place-(dev|prod)"}[5m]))) + - record: codeplace:public_endpoint_availability5m + expr: avg by (environment, namespace, service, instance) (avg_over_time(probe_success{probe_type="public-http"}[5m])) + - record: codeplace:judge_available_total + expr: sum by (namespace) (codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) + - record: codeplace:judge_waiting_queue_length + expr: max by (namespace) (codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"}) - name: codeplace.p0 interval: 15s rules: diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 4a898a55..4f928be3 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -79,6 +79,10 @@ alert_instances = {} for group in rules["spec"]["groups"]: expected_priority = "P0" if group["name"].endswith(".p0") else "P1" if group["name"].endswith(".p1") else None for rule in group.get("rules", []): + if rule.get("record"): + if not rule.get("expr"): + raise SystemExit(f"{rule['record']} is missing expr") + continue alert_name = rule.get("alert") if not alert_name: raise SystemExit(f"rule in {group['name']} is missing alert name") From 374f083e2606ff4864d8f2d96b615f5d879b409f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:26:37 +0900 Subject: [PATCH 047/147] =?UTF-8?q?=ED=95=B5=EC=8B=AC=20P0=20=EC=95=8C?= =?UTF-8?q?=EB=A6=BC=20=EB=88=84=EB=9D=BD=20=EC=A1=B0=EA=B1=B4=20=EB=B3=B4?= =?UTF-8?q?=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 6 +-- kubernetes/monitoring/prometheus-rules.yaml | 51 +++++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 746d7565..1a549226 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -226,15 +226,15 @@ Email fallback은 SMTP host/from/to/auth가 필요하므로 기본 kustomization P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합니다. -- `BackendTargetDown`: backend scrape 실패 1분 지속. +- `BackendTargetDown`: backend scrape 실패 또는 target series missing 1분 지속. - `Api5xxSpike`: backend 5xx 비율 5% 초과 2분 지속. - `Ingress5xxSpike`: Traefik 5xx 비율 5% 초과 2분 지속. - `PublicEndpointDown`: prod 공개 HTTPS endpoint probe 실패 1분 지속. frontend와 hub-auth를 포함합니다. - `PublicEndpointTLSCritical`: prod 공개 HTTPS 인증서 만료 3일 이내. - `JudgeAllUnavailable`: 사용 가능한 judge-server 0개 1분 지속. - `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. -- `PostgresUnavailable`: PostgreSQL Pod not ready 1분 지속. -- `RedisUnavailable`: Redis Pod not ready 1분 지속. +- `PostgresUnavailable`: PostgreSQL Pod not ready 또는 readiness series missing 1분 지속. +- `RedisUnavailable`: Redis Pod not ready 또는 readiness series missing 1분 지속. - `PostgresExporterDown`: CNPG PostgreSQL exporter metric 수집 실패 2분 지속. - `RedisExporterDown`: Redis exporter metric 수집 실패 2분 지속. - `LokiUnavailable`: Loki single-binary Pod not ready 1분 지속. diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index abd28d3d..37c9f246 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -27,14 +27,25 @@ spec: interval: 15s rules: - alert: BackendTargetDown - expr: up{job="backend", namespace=~"code-place-(dev|prod)"} == 0 + expr: (min(up{job="backend", namespace="code-place-dev"}) == 0) or absent(up{job="backend", namespace="code-place-dev"}) for: 1m labels: + namespace: code-place-dev + severity: critical + priority: P0 + annotations: + summary: "Backend metrics target is down in dev" + description: "Prometheus cannot scrape the dev backend /metrics endpoint, or the backend target series is missing, for 1 minute." + - alert: BackendTargetDown + expr: (min(up{job="backend", namespace="code-place-prod"}) == 0) or absent(up{job="backend", namespace="code-place-prod"}) + for: 1m + labels: + namespace: code-place-prod severity: critical priority: P0 annotations: - summary: "Backend metrics target is down" - description: "Prometheus cannot scrape the backend /metrics endpoint for 1 minute." + summary: "Backend metrics target is down in prod" + description: "Prometheus cannot scrape the prod backend /metrics endpoint, or the backend target series is missing, for 1 minute." - alert: Api5xxSpike expr: sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)", status_code=~"5.."}[2m])) / clamp_min(sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)"}[2m])), 1) > 0.05 for: 2m @@ -102,23 +113,45 @@ spec: summary: "Judge-server heartbeat is stale" description: "A judge-server heartbeat is older than 180 seconds." - alert: PostgresUnavailable - expr: min by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"postgres-[0-9]+"}) == 0 + expr: (min(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) + for: 1m + labels: + namespace: code-place-dev + severity: critical + priority: P0 + annotations: + summary: "PostgreSQL pod is not ready in dev" + description: "At least one dev PostgreSQL pod has not been ready, or PostgreSQL pod readiness metrics are missing, for 1 minute." + - alert: PostgresUnavailable + expr: (min(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"postgres-[0-9]+"}) + for: 1m + labels: + namespace: code-place-prod + severity: critical + priority: P0 + annotations: + summary: "PostgreSQL pod is not ready in prod" + description: "At least one prod PostgreSQL pod has not been ready, or PostgreSQL pod readiness metrics are missing, for 1 minute." + - alert: RedisUnavailable + expr: (min(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) for: 1m labels: + namespace: code-place-dev severity: critical priority: P0 annotations: - summary: "PostgreSQL pod is not ready" - description: "At least one PostgreSQL pod has not been ready for 1 minute." + summary: "Redis pod is not ready in dev" + description: "At least one dev Redis pod has not been ready, or Redis pod readiness metrics are missing, for 1 minute." - alert: RedisUnavailable - expr: min by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0 + expr: (min(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"redis-.*|redis-replication-.*"}) for: 1m labels: + namespace: code-place-prod severity: critical priority: P0 annotations: - summary: "Redis pod is not ready" - description: "At least one Redis pod has not been ready for 1 minute." + summary: "Redis pod is not ready in prod" + description: "At least one prod Redis pod has not been ready, or Redis pod readiness metrics are missing, for 1 minute." - alert: PostgresExporterDown expr: (min(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) == 0) or absent(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) for: 2m From e32953f4d7c7b8c94dcca566be6119617921042f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:29:10 +0900 Subject: [PATCH 048/147] =?UTF-8?q?=EC=84=9C=EB=B9=84=EC=8A=A4=20=EC=97=94?= =?UTF-8?q?=EB=93=9C=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=EA=B4=80=EC=B8=A1?= =?UTF-8?q?=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 4 ++- .../infra/observability-runbook/_index.md | 27 +++++++++++++++++++ .../grafana-dashboard-codeplace.yaml | 20 ++++++++++++++ kubernetes/monitoring/prometheus-rules.yaml | 19 +++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 1a549226..50b054c0 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -279,6 +279,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `KubernetesPodUnschedulable`: Pending 또는 Unschedulable 상태 10분 지속. - `CodePlaceDeploymentUnavailable`: 주요 Deployment unavailable replica 5분 지속. - `CodePlaceDeploymentRolloutStuck`: Deployment observed generation 미반영 10분 지속. +- `CodePlaceServiceNoReadyEndpoints`: 주요 Service ready endpoint 0개 2분 지속. +- `VLLMServiceNoReadyEndpoints`: prod vLLM Service ready endpoint 0개 2분 지속. - `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. - `CodePlaceContainerMemoryHigh`: 주요 컨테이너 memory limit 사용률 85% 초과 10분 지속. - `PostgresCollectorError`: CNPG PostgreSQL metrics collection error 5분 지속. @@ -332,7 +334,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`, `codeplace-grafana-http`가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev/Grafana availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission status, oldest in-flight submission age, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission status, oldest in-flight submission age, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, Service ready endpoint, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index c9b591ba..138c0efb 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -788,6 +788,33 @@ kube_pod_container_status_waiting_reason{namespace=""} - observed generation이 metadata generation을 따라가지 못하면 Deployment controller reconcile, ReplicaSet 생성 실패, admission/image pull/secret 문제를 확인합니다. - 배포 직후 API 5xx/latency와 같이 발생하면 사용자 영향이 있는 rollout 장애로 봅니다. +### CodePlaceServiceNoReadyEndpoints / VLLMServiceNoReadyEndpoints + +확인: + +```sh +kubectl -n get svc,endpoints,endpointslice +kubectl -n describe svc +kubectl -n get pod --show-labels +kubectl -n get event --sort-by=.lastTimestamp | tail -50 +``` + +PromQL: + +```promql +kube_endpoint_address_available{namespace=""} +kube_endpoint_address_not_ready{namespace=""} +kube_pod_status_ready{namespace="", condition="true"} +kube_deployment_status_replicas_unavailable{namespace=""} +``` + +판단: + +- Pod는 ready인데 endpoint가 0이면 Service selector label과 Pod label 불일치를 먼저 봅니다. +- Pod readiness가 같이 0이면 rollout/readiness/image pull/crash loop 문제로 봅니다. +- ingress 5xx와 같이 발생하면 Traefik route 자체보다 Service endpoint 부재가 직접 원인일 가능성이 큽니다. +- vLLM endpoint가 0이면 AI hint path만 영향받을 수 있으므로 `CodePlace AI Inference` dashboard와 GPU node scheduling 상태를 같이 봅니다. + ### CodePlaceContainerCPUHigh / CodePlaceContainerMemoryHigh 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 39ca1ba6..895b0c21 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -375,6 +375,26 @@ data: {"expr": "max by (namespace, deployment) (kube_deployment_status_observed_generation{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"} != on (namespace, deployment) kube_deployment_metadata_generation{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} generation mismatch"} ] }, + { + "type": "timeseries", + "title": "Service Ready Endpoints", + "gridPos": {"x": 0, "y": 40, "w": 24, "h": 6}, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "targets": [ + {"expr": "max by (namespace, endpoint) (kube_endpoint_address_available{namespace=~\"$namespace\", endpoint=~\"backend|frontend|hub-auth|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{endpoint}} ready"} + ] + }, { "type": "row", "title": "Data Stores", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 37c9f246..50895d7f 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -544,6 +544,25 @@ spec: annotations: summary: "CodePlace deployment rollout is stuck" description: "A CodePlace deployment observed generation has not caught up with metadata generation for 10 minutes. Check deployment events, ReplicaSet state, and rollout history." + - alert: CodePlaceServiceNoReadyEndpoints + expr: max by (namespace, endpoint) (kube_endpoint_address_available{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server"}) < 1 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace service has no ready endpoints" + description: "A CodePlace Service has had zero ready endpoints for 2 minutes. Check Service selector labels, Pod readiness, EndpointSlice state, and rollout events." + - alert: VLLMServiceNoReadyEndpoints + expr: max by (namespace, endpoint) (kube_endpoint_address_available{namespace="code-place-prod", endpoint="vllm"}) < 1 + for: 2m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "vLLM service has no ready endpoints" + description: "The prod vLLM Service has had zero ready endpoints for 2 minutes. AI hint routing may fail even if the Service object exists." - alert: CodePlaceContainerCPUHigh expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 0.001) > 0.8 for: 10m From 96b15906b0496c7e62163affa391c87f78a3a003 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:35:18 +0900 Subject: [PATCH 049/147] =?UTF-8?q?AI=20=ED=9E=8C=ED=8A=B8=20=EB=B0=B1?= =?UTF-8?q?=EC=97=94=EB=93=9C=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/problem/llm_hint.py | 15 +++++++ backend/problem/tests.py | 39 ++++++++++++++++++- backend/utils/observability_metrics.py | 11 ++++++ .../observability-improvement-plan/_index.md | 6 ++- .../infra/observability-runbook/_index.md | 24 ++++++++++++ .../grafana-dashboard-ai-inference.yaml | 33 +++++++++++++--- kubernetes/monitoring/prometheus-rules.yaml | 18 +++++++++ 7 files changed, 138 insertions(+), 8 deletions(-) diff --git a/backend/problem/llm_hint.py b/backend/problem/llm_hint.py index 588d45a5..70e6b6e1 100644 --- a/backend/problem/llm_hint.py +++ b/backend/problem/llm_hint.py @@ -7,6 +7,8 @@ from django.utils.html import escape, strip_tags import time +from utils.observability_metrics import AI_HINT_DURATION_SECONDS, AI_HINT_REQUESTS_TOTAL + LOCAL_VLLM_CHAT_COMPLETIONS_URL = "http://localhost:8000/v1/chat/completions" CLUSTER_VLLM_CHAT_COMPLETIONS_URL = "http://vllm.code-place-prod:8000/v1/chat/completions" VLLM_MODEL = "Qwen/Qwen3.5-9B" @@ -316,6 +318,8 @@ def stream_problem_hint(problem, previous_hints=None, user_code=None): return response = None + started_at = time.monotonic() + status = "success" try: response = requests.post( get_vllm_chat_completions_url(), @@ -338,13 +342,24 @@ def stream_problem_hint(problem, previous_hints=None, user_code=None): try: chunk = json.loads(payload) except ValueError as exc: + status = "stream_parse_error" raise LLMHintError("Failed to parse vLLM stream") from exc text = _extract_stream_delta(chunk) if text: yield text except requests.RequestException as exc: + status = "request_error" raise LLMHintError("Failed to call vLLM") from exc + except GeneratorExit: + status = "client_closed" + raise + except Exception: + if status == "success": + status = "error" + raise finally: + AI_HINT_REQUESTS_TOTAL.labels(status=status).inc() + AI_HINT_DURATION_SECONDS.labels(status=status).observe(max(time.monotonic() - started_at, 0)) if response is not None: response.close() diff --git a/backend/problem/tests.py b/backend/problem/tests.py index 28a94971..b92cb6f3 100644 --- a/backend/problem/tests.py +++ b/backend/problem/tests.py @@ -23,7 +23,7 @@ from utils.constants import CONTEST_PASSWORD_SESSION_KEY from .llm_hint import (CLUSTER_VLLM_CHAT_COMPLETIONS_URL, LOCAL_VLLM_CHAT_COMPLETIONS_URL, VLLM_CONNECT_TIMEOUT_SEC, VLLM_MODEL, VLLM_STREAM_READ_TIMEOUT_SEC, - get_vllm_chat_completions_url) + LLMHintError, get_vllm_chat_completions_url, stream_problem_hint) from .views.admin import TestCaseAPI from .utils import parse_problem_template @@ -320,6 +320,43 @@ def test_stream_llm_hint(self, mocked_post): (VLLM_CONNECT_TIMEOUT_SEC, VLLM_STREAM_READ_TIMEOUT_SEC), ) + @mock.patch("problem.llm_hint.AI_HINT_DURATION_SECONDS") + @mock.patch("problem.llm_hint.AI_HINT_REQUESTS_TOTAL") + @mock.patch("problem.llm_hint.requests.post") + def test_stream_llm_hint_records_success_metric(self, mocked_post, requests_total, duration_seconds): + mocked_post.return_value = self._mock_streaming_response([ + 'data: {"choices":[{"delta":{"content":"힌트"}}]}', + "data: [DONE]", + ]) + request_labels = mock.Mock() + duration_labels = mock.Mock() + requests_total.labels.return_value = request_labels + duration_seconds.labels.return_value = duration_labels + + self.assertEqual("".join(stream_problem_hint(self.problem)), "힌트") + + requests_total.labels.assert_called_once_with(status="success") + request_labels.inc.assert_called_once() + duration_seconds.labels.assert_called_once_with(status="success") + duration_labels.observe.assert_called_once() + + @mock.patch("problem.llm_hint.AI_HINT_DURATION_SECONDS") + @mock.patch("problem.llm_hint.AI_HINT_REQUESTS_TOTAL") + @mock.patch("problem.llm_hint.requests.post", side_effect=requests.Timeout) + def test_stream_llm_hint_records_request_error_metric(self, mocked_post, requests_total, duration_seconds): + request_labels = mock.Mock() + duration_labels = mock.Mock() + requests_total.labels.return_value = request_labels + duration_seconds.labels.return_value = duration_labels + + with self.assertRaises(LLMHintError): + list(stream_problem_hint(self.problem)) + + requests_total.labels.assert_called_once_with(status="request_error") + request_labels.inc.assert_called_once() + duration_seconds.labels.assert_called_once_with(status="request_error") + duration_labels.observe.assert_called_once() + @mock.patch("problem.llm_hint.requests.post") def test_stream_llm_hint_problem_limit(self, mocked_post): """한 문제당 5회 제한에 걸리는지 테스트""" diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index e6599995..7134eb51 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -52,6 +52,17 @@ "Total frontend runtime errors reported by browser clients.", ["surface", "error_type"], ) +AI_HINT_REQUESTS_TOTAL = Counter( + "codeplace_ai_hint_requests_total", + "Total backend AI hint streaming requests to the LLM service.", + ["status"], +) +AI_HINT_DURATION_SECONDS = Histogram( + "codeplace_ai_hint_duration_seconds", + "Backend AI hint streaming duration.", + ["status"], + buckets=(0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600), +) class CodePlaceCollector: diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 50b054c0..357dd2d7 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -49,6 +49,8 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_celery_task_count{task_name,status}` - `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_celery_task_last_runtime_seconds{task_name,status}` +- `codeplace_ai_hint_requests_total{status}` +- `codeplace_ai_hint_duration_seconds{status}` - `codeplace_postgres_connections` - `codeplace_postgres_max_connections` - `codeplace_postgres_long_transactions` @@ -313,6 +315,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `VLLMWaitingQueueHigh`: prod vLLM waiting request 5 초과 5분 지속. - `VLLMKVCacheHigh`: prod vLLM KV cache 사용률 90% 초과 10분 지속. - `VLLMRequestLatencyHigh`: prod vLLM p95 e2e latency 60초 초과 10분 지속. +- `AIHintBackendErrors`: backend AI hint stream request/parse/internal error 발생 2분 지속. +- `AIHintBackendLatencyHigh`: backend AI hint successful stream p95 duration 120초 초과 10분 지속. - `DCGMExporterUnavailable`: DCGM exporter DaemonSet unavailable 2분 지속. - `GPUUtilizationHigh`: GPU utilization 95% 초과 15분 지속. - `GPUMemoryHigh`: GPU framebuffer memory usage 90% 초과 10분 지속. @@ -341,7 +345,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 12. Grafana의 `CodePlace Monitoring Stack` dashboard에서 Prometheus, Alertmanager, Grafana, operator readiness, target scrape failure, rule evaluation failure, notification failure panel을 확인합니다. 13. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. 14. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. -15. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. +15. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, backend AI hint stream status/duration, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. 16. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. 17. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"}` 쿼리가 Kubernetes Warning event를 반환하는지 확인합니다. 18. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 138c0efb..2de67a4b 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -928,6 +928,30 @@ kubectl -n code-place-prod describe pvc vllm-hf-cache - e2e latency만 증가하면 prompt/generation token 증가, backend streaming read timeout, vLLM 로그의 model/engine warning을 확인합니다. 현재 vLLM은 prod 전용입니다. dev에서 AI hint를 별도 검증하려면 dev vLLM 배포와 ServiceMonitor 확장이 먼저 필요합니다. +### AIHintBackendErrors / AIHintBackendLatencyHigh + +확인: + +```promql +sum by (namespace, status) (increase(codeplace_ai_hint_requests_total{namespace=""}[5m])) +histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace="", status="success"}[10m]))) +{__name__="vllm:num_requests_waiting", namespace="code-place-prod"} +histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) +``` + +```sh +kubectl -n logs deploy/backend --tail=200 +kubectl -n code-place-prod get svc,endpoints vllm +kubectl -n code-place-prod logs deploy/vllm --tail=200 +``` + +판단: + +- `request_error`는 backend에서 vLLM Service까지의 endpoint, network, timeout, HTTP error를 먼저 확인합니다. +- `stream_parse_error`는 vLLM OpenAI-compatible stream format과 backend parser 호환성을 확인합니다. +- backend duration과 vLLM queue/e2e latency가 같이 높으면 vLLM/GPU 병목으로 봅니다. +- backend duration만 높으면 client stream read behavior, backend worker saturation, proxy timeout을 확인합니다. + ### DCGMExporterUnavailable / GPUUtilizationHigh / GPUMemoryHigh / GPUTemperatureHigh 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml index c18787b7..74b7992d 100644 --- a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml +++ b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml @@ -208,10 +208,31 @@ data: {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} ] }, + { + "type": "timeseries", + "title": "Backend AI Hint Streams", + "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (namespace, status) (rate(codeplace_ai_hint_requests_total{namespace=~\"code-place-(dev|prod)\"}[5m]))", "legendFormat": "{{namespace}} {{status}} streams/s"} + ] + }, + { + "type": "timeseries", + "title": "Backend AI Hint Duration", + "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=~\"code-place-(dev|prod)\", status=\"success\"}[10m])))", "legendFormat": "{{namespace}} success p95"}, + {"expr": "histogram_quantile(0.95, sum by (namespace, status, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=~\"code-place-(dev|prod)\", status!=\"success\"}[10m])))", "legendFormat": "{{namespace}} {{status}} p95"} + ] + }, { "type": "timeseries", "title": "GPU Utilization", - "gridPos": {"x": 0, "y": 20, "w": 6, "h": 8}, + "gridPos": {"x": 0, "y": 28, "w": 6, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -233,7 +254,7 @@ data: { "type": "timeseries", "title": "GPU FB Memory Usage", - "gridPos": {"x": 6, "y": 20, "w": 6, "h": 8}, + "gridPos": {"x": 6, "y": 28, "w": 6, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -255,7 +276,7 @@ data: { "type": "timeseries", "title": "GPU Temperature", - "gridPos": {"x": 12, "y": 20, "w": 6, "h": 8}, + "gridPos": {"x": 12, "y": 28, "w": 6, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -277,7 +298,7 @@ data: { "type": "timeseries", "title": "GPU XID Errors", - "gridPos": {"x": 18, "y": 20, "w": 6, "h": 8}, + "gridPos": {"x": 18, "y": 28, "w": 6, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -298,7 +319,7 @@ data: { "type": "timeseries", "title": "GPU Tensor / DRAM Activity", - "gridPos": {"x": 0, "y": 28, "w": 12, "h": 8}, + "gridPos": {"x": 0, "y": 36, "w": 12, "h": 8}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "percentunit"}}, "targets": [ @@ -309,7 +330,7 @@ data: { "type": "timeseries", "title": "GPU FB Memory Bytes", - "gridPos": {"x": 12, "y": 28, "w": 12, "h": 8}, + "gridPos": {"x": 12, "y": 36, "w": 12, "h": 8}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 50895d7f..f4e12177 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -872,6 +872,24 @@ spec: annotations: summary: "vLLM request latency is high" description: "vLLM p95 end-to-end request latency has been above 60 seconds for 10 minutes." + - alert: AIHintBackendErrors + expr: sum by (namespace, status) (increase(codeplace_ai_hint_requests_total{namespace=~"code-place-(dev|prod)", status=~"request_error|stream_parse_error|error"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "Backend AI hint stream failures detected" + description: "Backend AI hint streaming has reported request, parse, or internal errors in the last 5 minutes. Check backend logs, vLLM Service endpoint, vLLM metrics, and request_id-linked traces." + - alert: AIHintBackendLatencyHigh + expr: histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=~"code-place-(dev|prod)", status="success"}[10m]))) > 120 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "Backend AI hint stream latency is high" + description: "Backend AI hint successful stream p95 duration has been above 120 seconds for 10 minutes. Check vLLM queue, GPU utilization, and client stream behavior." - alert: DCGMExporterUnavailable expr: (kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"} == 0) or (kube_daemonset_status_number_available{namespace="monitoring", daemonset="dcgm-exporter"} < kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"}) or absent(kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"}) for: 2m From 7c2587993e758431a0fa8a937bdbe20b18549e50 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:38:43 +0900 Subject: [PATCH 050/147] =?UTF-8?q?AI=20=ED=9E=8C=ED=8A=B8=20API=20?= =?UTF-8?q?=EA=B2=B0=EA=B3=BC=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/problem/tests.py | 31 +++++ backend/problem/views/oj.py | 108 ++++++++++++------ backend/utils/observability_metrics.py | 5 + .../observability-improvement-plan/_index.md | 4 +- .../infra/observability-runbook/_index.md | 5 +- .../grafana-dashboard-ai-inference.yaml | 14 ++- kubernetes/monitoring/prometheus-rules.yaml | 9 ++ 7 files changed, 134 insertions(+), 42 deletions(-) diff --git a/backend/problem/tests.py b/backend/problem/tests.py index b92cb6f3..25d4f489 100644 --- a/backend/problem/tests.py +++ b/backend/problem/tests.py @@ -320,6 +320,22 @@ def test_stream_llm_hint(self, mocked_post): (VLLM_CONNECT_TIMEOUT_SEC, VLLM_STREAM_READ_TIMEOUT_SEC), ) + @mock.patch("problem.views.oj.AI_HINT_API_OUTCOME_TOTAL") + @mock.patch("problem.llm_hint.requests.post") + def test_stream_llm_hint_records_api_success_outcome(self, mocked_post, outcome_total): + mocked_post.return_value = self._mock_streaming_response([ + 'data: {"choices":[{"delta":{"content":"힌트"}}]}', + "data: [DONE]", + ]) + labels = mock.Mock() + outcome_total.labels.return_value = labels + + resp = self.client.get(f"{self.url}?problem_id={self.problem._id}") + self._streaming_body(resp) + + outcome_total.labels.assert_called_once_with(status="success", scope="practice") + labels.inc.assert_called_once() + @mock.patch("problem.llm_hint.AI_HINT_DURATION_SECONDS") @mock.patch("problem.llm_hint.AI_HINT_REQUESTS_TOTAL") @mock.patch("problem.llm_hint.requests.post") @@ -357,6 +373,21 @@ def test_stream_llm_hint_records_request_error_metric(self, mocked_post, request duration_seconds.labels.assert_called_once_with(status="request_error") duration_labels.observe.assert_called_once() + @mock.patch("problem.views.oj.AI_HINT_API_OUTCOME_TOTAL") + @mock.patch("problem.llm_hint.requests.post") + def test_stream_llm_hint_records_problem_limit_outcome(self, mocked_post, outcome_total): + for i in range(5): + ProblemAIHintLog.objects.create(user=self.user, problem=self.problem, hint_content=f"더미 {i}") + labels = mock.Mock() + outcome_total.labels.return_value = labels + + resp = self.client.get(f"{self.url}?problem_id={self.problem._id}") + self._streaming_body(resp) + + outcome_total.labels.assert_called_once_with(status="problem_limit_exceeded", scope="practice") + labels.inc.assert_called_once() + mocked_post.assert_not_called() + @mock.patch("problem.llm_hint.requests.post") def test_stream_llm_hint_problem_limit(self, mocked_post): """한 문제당 5회 제한에 걸리는지 테스트""" diff --git a/backend/problem/views/oj.py b/backend/problem/views/oj.py index af383f7c..03eddf60 100644 --- a/backend/problem/views/oj.py +++ b/backend/problem/views/oj.py @@ -14,6 +14,7 @@ from submission.models import JudgeStatus, Submission from utils.api import APIView from utils.constants import Difficulty, ProblemField, Tier +from utils.observability_metrics import AI_HINT_API_OUTCOME_TOTAL from ..llm_hint import LLMHintError, MAX_USER_CODE_LENGTH, stream_problem_hint from ..models import (Problem, ProblemRuleType, ProblemTag, get_default_week_info, ProblemAIHintLog) @@ -164,17 +165,41 @@ def _error_events(self, message, err="llm-hint-unavailable"): yield self._format_sse_event("app-error", {"error": err, "message": message}) yield self._format_sse_event("done", {"done": True}) - def _error_response(self, message, err="llm-hint-unavailable"): + def _record_outcome(self, status, scope): + AI_HINT_API_OUTCOME_TOTAL.labels(status=status, scope=scope).inc() + + @staticmethod + def _persist_hint_log(hint_log, full_text_chunks): + full_text = "".join(full_text_chunks) + if full_text.strip(): + if hint_log: + hint_log.hint_content = full_text + hint_log.save(update_fields=['hint_content']) + return True + + if hint_log: + hint_log.delete() + return False + + def _error_response(self, message, err="llm-hint-unavailable", metric_status=None, scope="practice"): + if metric_status: + self._record_outcome(metric_status, scope) return self._streaming_response(self._error_events(message, err=err)) def get(self, request): - if not request.user.is_authenticated: - return self._error_response("로그인이 필요합니다.", err="permission-denied") - problem_id = request.GET.get("problem_id") contest_id = request.GET.get("contest_id") + scope = "contest" if contest_id else "practice" + if not request.user.is_authenticated: + return self._error_response( + "로그인이 필요합니다.", + err="permission-denied", + metric_status="permission_denied", + scope=scope, + ) + if not problem_id: - return self._error_response("문제 번호가 필요합니다.") + return self._error_response("문제 번호가 필요합니다.", metric_status="bad_request", scope=scope) # 사용자가 현재 작성 중인 코드 (없으면 None) # 서버에서 한 번 더 길이를 제한해 과도하게 큰 요청을 차단 @@ -185,7 +210,12 @@ def get(self, request): try: contest = Contest.objects.select_related("created_by").get(id=contest_id, visible=True) except Contest.DoesNotExist: - return self._error_response("대회를 찾을 수 없습니다.", err="permission-denied") + return self._error_response( + "대회를 찾을 수 없습니다.", + err="permission-denied", + metric_status="permission_denied", + scope=scope, + ) if not request.user.is_contest_admin(contest): if contest.contest_type == ContestType.PASSWORD_PROTECTED_CONTEST: @@ -193,22 +223,36 @@ def get(self, request): request.session.get(CONTEST_PASSWORD_SESSION_KEY, {}).get(contest.id), contest.password, ): - return self._error_response("비밀번호가 올바르지 않거나 만료되었습니다.", err="permission-denied") + return self._error_response( + "비밀번호가 올바르지 않거나 만료되었습니다.", + err="permission-denied", + metric_status="permission_denied", + scope=scope, + ) if contest.status == ContestStatus.CONTEST_NOT_START: - return self._error_response("아직 시작하지 않은 대회입니다.", err="permission-denied") + return self._error_response( + "아직 시작하지 않은 대회입니다.", + err="permission-denied", + metric_status="permission_denied", + scope=scope, + ) if not contest.ai_assistant_enabled: - return self._error_response("이 대회에서는 AI 조교를 사용할 수 없습니다.") + return self._error_response( + "이 대회에서는 AI 조교를 사용할 수 없습니다.", + metric_status="disabled", + scope=scope, + ) try: problem = Problem.objects.get(_id=problem_id, contest_id=contest_id, visible=True) except Problem.DoesNotExist: - return self._error_response("문제를 찾을 수 없습니다.") + return self._error_response("문제를 찾을 수 없습니다.", metric_status="problem_not_found", scope=scope) else: try: problem = Problem.objects.get(_id=problem_id, contest_id__isnull=True, visible=True) except Problem.DoesNotExist: - return self._error_response("문제를 찾을 수 없습니다.") + return self._error_response("문제를 찾을 수 없습니다.", metric_status="problem_not_found", scope=scope) hint_log = None previous_hints = [] @@ -225,7 +269,11 @@ def get(self, request): if problem_hint_count >= 5: return self._error_response( - "이 문제에 대한 AI 조교 사용 횟수(5회)를 모두 소진했습니다. 이전 답변을 복습해 보세요.", err="problem-limit-exceeded") + "이 문제에 대한 AI 조교 사용 횟수(5회)를 모두 소진했습니다. 이전 답변을 복습해 보세요.", + err="problem-limit-exceeded", + metric_status="problem_limit_exceeded", + scope=scope, + ) # 이전 힌트 문자열 추출 previous_hints = [log.hint_content for log in previous_logs if log.hint_content.strip()] @@ -244,48 +292,32 @@ def generator(): for chunk in stream_problem_hint(problem, previous_hints=previous_hints, user_code=user_code): full_text_chunks.append(chunk) yield self._format_sse_event("chunk", {"text": chunk}) - full_text = "".join(full_text_chunks) - if full_text.strip(): - hint_log.hint_content = full_text - hint_log.save(update_fields=['hint_content']) - elif hint_log: - hint_log.delete() + if self._persist_hint_log(hint_log, full_text_chunks): + self._record_outcome("success", scope) + else: + self._record_outcome("empty_response", scope) yield self._format_sse_event("done", {"done": True}) except LLMHintError as exc: logger.warning("Failed to stream LLM hint for problem %s: %s", problem_id, exc) - full_text = "".join(full_text_chunks) - if full_text.strip(): - # (1) 일부라도 생성된 텍스트가 있다면 그 시점까지의 내용을 저장 - hint_log.hint_content = full_text - hint_log.save(update_fields=['hint_content']) - elif hint_log: - # (2) 생성된 텍스트가 아예 없다면 선제 생성한 로그 삭제 및 횟수 복구 - hint_log.delete() + self._persist_hint_log(hint_log, full_text_chunks) + self._record_outcome("llm_error", scope) yield from self._error_events("힌트를 생성하지 못했습니다.", err="llm-hint-unavailable") except (GeneratorExit, BrokenPipeError, ConnectionResetError): logger.info("LLM hint stream client disconnected for problem %s", problem_id) - full_text = "".join(full_text_chunks) - if full_text.strip(): - hint_log.hint_content = full_text - hint_log.save(update_fields=['hint_content']) - elif hint_log: - hint_log.delete() + self._persist_hint_log(hint_log, full_text_chunks) + self._record_outcome("client_disconnected", scope) return except Exception: logger.exception("Unexpected error while streaming LLM hint for problem %s", problem_id) - full_text = "".join(full_text_chunks) - if full_text.strip(): - hint_log.hint_content = full_text - hint_log.save(update_fields=['hint_content']) - elif hint_log: - hint_log.delete() + self._persist_hint_log(hint_log, full_text_chunks) + self._record_outcome("unexpected_error", scope) yield from self._error_events("힌트를 생성하지 못했습니다.", err="llm-hint-unavailable") return self._streaming_response(generator()) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 7134eb51..9ce9f437 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -63,6 +63,11 @@ ["status"], buckets=(0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600), ) +AI_HINT_API_OUTCOME_TOTAL = Counter( + "codeplace_ai_hint_api_outcome_total", + "Total user-facing AI hint API outcomes.", + ["status", "scope"], +) class CodePlaceCollector: diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 357dd2d7..2c38c179 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -49,6 +49,7 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_celery_task_count{task_name,status}` - `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_celery_task_last_runtime_seconds{task_name,status}` +- `codeplace_ai_hint_api_outcome_total{status,scope}` - `codeplace_ai_hint_requests_total{status}` - `codeplace_ai_hint_duration_seconds{status}` - `codeplace_postgres_connections` @@ -317,6 +318,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `VLLMRequestLatencyHigh`: prod vLLM p95 e2e latency 60초 초과 10분 지속. - `AIHintBackendErrors`: backend AI hint stream request/parse/internal error 발생 2분 지속. - `AIHintBackendLatencyHigh`: backend AI hint successful stream p95 duration 120초 초과 10분 지속. +- `AIHintAPIFailures`: AI hint 사용자-facing API에서 LLM error, unexpected error, empty response 발생 2분 지속. - `DCGMExporterUnavailable`: DCGM exporter DaemonSet unavailable 2분 지속. - `GPUUtilizationHigh`: GPU utilization 95% 초과 15분 지속. - `GPUMemoryHigh`: GPU framebuffer memory usage 90% 초과 10분 지속. @@ -345,7 +347,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 12. Grafana의 `CodePlace Monitoring Stack` dashboard에서 Prometheus, Alertmanager, Grafana, operator readiness, target scrape failure, rule evaluation failure, notification failure panel을 확인합니다. 13. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. 14. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. -15. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, backend AI hint stream status/duration, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. +15. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, AI hint API outcome, backend AI hint stream status/duration, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. 16. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. 17. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"}` 쿼리가 Kubernetes Warning event를 반환하는지 확인합니다. 18. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 2de67a4b..ccdd47f3 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -928,11 +928,12 @@ kubectl -n code-place-prod describe pvc vllm-hf-cache - e2e latency만 증가하면 prompt/generation token 증가, backend streaming read timeout, vLLM 로그의 model/engine warning을 확인합니다. 현재 vLLM은 prod 전용입니다. dev에서 AI hint를 별도 검증하려면 dev vLLM 배포와 ServiceMonitor 확장이 먼저 필요합니다. -### AIHintBackendErrors / AIHintBackendLatencyHigh +### AIHintAPIFailures / AIHintBackendErrors / AIHintBackendLatencyHigh 확인: ```promql +sum by (namespace, scope, status) (increase(codeplace_ai_hint_api_outcome_total{namespace=""}[5m])) sum by (namespace, status) (increase(codeplace_ai_hint_requests_total{namespace=""}[5m])) histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace="", status="success"}[10m]))) {__name__="vllm:num_requests_waiting", namespace="code-place-prod"} @@ -949,6 +950,8 @@ kubectl -n code-place-prod logs deploy/vllm --tail=200 - `request_error`는 backend에서 vLLM Service까지의 endpoint, network, timeout, HTTP error를 먼저 확인합니다. - `stream_parse_error`는 vLLM OpenAI-compatible stream format과 backend parser 호환성을 확인합니다. +- API outcome의 `permission_denied`, `problem_limit_exceeded`, `disabled`, `problem_not_found`는 시스템 장애보다 사용자/대회 상태로 먼저 봅니다. +- API outcome의 `llm_error`, `unexpected_error`, `empty_response`는 사용자 영향이 있는 실패입니다. - backend duration과 vLLM queue/e2e latency가 같이 높으면 vLLM/GPU 병목으로 봅니다. - backend duration만 높으면 client stream read behavior, backend worker saturation, proxy timeout을 확인합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml index 74b7992d..f0ccc861 100644 --- a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml +++ b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml @@ -208,10 +208,20 @@ data: {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} ] }, + { + "type": "timeseries", + "title": "AI Hint API Outcomes", + "gridPos": {"x": 0, "y": 20, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (namespace, scope, status) (rate(codeplace_ai_hint_api_outcome_total{namespace=~\"code-place-(dev|prod)\"}[5m]))", "legendFormat": "{{namespace}} {{scope}} {{status}}/s"} + ] + }, { "type": "timeseries", "title": "Backend AI Hint Streams", - "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "gridPos": {"x": 8, "y": 20, "w": 8, "h": 8}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "ops"}}, "targets": [ @@ -221,7 +231,7 @@ data: { "type": "timeseries", "title": "Backend AI Hint Duration", - "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "gridPos": {"x": 16, "y": 20, "w": 8, "h": 8}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "s"}}, "targets": [ diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index f4e12177..88d10b37 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -890,6 +890,15 @@ spec: annotations: summary: "Backend AI hint stream latency is high" description: "Backend AI hint successful stream p95 duration has been above 120 seconds for 10 minutes. Check vLLM queue, GPU utilization, and client stream behavior." + - alert: AIHintAPIFailures + expr: sum by (namespace, scope, status) (increase(codeplace_ai_hint_api_outcome_total{namespace=~"code-place-(dev|prod)", status=~"llm_error|unexpected_error|empty_response"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "User-facing AI hint API failures detected" + description: "AI hint API returned LLM errors, unexpected errors, or empty responses in the last 5 minutes. Compare API outcome status with backend stream and vLLM panels." - alert: DCGMExporterUnavailable expr: (kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"} == 0) or (kube_daemonset_status_number_available{namespace="monitoring", daemonset="dcgm-exporter"} < kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"}) or absent(kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"}) for: 2m From aa9520a89d5278c353f1a032a07bab15063807e6 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:44:08 +0900 Subject: [PATCH 051/147] =?UTF-8?q?=EC=A0=9C=EC=B6=9C=20=EC=83=9D=EC=84=B1?= =?UTF-8?q?=20=EA=B2=BD=EB=A1=9C=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/submission/tests.py | 27 ++++++++++ backend/submission/views/oj.py | 43 +++++++++++---- backend/utils/observability_metrics.py | 5 ++ .../observability-improvement-plan/_index.md | 4 +- .../infra/observability-runbook/_index.md | 19 +++++++ .../grafana-dashboard-codeplace.yaml | 53 +++++++++++-------- kubernetes/monitoring/prometheus-rules.yaml | 9 ++++ 7 files changed, 128 insertions(+), 32 deletions(-) diff --git a/backend/submission/tests.py b/backend/submission/tests.py index e0695f7a..d32cf37b 100644 --- a/backend/submission/tests.py +++ b/backend/submission/tests.py @@ -57,6 +57,20 @@ def test_create_submission_success(self): self.assertSuccess(resp) self.assertIn("submission_id", resp.data["data"]) + @mock.patch("submission.views.oj.SUBMISSION_CREATE_OUTCOME_TOTAL") + def test_create_submission_records_success_outcome(self, outcome_total): + data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) + data["problem_id"] = self.problem.id + labels = mock.Mock() + outcome_total.labels.return_value = labels + + with mock.patch('judge.tasks.judge_task.apply_async'): + resp = self.client.post(self.url, data=data) + + self.assertSuccess(resp) + outcome_total.labels.assert_called_once_with(status="success", scope="practice") + labels.inc.assert_called_once() + def test_create_submission_nonexistent_problem(self): data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) data["problem_id"] = 99999 @@ -64,6 +78,19 @@ def test_create_submission_nonexistent_problem(self): resp = self.client.post(self.url, data=data) self.assertFailed(resp, "Problem not exist") + @mock.patch("submission.views.oj.SUBMISSION_CREATE_OUTCOME_TOTAL") + def test_create_submission_records_problem_not_found_outcome(self, outcome_total): + data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) + data["problem_id"] = 99999 + labels = mock.Mock() + outcome_total.labels.return_value = labels + + resp = self.client.post(self.url, data=data) + + self.assertFailed(resp, "Problem not exist") + outcome_total.labels.assert_called_once_with(status="problem_not_found", scope="practice") + labels.inc.assert_called_once() + def test_create_submission_invalid_language(self): data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) data["problem_id"] = self.problem.id diff --git a/backend/submission/views/oj.py b/backend/submission/views/oj.py index 6b1a9f41..a302bac0 100644 --- a/backend/submission/views/oj.py +++ b/backend/submission/views/oj.py @@ -1,4 +1,5 @@ import ipaddress +import logging from django.db.models import Q, Count from django.utils.dateparse import parse_datetime @@ -12,14 +13,21 @@ from utils.api import APIView, validate_serializer from utils.cache import cache from utils.captcha import Captcha +from utils.observability_metrics import SUBMISSION_CREATE_OUTCOME_TOTAL from utils.throttling import TokenBucket from ..models import JudgeStatus, Submission from ..serializers import (CreateSubmissionSerializer, SubmissionModelSerializer, ShareSubmissionSerializer) from ..serializers import SubmissionSafeModelSerializer, SubmissionListSerializer +logger = logging.getLogger(__name__) + class SubmissionAPI(APIView): + @staticmethod + def _record_create_outcome(status, scope): + SUBMISSION_CREATE_OUTCOME_TOTAL.labels(status=status, scope=scope).inc() + def throttling(self, request): # 使用 open_api 的请求暂不做限制 auth_method = getattr(request, "auth_method", "") @@ -53,9 +61,11 @@ def check_contest_permission(self, request): def post(self, request): data = request.data hide_id = False + scope = "contest" if data.get("contest_id") else "practice" if data.get("contest_id"): error = self.check_contest_permission(request) if error: + self._record_create_outcome("contest_permission_denied", scope) return error contest = self.contest if not contest.problem_details_permission(request.user): @@ -63,30 +73,45 @@ def post(self, request): if data.get("captcha"): if not Captcha(request).check(data["captcha"]): + self._record_create_outcome("invalid_captcha", scope) return self.error("Invalid captcha") error = self.throttling(request) if error: + self._record_create_outcome("throttled", scope) return self.error(error) try: problem = Problem.objects.get(id=data["problem_id"], contest_id=data.get("contest_id"), visible=True) except Problem.DoesNotExist: + self._record_create_outcome("problem_not_found", scope) return self.error("Problem not exist") if data["language"] not in problem.languages: + self._record_create_outcome("language_not_allowed", scope) return self.error(f"{data['language']} is now allowed in the problem") - submission = Submission.objects.create( - user_id=request.user.id, - username=request.user.username, - language=data["language"], - code=data["code"], - problem_id=problem.id, - ip=request.session["ip"], - contest_id=data.get("contest_id")) + try: + submission = Submission.objects.create( + user_id=request.user.id, + username=request.user.username, + language=data["language"], + code=data["code"], + problem_id=problem.id, + ip=request.session["ip"], + contest_id=data.get("contest_id")) + except Exception: + self._record_create_outcome("db_error", scope) + logger.exception("Failed to create submission for problem %s", data["problem_id"]) + raise # use this for debug # JudgeDispatcher(submission.id, problem.id).judge() - judge_task.apply_async(args=(submission.id, problem.id)) + try: + judge_task.apply_async(args=(submission.id, problem.id)) + except Exception: + self._record_create_outcome("enqueue_error", scope) + logger.exception("Failed to enqueue judge task for submission %s", submission.id) + raise + self._record_create_outcome("success", scope) if hide_id: return self.success() else: diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 9ce9f437..76fca723 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -68,6 +68,11 @@ "Total user-facing AI hint API outcomes.", ["status", "scope"], ) +SUBMISSION_CREATE_OUTCOME_TOTAL = Counter( + "codeplace_submission_create_outcome_total", + "Total user-facing submission create API outcomes.", + ["status", "scope"], +) class CodePlaceCollector: diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 2c38c179..b70b0fb1 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -42,6 +42,7 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_submission_status_count{status}` - `codeplace_submission_oldest_age_seconds{status}` - `codeplace_submission_judge_duration_seconds` +- `codeplace_submission_create_outcome_total{status,scope}` - `codeplace_waiting_queue_length` - `codeplace_judge_server_available{hostname}` - `codeplace_judge_server_last_heartbeat_age_seconds{hostname}` @@ -267,6 +268,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PrometheusAlertmanagerDiscoveryFailed`: Prometheus가 Alertmanager target을 0개 discovery 2분 지속. - `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. +- `SubmissionCreateSystemFailures`: 제출 생성 API에서 DB 또는 judge enqueue 실패 발생 2분 지속. - `PendingSubmissionsStuck`: oldest pending submission age 300초 초과 3분 지속. - `JudgingSubmissionsStuck`: oldest judging submission age 900초 초과 5분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. @@ -340,7 +342,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`, `codeplace-grafana-http`가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev/Grafana availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission status, oldest in-flight submission age, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, Service ready endpoint, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission status, submission create outcome, oldest in-flight submission age, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, Service ready endpoint, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index ccdd47f3..9b2daec1 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -643,6 +643,25 @@ kubectl -n get pod -l app=judge-server - judging age가 증가하면 judge-server 실행, judge-server `/judge` 호출, result save path, worker 로그의 `judge.tasks.judge_task`를 봅니다. - pending/judging count는 낮지만 oldest age만 높으면 특정 제출이 고립된 상태입니다. 사용자 영향은 제한적일 수 있으나 결과 저장/상태 전환 실패를 확인합니다. +### SubmissionCreateSystemFailures + +확인: + +```promql +sum by (namespace, scope, status) (increase(codeplace_submission_create_outcome_total{namespace=""}[5m])) +``` + +```sh +kubectl -n logs deploy/backend --tail=200 +kubectl -n logs deploy/celery-worker --tail=200 +``` + +판단: + +- `db_error`는 Submission row 생성이 실패한 상태입니다. PostgreSQL readiness, connection usage, lock wait, backend exception log를 봅니다. +- `enqueue_error`는 Submission row는 생성됐지만 judge task queue 등록이 실패한 상태입니다. Redis/Celery broker, worker readiness, queue metric을 먼저 봅니다. +- `throttled`, `invalid_captcha`, `problem_not_found`, `language_not_allowed`, `contest_permission_denied`는 사용자 입력/권한/문제 상태로 보고 시스템 장애 알림 대상에서 제외합니다. + ### CeleryWorkerRestarting / CeleryBeatDown 확인: diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 895b0c21..64f167bd 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -280,17 +280,26 @@ data: {"expr": "topk(10, codeplace_judge_server_task_number{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{hostname}} tasks"} ] }, + { + "type": "timeseries", + "title": "Submission Create Outcomes", + "gridPos": {"x": 0, "y": 28, "w": 24, "h": 8}, + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (namespace, scope, status) (rate(codeplace_submission_create_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{scope}} {{status}}/s"} + ] + }, { "type": "row", "title": "Platform", - "gridPos": {"x": 0, "y": 28, "w": 24, "h": 1}, + "gridPos": {"x": 0, "y": 36, "w": 24, "h": 1}, "collapsed": false, "panels": [] }, { "type": "stat", "title": "Pod Ready", - "gridPos": {"x": 0, "y": 29, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 37, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -311,7 +320,7 @@ data: { "type": "timeseries", "title": "Pod Restarts", - "gridPos": {"x": 6, "y": 29, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 37, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -320,7 +329,7 @@ data: { "type": "timeseries", "title": "Container CPU Usage", - "gridPos": {"x": 12, "y": 29, "w": 6, "h": 5}, + "gridPos": {"x": 12, "y": 37, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "cores"}}, "targets": [ {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} @@ -329,7 +338,7 @@ data: { "type": "timeseries", "title": "Container Memory Usage", - "gridPos": {"x": 18, "y": 29, "w": 6, "h": 5}, + "gridPos": {"x": 18, "y": 37, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} @@ -338,7 +347,7 @@ data: { "type": "timeseries", "title": "PVC Usage", - "gridPos": {"x": 0, "y": 34, "w": 8, "h": 6}, + "gridPos": {"x": 0, "y": 42, "w": 8, "h": 6}, "fieldConfig": { "defaults": { "unit": "percentunit", @@ -359,7 +368,7 @@ data: { "type": "timeseries", "title": "Node Pressure", - "gridPos": {"x": 8, "y": 34, "w": 8, "h": 6}, + "gridPos": {"x": 8, "y": 42, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "kube_node_status_condition{condition=~\"MemoryPressure|DiskPressure|PIDPressure\", status=\"true\"}", "legendFormat": "{{node}} {{condition}}"} @@ -368,7 +377,7 @@ data: { "type": "timeseries", "title": "Deployment Unavailable Replicas", - "gridPos": {"x": 16, "y": 34, "w": 8, "h": 6}, + "gridPos": {"x": 16, "y": 42, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "max by (namespace, deployment) (kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} unavailable"}, @@ -378,7 +387,7 @@ data: { "type": "timeseries", "title": "Service Ready Endpoints", - "gridPos": {"x": 0, "y": 40, "w": 24, "h": 6}, + "gridPos": {"x": 0, "y": 48, "w": 24, "h": 6}, "fieldConfig": { "defaults": { "unit": "short", @@ -398,14 +407,14 @@ data: { "type": "row", "title": "Data Stores", - "gridPos": {"x": 0, "y": 46, "w": 24, "h": 1}, + "gridPos": {"x": 0, "y": 54, "w": 24, "h": 1}, "collapsed": false, "panels": [] }, { "type": "stat", "title": "PostgreSQL Ready", - "gridPos": {"x": 0, "y": 47, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 55, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -426,7 +435,7 @@ data: { "type": "stat", "title": "Redis Ready", - "gridPos": {"x": 6, "y": 47, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 55, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -447,7 +456,7 @@ data: { "type": "stat", "title": "PostgreSQL Exporter Up", - "gridPos": {"x": 12, "y": 47, "w": 6, "h": 5}, + "gridPos": {"x": 12, "y": 55, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -468,7 +477,7 @@ data: { "type": "stat", "title": "Redis Exporter Up", - "gridPos": {"x": 18, "y": 47, "w": 6, "h": 5}, + "gridPos": {"x": 18, "y": 55, "w": 6, "h": 5}, "fieldConfig": { "defaults": { "unit": "short", @@ -489,7 +498,7 @@ data: { "type": "timeseries", "title": "PostgreSQL Restarts", - "gridPos": {"x": 0, "y": 52, "w": 6, "h": 5}, + "gridPos": {"x": 0, "y": 60, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -498,7 +507,7 @@ data: { "type": "timeseries", "title": "Redis Restarts", - "gridPos": {"x": 6, "y": 52, "w": 6, "h": 5}, + "gridPos": {"x": 6, "y": 60, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -507,7 +516,7 @@ data: { "type": "timeseries", "title": "PostgreSQL WAL Bytes", - "gridPos": {"x": 12, "y": 52, "w": 6, "h": 5}, + "gridPos": {"x": 12, "y": 60, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "Bps"}}, "targets": [ {"expr": "sum by (namespace) (rate(cnpg_collector_wal_bytes{namespace=~\"$namespace\", cluster=\"postgres\"}[5m]))", "legendFormat": "{{namespace}}"} @@ -516,7 +525,7 @@ data: { "type": "timeseries", "title": "Redis Memory Usage", - "gridPos": {"x": 18, "y": 52, "w": 6, "h": 5}, + "gridPos": {"x": 18, "y": 60, "w": 6, "h": 5}, "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} used"}, @@ -526,7 +535,7 @@ data: { "type": "timeseries", "title": "PostgreSQL Connections", - "gridPos": {"x": 0, "y": 57, "w": 8, "h": 6}, + "gridPos": {"x": 0, "y": 65, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "codeplace_postgres_connections{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} connections"}, @@ -536,7 +545,7 @@ data: { "type": "timeseries", "title": "PostgreSQL Long Tx and Lock Waits", - "gridPos": {"x": 8, "y": 57, "w": 8, "h": 6}, + "gridPos": {"x": 8, "y": 65, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "codeplace_postgres_long_transactions{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} long tx"}, @@ -546,7 +555,7 @@ data: { "type": "timeseries", "title": "Redis Clients and Rejections", - "gridPos": {"x": 16, "y": 57, "w": 8, "h": 6}, + "gridPos": {"x": 16, "y": 65, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "codeplace_redis_connected_clients{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} clients"}, @@ -557,7 +566,7 @@ data: { "type": "timeseries", "title": "Frontend Runtime Errors", - "gridPos": {"x": 0, "y": 63, "w": 12, "h": 6}, + "gridPos": {"x": 0, "y": 71, "w": 12, "h": 6}, "fieldConfig": { "defaults": { "unit": "ops", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 88d10b37..2673ccc2 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -428,6 +428,15 @@ spec: annotations: summary: "Judging submissions are stuck" description: "The oldest judging submission has been judging for more than 15 minutes. Check judge-server execution, callback/result save path, and worker logs." + - alert: SubmissionCreateSystemFailures + expr: sum by (namespace, scope, status) (increase(codeplace_submission_create_outcome_total{namespace=~"code-place-(dev|prod)", status=~"db_error|enqueue_error"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "Submission create path has system failures" + description: "Submission creation is reporting DB or judge enqueue failures. Check backend logs, PostgreSQL, Redis/Celery broker, and judge task queue." - alert: CeleryWorkerRestarting expr: sum by (namespace) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"celery-worker-.*"}[15m])) > 3 labels: From bc3923854bfe777fc5e8b96879cb537513da9eb6 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:47:12 +0900 Subject: [PATCH 052/147] =?UTF-8?q?=EC=B1=84=EC=A0=90=20=EC=9E=91=EC=97=85?= =?UTF-8?q?=20=EA=B2=B0=EA=B3=BC=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/judge/tasks.py | 39 ++++++++++++++--- backend/judge/tests.py | 43 +++++++++++++++++++ backend/utils/observability_metrics.py | 5 +++ .../observability-improvement-plan/_index.md | 4 +- .../infra/observability-runbook/_index.md | 5 ++- .../grafana-dashboard-codeplace.yaml | 5 ++- kubernetes/monitoring/prometheus-rules.yaml | 9 ++++ 7 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 backend/judge/tests.py diff --git a/backend/judge/tasks.py b/backend/judge/tasks.py index bc58d51c..e8ac3380 100644 --- a/backend/judge/tasks.py +++ b/backend/judge/tasks.py @@ -1,21 +1,50 @@ import celery +import logging from account.models import User from utils.shortcuts import CELERY_TASK_ARGS from utils.observability_tracing import get_tracer +from utils.observability_metrics import JUDGE_TASK_OUTCOME_TOTAL from submission.models import Submission from judge.dispatcher import JudgeDispatcher +logger = logging.getLogger(__name__) tracer = get_tracer(__name__) +def _record_judge_task_outcome(status, scope): + JUDGE_TASK_OUTCOME_TOTAL.labels(status=status, scope=scope).inc() + + @celery.shared_task(**CELERY_TASK_ARGS()) def judge_task(submission_id, problem_id): with tracer.start_as_current_span("judge_task") as span: span.set_attribute("codeplace.submission_id", submission_id) span.set_attribute("codeplace.problem_id", problem_id) - uid = Submission.objects.get(id=submission_id).user_id - if User.objects.get(id=uid).is_disabled: - span.set_attribute("codeplace.judge.skipped", "user_disabled") - return - JudgeDispatcher(submission_id, problem_id).judge() + scope = "unknown" + try: + submission = Submission.objects.get(id=submission_id) + scope = "contest" if submission.contest_id else "practice" + except Submission.DoesNotExist: + _record_judge_task_outcome("submission_missing", scope) + logger.exception("Judge task submission does not exist: %s", submission_id) + raise + + try: + user = User.objects.get(id=submission.user_id) + if user.is_disabled: + span.set_attribute("codeplace.judge.skipped", "user_disabled") + _record_judge_task_outcome("user_disabled", scope) + return + + JudgeDispatcher(submission_id, problem_id).judge() + except User.DoesNotExist: + _record_judge_task_outcome("user_missing", scope) + logger.exception("Judge task user does not exist for submission %s", submission_id) + raise + except Exception: + _record_judge_task_outcome("error", scope) + logger.exception("Judge task failed for submission %s", submission_id) + raise + else: + _record_judge_task_outcome("success", scope) diff --git a/backend/judge/tests.py b/backend/judge/tests.py new file mode 100644 index 00000000..a2ea46f0 --- /dev/null +++ b/backend/judge/tests.py @@ -0,0 +1,43 @@ +from unittest import mock + +from django.test import SimpleTestCase + +from judge.tasks import judge_task + + +class JudgeTaskObservabilityTest(SimpleTestCase): + + @mock.patch("judge.tasks.JUDGE_TASK_OUTCOME_TOTAL") + @mock.patch("judge.tasks.JudgeDispatcher") + @mock.patch("judge.tasks.User") + @mock.patch("judge.tasks.Submission") + def test_judge_task_records_success_outcome(self, submission_model, user_model, dispatcher, outcome_total): + submission = mock.Mock(user_id=1, contest_id=None) + submission_model.objects.get.return_value = submission + user_model.objects.get.return_value = mock.Mock(is_disabled=False) + labels = mock.Mock() + outcome_total.labels.return_value = labels + + judge_task.run(10, 20) + + dispatcher.assert_called_once_with(10, 20) + dispatcher.return_value.judge.assert_called_once() + outcome_total.labels.assert_called_once_with(status="success", scope="practice") + labels.inc.assert_called_once() + + @mock.patch("judge.tasks.JUDGE_TASK_OUTCOME_TOTAL") + @mock.patch("judge.tasks.JudgeDispatcher") + @mock.patch("judge.tasks.User") + @mock.patch("judge.tasks.Submission") + def test_judge_task_records_disabled_user_outcome(self, submission_model, user_model, dispatcher, outcome_total): + submission = mock.Mock(user_id=1, contest_id=7) + submission_model.objects.get.return_value = submission + user_model.objects.get.return_value = mock.Mock(is_disabled=True) + labels = mock.Mock() + outcome_total.labels.return_value = labels + + judge_task.run(10, 20) + + dispatcher.assert_not_called() + outcome_total.labels.assert_called_once_with(status="user_disabled", scope="contest") + labels.inc.assert_called_once() diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 76fca723..48ec11e5 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -73,6 +73,11 @@ "Total user-facing submission create API outcomes.", ["status", "scope"], ) +JUDGE_TASK_OUTCOME_TOTAL = Counter( + "codeplace_judge_task_outcome_total", + "Total Celery judge task outcomes.", + ["status", "scope"], +) class CodePlaceCollector: diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index b70b0fb1..52c94323 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -43,6 +43,7 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_submission_oldest_age_seconds{status}` - `codeplace_submission_judge_duration_seconds` - `codeplace_submission_create_outcome_total{status,scope}` +- `codeplace_judge_task_outcome_total{status,scope}` - `codeplace_waiting_queue_length` - `codeplace_judge_server_available{hostname}` - `codeplace_judge_server_last_heartbeat_age_seconds{hostname}` @@ -269,6 +270,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `SubmissionCreateSystemFailures`: 제출 생성 API에서 DB 또는 judge enqueue 실패 발생 2분 지속. +- `JudgeTaskFailures`: `judge_task`에서 submission/user lookup 또는 dispatch 실패 발생 2분 지속. - `PendingSubmissionsStuck`: oldest pending submission age 300초 초과 3분 지속. - `JudgingSubmissionsStuck`: oldest judging submission age 900초 초과 5분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. @@ -342,7 +344,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`, `codeplace-grafana-http`가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev/Grafana availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission status, submission create outcome, oldest in-flight submission age, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, Service ready endpoint, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission status, submission/judge outcome, oldest in-flight submission age, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, Service ready endpoint, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 9b2daec1..de40ef8a 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -643,12 +643,13 @@ kubectl -n get pod -l app=judge-server - judging age가 증가하면 judge-server 실행, judge-server `/judge` 호출, result save path, worker 로그의 `judge.tasks.judge_task`를 봅니다. - pending/judging count는 낮지만 oldest age만 높으면 특정 제출이 고립된 상태입니다. 사용자 영향은 제한적일 수 있으나 결과 저장/상태 전환 실패를 확인합니다. -### SubmissionCreateSystemFailures +### SubmissionCreateSystemFailures / JudgeTaskFailures 확인: ```promql sum by (namespace, scope, status) (increase(codeplace_submission_create_outcome_total{namespace=""}[5m])) +sum by (namespace, scope, status) (increase(codeplace_judge_task_outcome_total{namespace=""}[5m])) ``` ```sh @@ -660,6 +661,8 @@ kubectl -n logs deploy/celery-worker --tail=200 - `db_error`는 Submission row 생성이 실패한 상태입니다. PostgreSQL readiness, connection usage, lock wait, backend exception log를 봅니다. - `enqueue_error`는 Submission row는 생성됐지만 judge task queue 등록이 실패한 상태입니다. Redis/Celery broker, worker readiness, queue metric을 먼저 봅니다. +- judge task의 `submission_missing`, `user_missing`, `error`는 Celery worker 내부 실패입니다. worker log의 exception, Submission/User row 상태, judge-server 호출, result save path를 봅니다. +- judge task의 `user_disabled`는 비활성 사용자 제출 skip으로 보고 시스템 장애 알림 대상에서 제외합니다. - `throttled`, `invalid_captcha`, `problem_not_found`, `language_not_allowed`, `contest_permission_denied`는 사용자 입력/권한/문제 상태로 보고 시스템 장애 알림 대상에서 제외합니다. ### CeleryWorkerRestarting / CeleryBeatDown diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 64f167bd..52c205a6 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -282,11 +282,12 @@ data: }, { "type": "timeseries", - "title": "Submission Create Outcomes", + "title": "Submission / Judge Outcomes", "gridPos": {"x": 0, "y": 28, "w": 24, "h": 8}, "fieldConfig": {"defaults": {"unit": "ops"}}, "targets": [ - {"expr": "sum by (namespace, scope, status) (rate(codeplace_submission_create_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{scope}} {{status}}/s"} + {"expr": "sum by (namespace, scope, status) (rate(codeplace_submission_create_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} create {{scope}} {{status}}/s"}, + {"expr": "sum by (namespace, scope, status) (rate(codeplace_judge_task_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} judge {{scope}} {{status}}/s"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 2673ccc2..dd99cc33 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -437,6 +437,15 @@ spec: annotations: summary: "Submission create path has system failures" description: "Submission creation is reporting DB or judge enqueue failures. Check backend logs, PostgreSQL, Redis/Celery broker, and judge task queue." + - alert: JudgeTaskFailures + expr: sum by (namespace, scope, status) (increase(codeplace_judge_task_outcome_total{namespace=~"code-place-(dev|prod)", status=~"error|submission_missing|user_missing"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "Judge task failures detected" + description: "Celery judge_task is failing before or during judge dispatch. Check celery-worker logs, submission/user rows, judge-server availability, and result save path." - alert: CeleryWorkerRestarting expr: sum by (namespace) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"celery-worker-.*"}[15m])) > 3 labels: From 6f562940e042aaa42522a58579e4ab07a871e2f4 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:52:06 +0900 Subject: [PATCH 053/147] =?UTF-8?q?=EC=A0=95=EA=B8=B0=20=EC=9E=91=EC=97=85?= =?UTF-8?q?=20=EC=8B=A0=EC=84=A0=EB=8F=84=20=EA=B4=80=EC=B8=A1=EC=84=B1=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/celery_observability.py | 2 ++ backend/utils/observability_metrics.py | 34 +++++++++++++++++++ backend/utils/tests.py | 13 +++++++ .../observability-improvement-plan/_index.md | 5 +++ .../infra/observability-runbook/_index.md | 6 +++- .../grafana-dashboard-codeplace.yaml | 5 +-- kubernetes/monitoring/prometheus-rules.yaml | 27 +++++++++++++++ 7 files changed, 89 insertions(+), 3 deletions(-) diff --git a/backend/utils/celery_observability.py b/backend/utils/celery_observability.py index 1b09ccc0..5f64ec00 100644 --- a/backend/utils/celery_observability.py +++ b/backend/utils/celery_observability.py @@ -13,6 +13,7 @@ CELERY_TASK_RUNTIME_COUNT_KEY = "observability:celery:task_runtime_count" CELERY_TASK_RUNTIME_SUM_KEY = "observability:celery:task_runtime_sum" CELERY_TASK_RUNTIME_LAST_KEY = "observability:celery:task_runtime_last" +CELERY_TASK_LAST_SEEN_AT_KEY = "observability:celery:task_last_seen_at" CELERY_TASK_RUNTIME_BUCKETS = (1, 3, 5, 10, 30, 60, 120, 300, 600, float("inf")) _task_start_times = {} @@ -53,6 +54,7 @@ def _record_task(task_name, status, duration_seconds=None): status = (status or "unknown").lower() try: cache.hincrby(CELERY_TASK_TOTAL_KEY, _field(task_name, status), 1) + _hsetfloat(CELERY_TASK_LAST_SEEN_AT_KEY, _field(task_name, status), time.time()) if duration_seconds is not None: _hincrbyfloat(CELERY_TASK_RUNTIME_SUM_KEY, task_name, duration_seconds) cache.hincrby(CELERY_TASK_RUNTIME_COUNT_KEY, task_name, 1) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 48ec11e5..3c6bfc2c 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -13,6 +13,7 @@ CELERY_TASK_RUNTIME_BUCKET_KEY, CELERY_TASK_RUNTIME_BUCKETS, CELERY_TASK_RUNTIME_COUNT_KEY, + CELERY_TASK_LAST_SEEN_AT_KEY, CELERY_TASK_RUNTIME_LAST_KEY, CELERY_TASK_RUNTIME_SUM_KEY, CELERY_TASK_TOTAL_KEY, @@ -159,6 +160,16 @@ def describe(self): "codeplace_redis_rejected_connections_total", "Total Redis rejected connections from INFO stats.", ) + yield GaugeMetricFamily( + "codeplace_celery_task_last_seen_age_seconds", + "Seconds since each Celery task status was last observed.", + labels=["task_name", "status"], + ) + yield GaugeMetricFamily( + "codeplace_celery_task_last_success_age_seconds", + "Seconds since each Celery task last completed successfully.", + labels=["task_name"], + ) def collect(self): yield from self._submission_status_count() @@ -303,17 +314,30 @@ def _celery_task_metrics(self): "Latest observed Celery task runtime grouped by task name and status.", labels=["task_name", "status"], ) + last_seen_age_metric = GaugeMetricFamily( + "codeplace_celery_task_last_seen_age_seconds", + "Seconds since each Celery task status was last observed.", + labels=["task_name", "status"], + ) + last_success_age_metric = GaugeMetricFamily( + "codeplace_celery_task_last_success_age_seconds", + "Seconds since each Celery task last completed successfully.", + labels=["task_name"], + ) try: task_totals = self._redis_hash(CELERY_TASK_TOTAL_KEY) runtime_buckets = self._redis_hash(CELERY_TASK_RUNTIME_BUCKET_KEY) runtime_counts = self._redis_hash(CELERY_TASK_RUNTIME_COUNT_KEY) runtime_sums = self._redis_hash(CELERY_TASK_RUNTIME_SUM_KEY) last_runtimes = self._redis_hash(CELERY_TASK_RUNTIME_LAST_KEY) + last_seen_at = self._redis_hash(CELERY_TASK_LAST_SEEN_AT_KEY) except Exception as e: logger.warning("Failed to collect Celery task metrics: %s", e) yield total_metric yield runtime_metric yield last_runtime_metric + yield last_seen_age_metric + yield last_success_age_metric return for field, value in task_totals.items(): @@ -340,9 +364,19 @@ def _celery_task_metrics(self): task_name, status = self._split_field(field, 2) last_runtime_metric.add_metric([task_name, status], float(value)) + now_timestamp = timezone.now().timestamp() + for field, value in last_seen_at.items(): + task_name, status = self._split_field(field, 2) + age_seconds = max(now_timestamp - float(value), 0) + last_seen_age_metric.add_metric([task_name, status], age_seconds) + if status == "success": + last_success_age_metric.add_metric([task_name], age_seconds) + yield total_metric yield runtime_metric yield last_runtime_metric + yield last_seen_age_metric + yield last_success_age_metric def _postgres_metrics(self): connections_metric = GaugeMetricFamily( diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 36034be5..8a1f9c82 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -103,6 +103,10 @@ def test_celery_task_metrics_are_collected_from_redis(self): celery_observability.CELERY_TASK_RUNTIME_LAST_KEY: { b"judge.tasks.judge_task|success": b"1.5", }, + celery_observability.CELERY_TASK_LAST_SEEN_AT_KEY: { + b"judge.tasks.judge_task|success": str(timezone.now().timestamp() - 30).encode(), + b"judge.tasks.judge_task|failure": str(timezone.now().timestamp() - 10).encode(), + }, } with patch("utils.observability_metrics.cache.hgetall", side_effect=lambda key: hashes.get(key, {})): @@ -118,6 +122,15 @@ def test_celery_task_metrics_are_collected_from_redis(self): bucket_samples = [sample for sample in runtime_samples if sample.name.endswith("_bucket")] self.assertTrue(any(sample.labels["le"] == "+Inf" and sample.value == 3 for sample in bucket_samples)) self.assertTrue(any(sample.name.endswith("_sum") and sample.value == 4.5 for sample in runtime_samples)) + last_seen_samples = samples["codeplace_celery_task_last_seen_age_seconds"] + self.assertTrue(any( + sample.labels == {"task_name": "judge.tasks.judge_task", "status": "success"} + and sample.value >= 29 + for sample in last_seen_samples + )) + last_success_samples = samples["codeplace_celery_task_last_success_age_seconds"] + self.assertEqual(last_success_samples[0].labels, {"task_name": "judge.tasks.judge_task"}) + self.assertGreaterEqual(last_success_samples[0].value, 29) last_runtime = samples["codeplace_celery_task_last_runtime_seconds"][0] self.assertEqual(last_runtime.value, 1.5) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 52c94323..14ea4161 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -51,6 +51,8 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_celery_task_count{task_name,status}` - `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_celery_task_last_runtime_seconds{task_name,status}` +- `codeplace_celery_task_last_seen_age_seconds{task_name,status}` +- `codeplace_celery_task_last_success_age_seconds{task_name}` - `codeplace_ai_hint_api_outcome_total{status,scope}` - `codeplace_ai_hint_requests_total{status}` - `codeplace_ai_hint_duration_seconds{status}` @@ -278,6 +280,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `CeleryTaskFailures`: task failure 발생 2분 지속. - `CeleryTaskRetries`: task retry 3회 초과/10분, 5분 지속. - `CeleryTaskRuntimeHigh`: task p95 runtime 120초 초과 10분 지속. +- `CeleryMinuteScheduledTaskStale`: 1분 주기 점수 변동 작업 성공 age 10분 초과. +- `CeleryDailyScheduledTaskStale`: 일간 점수 기준 작업 성공 age 26시간 초과. +- `CeleryWeeklyScheduledTaskStale`: 주간 문제 통계/보너스 문제 작업 성공 age 8일 초과. - `PodCrashLooping`: 주요 Pod restart 증가 5분 지속. - `CodePlacePodNotReady`: 주요 앱/DB/Redis Pod not ready 5분 지속. - `KubernetesPodImagePullBackOff`: image pull 실패 2분 지속. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index de40ef8a..6714430a 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -682,7 +682,7 @@ Grafana Explore: {namespace="", container=~"celery-worker|celery-beat"} ``` -### CeleryTaskFailures / CeleryTaskRetries / CeleryTaskRuntimeHigh +### CeleryTaskFailures / CeleryTaskRetries / CeleryTaskRuntimeHigh / ScheduledTaskStale 확인: @@ -690,6 +690,8 @@ Grafana Explore: sum by (task_name, status) (increase(codeplace_celery_task_count{namespace=""}[10m])) histogram_quantile(0.95, sum by (task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=""}[10m]))) codeplace_celery_task_last_runtime_seconds{namespace=""} +codeplace_celery_task_last_seen_age_seconds{namespace=""} +codeplace_celery_task_last_success_age_seconds{namespace=""} ``` Grafana Explore: @@ -705,6 +707,8 @@ Grafana Explore: - 특정 task만 실패하면 최근 코드 변경, 입력 데이터, 외부 의존성을 우선 확인합니다. - runtime만 증가하면 DB/Redis 지연, lock, worker CPU/memory, queue backlog를 같이 봅니다. - `judge.tasks.judge_task` 실패와 `JudgeWaitingQueueBacklog`가 같이 발생하면 judge-server availability와 Redis `waiting_queue`를 우선 확인합니다. +- scheduled task stale은 beat가 task를 발행하지 못했거나 worker가 성공 종료하지 못한 상태입니다. + `calculate_user_score_fluctuation`은 1분 주기라 10분 stale이면 빠르게 확인하고, `calculate_user_score_basis`는 일간, `update_weekly_stats`와 `update_bonus_problem`은 주간 기준으로 봅니다. - task metric은 worker가 Redis에 누적하고 backend `/metrics` collector가 노출합니다. worker 로그에는 `task_id`, `task_name`, `task_status`, `duration_ms`, `trace_id`가 포함됩니다. ### PodCrashLooping / CodePlacePodNotReady diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 52c205a6..a53f1e93 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -224,11 +224,12 @@ data: }, { "type": "timeseries", - "title": "Celery Task Runtime p95", + "title": "Celery Task Runtime / Freshness", "gridPos": {"x": 8, "y": 13, "w": 8, "h": 7}, "fieldConfig": {"defaults": {"unit": "s"}}, "targets": [ - {"expr": "histogram_quantile(0.95, sum by (namespace, task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} {{task_name}} p95"} + {"expr": "histogram_quantile(0.95, sum by (namespace, task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} {{task_name}} p95"}, + {"expr": "codeplace_celery_task_last_success_age_seconds{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{task_name}} last success age"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index dd99cc33..c9bc9757 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -490,6 +490,33 @@ spec: annotations: summary: "Celery task runtime is high" description: "Celery task p95 runtime has been above 120 seconds for 10 minutes." + - alert: CeleryMinuteScheduledTaskStale + expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_fluctuation"} > 600 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Minute scheduled Celery task is stale" + description: "calculate_user_score_fluctuation has not completed successfully for more than 10 minutes. Check celery-beat, celery-worker, Redis broker, and task logs." + - alert: CeleryDailyScheduledTaskStale + expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_basis"} > 93600 + for: 30m + labels: + severity: warning + priority: P1 + annotations: + summary: "Daily scheduled Celery task is stale" + description: "calculate_user_score_basis has not completed successfully for more than 26 hours. Check celery-beat, celery-worker, Redis broker, and task logs." + - alert: CeleryWeeklyScheduledTaskStale + expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name=~"problem.tasks.update_weekly_stats|problem.tasks.update_bonus_problem"} > 691200 + for: 1h + labels: + severity: warning + priority: P1 + annotations: + summary: "Weekly scheduled Celery task is stale" + description: "A weekly problem maintenance task has not completed successfully for more than 8 days. Check celery-beat, celery-worker, Redis broker, and task logs." - alert: PodCrashLooping expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|vllm-.*"}[5m])) > 0 for: 5m From f1792a618f5b190ba5cd2179fd5aba1354551dfc Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:53:38 +0900 Subject: [PATCH 054/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?=EA=B2=80=EC=A6=9D=20=EB=B2=94=EC=9C=84=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/validate.sh | 58 ++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 4f928be3..c93455f0 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -41,7 +41,13 @@ yaml_paths = [ root / "logs" / "loki-values.yaml", root / "logs" / "alloy-values.yaml", root / "grafana-dashboard-codeplace.yaml", + root / "grafana-dashboard-ai-inference.yaml", + root / "grafana-dashboard-kubernetes-events.yaml", root / "grafana-dashboard-logs.yaml", + root / "grafana-dashboard-monitoring-stack.yaml", + root / "grafana-dashboard-public-endpoints.yaml", + root / "grafana-dashboard-storage.yaml", + root / "grafana-dashboard-traces.yaml", ] for path in yaml_paths: @@ -104,6 +110,19 @@ for group in rules["spec"]["groups"]: for alert_name, labels_list in alert_instances.items(): if len(labels_list) > 1 and any("namespace" not in labels for labels in labels_list): raise SystemExit(f"duplicate alert {alert_name} must include static namespace labels") +required_alerts = { + "SubmissionCreateSystemFailures", + "JudgeTaskFailures", + "CeleryMinuteScheduledTaskStale", + "CeleryDailyScheduledTaskStale", + "CeleryWeeklyScheduledTaskStale", + "AIHintAPIFailures", + "AIHintBackendErrors", + "AIHintBackendLatencyHigh", +} +missing_alerts = required_alerts - set(alert_instances) +if missing_alerts: + raise SystemExit(f"missing required observability alerts: {sorted(missing_alerts)}") print("PROMETHEUS RULE SHAPE OK") alertmanager = list(yaml.safe_load_all((root / "alertmanager-config.yaml").read_text()))[0] @@ -203,7 +222,8 @@ if "./alertmanager-config-email.example.yaml" in kustomization.get("resources", raise SystemExit("email fallback example must not be included in default kustomization") print("MONITORING KUSTOMIZATION SHAPE OK") -for dashboard_path in (root / "grafana-dashboard-codeplace.yaml", root / "grafana-dashboard-logs.yaml"): +all_dashboard_paths = sorted(root.glob("grafana-dashboard-*.yaml")) +for dashboard_path in all_dashboard_paths: dashboard_map = yaml.safe_load(dashboard_path.read_text()) for key, value in dashboard_map.get("data", {}).items(): dashboard = json.loads(value) @@ -220,6 +240,42 @@ for dashboard_path in (root / "grafana-dashboard-codeplace.yaml", root / "grafan for target in panel.get("targets", []): if not target.get("expr"): raise SystemExit(f"{dashboard_path}:{key}:{panel.get('title')} has target missing expr") + +dashboard_requirements = { + "grafana-dashboard-codeplace.yaml": { + "Submission / Judge Outcomes": [ + "codeplace_submission_create_outcome_total", + "codeplace_judge_task_outcome_total", + ], + "Celery Task Runtime / Freshness": [ + "codeplace_celery_task_last_success_age_seconds", + ], + }, + "grafana-dashboard-ai-inference.yaml": { + "AI Hint API Outcomes": ["codeplace_ai_hint_api_outcome_total"], + "Backend AI Hint Streams": ["codeplace_ai_hint_requests_total"], + "Backend AI Hint Duration": ["codeplace_ai_hint_duration_seconds_bucket"], + }, +} +dashboard_by_name = {path.name: yaml.safe_load(path.read_text()) for path in all_dashboard_paths} +for dashboard_name, panel_requirements in dashboard_requirements.items(): + dashboard_map = dashboard_by_name.get(dashboard_name) + if not dashboard_map: + raise SystemExit(f"missing dashboard {dashboard_name}") + dashboards = [json.loads(value) for value in dashboard_map.get("data", {}).values()] + panels = { + panel.get("title"): panel + for dashboard in dashboards + for panel in dashboard.get("panels", []) + } + for title, required_expr_parts in panel_requirements.items(): + panel = panels.get(title) + if not panel: + raise SystemExit(f"{dashboard_name} missing panel {title}") + exprs = "\n".join(target.get("expr", "") for target in panel.get("targets", [])) + for expr_part in required_expr_parts: + if expr_part not in exprs: + raise SystemExit(f"{dashboard_name}:{title} missing expr containing {expr_part}") print("GRAFANA DASHBOARD SHAPE OK") PY From c9e816453d3ea2d2c44086ae6946ff0455855b6f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:55:54 +0900 Subject: [PATCH 055/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?=EC=8A=A4=EB=AA=A8=ED=81=AC=20=EC=B2=B4=ED=81=AC=20=EB=B2=94?= =?UTF-8?q?=EC=9C=84=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 1 + kubernetes/monitoring/smoke-check.sh | 56 ++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 14ea4161..aacd6d05 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -221,6 +221,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. - `dcgm-exporter.yaml`: vLLM GPU node용 NVIDIA DCGM exporter DaemonSet/Service/ServiceMonitor. - `validate.sh`: monitoring YAML, Grafana dashboard JSON, kustomize render, 선택적 promtool/Helm render 검증 스크립트. +- `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, OTel/Tempo/Event exporter, optional Loki/Alloy/DCGM 상태를 확인하는 smoke 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. `alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index 7364c5be..66dd589c 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -42,6 +42,7 @@ require_cmd kubectl echo "==> checking Prometheus Operator CRDs" for crd in \ prometheusrules.monitoring.coreos.com \ + probes.monitoring.coreos.com \ servicemonitors.monitoring.coreos.com \ podmonitors.monitoring.coreos.com \ alertmanagerconfigs.monitoring.coreos.com; do @@ -56,11 +57,41 @@ ok "namespace $NAMESPACE" for resource in \ "prometheusrule/codeplace-fast-alerts" \ "servicemonitor/backend" \ + "servicemonitor/blackbox-exporter" \ + "servicemonitor/dcgm-exporter" \ + "servicemonitor/kubernetes-event-exporter" \ + "servicemonitor/longhorn" \ + "servicemonitor/otel-collector" \ + "servicemonitor/tempo" \ + "servicemonitor/vllm" \ "podmonitor/postgres" \ "podmonitor/redis" \ "alertmanagerconfig/codeplace-alert-routing" \ "configmap/grafana-dashboard-codeplace" \ - "configmap/grafana-dashboard-codeplace-logs"; do + "configmap/grafana-dashboard-codeplace-ai-inference" \ + "configmap/grafana-dashboard-codeplace-kubernetes-events" \ + "configmap/grafana-dashboard-codeplace-logs" \ + "configmap/grafana-dashboard-codeplace-monitoring-stack" \ + "configmap/grafana-dashboard-codeplace-public-endpoints" \ + "configmap/grafana-dashboard-codeplace-storage" \ + "configmap/grafana-dashboard-codeplace-traces" \ + "configmap/kubernetes-event-exporter-config" \ + "configmap/otel-collector-config" \ + "configmap/tempo-config" \ + "deployment/blackbox-exporter" \ + "deployment/kubernetes-event-exporter" \ + "deployment/otel-collector" \ + "deployment/tempo" \ + "service/blackbox-exporter" \ + "service/kubernetes-event-exporter" \ + "service/otel-collector" \ + "service/tempo" \ + "persistentvolumeclaim/tempo-data" \ + "probe/codeplace-public-dev-http" \ + "probe/codeplace-public-prod-http" \ + "probe/codeplace-grafana-http" \ + "probe/codeplace-hub-auth-dev-http" \ + "probe/codeplace-hub-auth-prod-http"; do kubectl -n "$NAMESPACE" get "$resource" >/dev/null ok "$resource" done @@ -71,7 +102,13 @@ require_label "$NAMESPACE" podmonitor postgres release kube-prometheus-stack require_label "$NAMESPACE" podmonitor redis release kube-prometheus-stack require_label "$NAMESPACE" alertmanagerconfig codeplace-alert-routing alertmanagerConfig codeplace require_label "$NAMESPACE" configmap grafana-dashboard-codeplace grafana_dashboard 1 +require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-ai-inference grafana_dashboard 1 +require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-kubernetes-events grafana_dashboard 1 require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-logs grafana_dashboard 1 +require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-monitoring-stack grafana_dashboard 1 +require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-public-endpoints grafana_dashboard 1 +require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-storage grafana_dashboard 1 +require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-traces grafana_dashboard 1 if resource_exists "$NAMESPACE" secret alertmanager-contact-points; then kubectl -n "$NAMESPACE" get secret alertmanager-contact-points \ @@ -92,6 +129,23 @@ for selector in \ ok "pods ready for $selector" done +echo "==> checking CodePlace monitoring pods" +for selector in \ + "app=blackbox-exporter" \ + "app.kubernetes.io/name=kubernetes-event-exporter" \ + "app=otel-collector" \ + "app=tempo"; do + kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l "$selector" --timeout=30s + ok "pods ready for $selector" +done + +if kubectl -n "$NAMESPACE" get daemonset dcgm-exporter >/dev/null 2>&1; then + desired="$(kubectl -n "$NAMESPACE" get daemonset dcgm-exporter -o jsonpath='{.status.desiredNumberScheduled}')" + available="$(kubectl -n "$NAMESPACE" get daemonset dcgm-exporter -o jsonpath='{.status.numberAvailable}')" + [ "$desired" = "$available" ] || fail "daemonset/dcgm-exporter available=$available desired=$desired" + ok "daemonset/dcgm-exporter available=$available desired=$desired" +fi + echo "==> checking optional logs stack" if [ "$(resource_count "$NAMESPACE" pod app.kubernetes.io/name=loki)" -gt 0 ]; then kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l app.kubernetes.io/name=loki --timeout=30s From 0d8f27fdb962a713ba805f77ab527f37f5fd8f3e Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 16:58:03 +0900 Subject: [PATCH 056/147] =?UTF-8?q?=EC=95=B1=20=EA=B4=80=EC=B8=A1=EC=84=B1?= =?UTF-8?q?=20=EB=B0=B0=EC=84=A0=20=EA=B2=80=EC=A6=9D=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 2 +- kubernetes/monitoring/validate.sh | 71 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index aacd6d05..fca47418 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -220,7 +220,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `grafana-dashboard-storage.yaml`: Longhorn volume/node/disk 상태와 storage capacity dashboard. - `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. - `dcgm-exporter.yaml`: vLLM GPU node용 NVIDIA DCGM exporter DaemonSet/Service/ServiceMonitor. -- `validate.sh`: monitoring YAML, Grafana dashboard JSON, kustomize render, 선택적 promtool/Helm render 검증 스크립트. +- `validate.sh`: monitoring YAML, Grafana dashboard JSON, monitoring/app kustomize render, app observability env/Service wiring, 선택적 promtool/Helm render 검증 스크립트. - `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, OTel/Tempo/Event exporter, optional Loki/Alloy/DCGM 상태를 확인하는 smoke 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index c93455f0..ff76e4da 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -283,6 +283,77 @@ echo "==> rendering kustomize monitoring manifests" kubectl kustomize "${MONITORING_DIR}" >/tmp/codeplace-monitoring-render.yaml echo "KUSTOMIZE OK kubernetes/monitoring" +echo "==> rendering application overlays for observability wiring" +for overlay in dev prod; do + kubectl kustomize "${ROOT_DIR}/kubernetes/overlays/${overlay}" >"/tmp/codeplace-${overlay}-render.yaml" + echo "KUSTOMIZE OK kubernetes/overlays/${overlay}" +done + +python3 - <<'PY' +from pathlib import Path +import sys + +try: + import yaml +except ImportError: + print("missing Python module yaml; cannot validate rendered application overlays", file=sys.stderr) + sys.exit(1) + + +def docs_for(path): + return [ + doc for doc in yaml.safe_load_all(Path(path).read_text()) + if isinstance(doc, dict) and doc.get("kind") + ] + + +def find_one(docs, kind, name, source): + matches = [ + doc for doc in docs + if doc.get("kind") == kind and doc.get("metadata", {}).get("name") == name + ] + if len(matches) != 1: + raise SystemExit(f"{source} expected one {kind}/{name}, got {len(matches)}") + return matches[0] + + +def env_map(deployment, container_name): + containers = deployment["spec"]["template"]["spec"].get("containers", []) + for container in containers: + if container.get("name") == container_name: + return { + env.get("name"): env.get("value") + for env in container.get("env", []) + if env.get("name") + } + raise SystemExit(f"{deployment['metadata']['name']} missing container {container_name}") + + +for overlay in ("dev", "prod"): + path = f"/tmp/codeplace-{overlay}-render.yaml" + docs = docs_for(path) + backend_service = find_one(docs, "Service", "backend", path) + port_names = {port.get("name") for port in backend_service.get("spec", {}).get("ports", [])} + if "api" not in port_names: + raise SystemExit(f"{overlay} backend Service must expose port name api for ServiceMonitor") + + for deployment_name, container_name in ( + ("backend", "backend"), + ("celery-worker", "celery-worker"), + ("celery-beat", "celery-beat"), + ): + deployment = find_one(docs, "Deployment", deployment_name, path) + env = env_map(deployment, container_name) + if env.get("JSON_LOGGING") != "1": + raise SystemExit(f"{overlay} {deployment_name} must keep JSON_LOGGING=1") + if env.get("OTEL_ENABLED") is None: + raise SystemExit(f"{overlay} {deployment_name} must define OTEL_ENABLED") + if env.get("OTEL_EXPORTER_OTLP_ENDPOINT") != "http://otel-collector.monitoring.svc.cluster.local:4317": + raise SystemExit(f"{overlay} {deployment_name} must send OTLP to monitoring otel-collector") + +print("APPLICATION OBSERVABILITY WIRING OK") +PY + if optional_cmd promtool; then echo "==> checking Prometheus rules with promtool" promtool check rules "${MONITORING_DIR}/prometheus-rules.yaml" From dbcd5c5ea56478ef7cc20b517af8b8249877b1b0 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 17:01:14 +0900 Subject: [PATCH 057/147] =?UTF-8?q?=EC=84=9C=EB=B9=84=EC=8A=A4=20=EC=97=94?= =?UTF-8?q?=EB=93=9C=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=EC=8A=A4=EB=AA=A8?= =?UTF-8?q?=ED=81=AC=20=EC=B2=B4=ED=81=AC=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 2 +- kubernetes/monitoring/smoke-check.sh | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index fca47418..9f6f72f1 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -221,7 +221,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. - `dcgm-exporter.yaml`: vLLM GPU node용 NVIDIA DCGM exporter DaemonSet/Service/ServiceMonitor. - `validate.sh`: monitoring YAML, Grafana dashboard JSON, monitoring/app kustomize render, app observability env/Service wiring, 선택적 promtool/Helm render 검증 스크립트. -- `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, OTel/Tempo/Event exporter, optional Loki/Alloy/DCGM 상태를 확인하는 smoke 검증 스크립트. +- `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, OTel/Tempo/Event exporter, Service endpoint, optional Loki/Alloy/DCGM 상태를 확인하는 smoke 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. `alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index 66dd589c..cb402ccb 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -25,6 +25,20 @@ resource_count() { kubectl -n "$1" get "$2" -l "$3" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | wc -w | tr -d ' ' } +require_service_endpoints() { + local namespace="$1" + local service="$2" + local addresses + addresses="$(kubectl -n "$namespace" get endpoints "$service" -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)" + if [ -z "$addresses" ]; then + addresses="$(kubectl -n "$namespace" get endpointslice \ + -l "kubernetes.io/service-name=$service" \ + -o jsonpath='{.items[*].endpoints[*].addresses[*]}' 2>/dev/null || true)" + fi + [ -n "$addresses" ] || fail "$namespace service/$service has no ready endpoints" + ok "$namespace service/$service endpoints" +} + require_label() { local namespace="$1" local resource="$2" @@ -139,17 +153,29 @@ for selector in \ ok "pods ready for $selector" done +for service in \ + blackbox-exporter \ + kubernetes-event-exporter \ + otel-collector \ + tempo; do + require_service_endpoints "$NAMESPACE" "$service" +done + if kubectl -n "$NAMESPACE" get daemonset dcgm-exporter >/dev/null 2>&1; then desired="$(kubectl -n "$NAMESPACE" get daemonset dcgm-exporter -o jsonpath='{.status.desiredNumberScheduled}')" available="$(kubectl -n "$NAMESPACE" get daemonset dcgm-exporter -o jsonpath='{.status.numberAvailable}')" [ "$desired" = "$available" ] || fail "daemonset/dcgm-exporter available=$available desired=$desired" ok "daemonset/dcgm-exporter available=$available desired=$desired" + require_service_endpoints "$NAMESPACE" dcgm-exporter fi echo "==> checking optional logs stack" if [ "$(resource_count "$NAMESPACE" pod app.kubernetes.io/name=loki)" -gt 0 ]; then kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l app.kubernetes.io/name=loki --timeout=30s ok "loki pods ready" + if resource_exists "$NAMESPACE" service loki-gateway; then + require_service_endpoints "$NAMESPACE" loki-gateway + fi else echo "SKIP loki pods: app.kubernetes.io/name=loki not found" fi @@ -174,6 +200,7 @@ for app_namespace in $APP_NAMESPACES; do ok "$app_namespace service/backend api port" kubectl -n "$app_namespace" wait --for=condition=Ready pod -l app=backend --timeout=30s ok "$app_namespace backend pods ready" + require_service_endpoints "$app_namespace" backend done echo "observability smoke check completed" From e4a199942b4167e5c7192b5e4b250c8ab49733ed Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 17:06:23 +0900 Subject: [PATCH 058/147] =?UTF-8?q?=ED=8A=B8=EB=9E=98=ED=94=BD=20=EC=9D=B8?= =?UTF-8?q?=EA=B7=B8=EB=A0=88=EC=8A=A4=20=EB=A9=94=ED=8A=B8=EB=A6=AD=20?= =?UTF-8?q?=EC=88=98=EC=A7=91=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 3 ++- kubernetes/monitoring/kustomization.yaml | 1 + kubernetes/monitoring/smoke-check.sh | 2 ++ .../monitoring/traefik-service-monitor.yaml | 18 +++++++++++++++ kubernetes/monitoring/validate.sh | 22 +++++++++++++++++++ kubernetes/setup/traefik/traefik-config.yaml | 8 +++++++ 6 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 kubernetes/monitoring/traefik-service-monitor.yaml diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 9f6f72f1..29c30df6 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -199,6 +199,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto 신규 monitoring 리소스는 `kubernetes/monitoring` 아래에 두고, 애플리케이션 `dev/prod` overlay에 포함하지 않습니다. - `backend-service-monitor.yaml`: backend `/metrics` scrape, interval 15s. +- `traefik-service-monitor.yaml`: kube-system Traefik Prometheus metrics scrape, interval 15s. `Ingress5xxSpike`의 `traefik_service_requests_total` 원천입니다. - `blackbox-exporter.yaml`: public endpoint synthetic probe용 Blackbox exporter. - `public-endpoint-probes.yaml`: dev/prod frontend, hub-auth, Grafana 공개 HTTPS URL Probe, interval 30s. - `prometheus-rules.yaml`: P0/P1 fast alert rules와 운영 요약 recording rules. @@ -221,7 +222,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. - `dcgm-exporter.yaml`: vLLM GPU node용 NVIDIA DCGM exporter DaemonSet/Service/ServiceMonitor. - `validate.sh`: monitoring YAML, Grafana dashboard JSON, monitoring/app kustomize render, app observability env/Service wiring, 선택적 promtool/Helm render 검증 스크립트. -- `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, OTel/Tempo/Event exporter, Service endpoint, optional Loki/Alloy/DCGM 상태를 확인하는 smoke 검증 스크립트. +- `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, Traefik metrics scrape 설정, OTel/Tempo/Event exporter, Service endpoint, optional Loki/Alloy/DCGM 상태를 확인하는 smoke 검증 스크립트. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. `alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index 5712e168..b140e15a 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -22,4 +22,5 @@ resources: - ./prometheus-rules.yaml - ./public-endpoint-probes.yaml - ./tempo.yaml + - ./traefik-service-monitor.yaml - ./vllm-service-monitor.yaml diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index cb402ccb..e141b300 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -77,6 +77,7 @@ for resource in \ "servicemonitor/longhorn" \ "servicemonitor/otel-collector" \ "servicemonitor/tempo" \ + "servicemonitor/traefik" \ "servicemonitor/vllm" \ "podmonitor/postgres" \ "podmonitor/redis" \ @@ -112,6 +113,7 @@ done require_label "$NAMESPACE" prometheusrule codeplace-fast-alerts release kube-prometheus-stack require_label "$NAMESPACE" servicemonitor backend release kube-prometheus-stack +require_label "$NAMESPACE" servicemonitor traefik release kube-prometheus-stack require_label "$NAMESPACE" podmonitor postgres release kube-prometheus-stack require_label "$NAMESPACE" podmonitor redis release kube-prometheus-stack require_label "$NAMESPACE" alertmanagerconfig codeplace-alert-routing alertmanagerConfig codeplace diff --git a/kubernetes/monitoring/traefik-service-monitor.yaml b/kubernetes/monitoring/traefik-service-monitor.yaml new file mode 100644 index 00000000..8a43b4bf --- /dev/null +++ b/kubernetes/monitoring/traefik-service-monitor.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: traefik + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - kube-system + selector: + matchLabels: + app.kubernetes.io/name: traefik + endpoints: + - port: metrics + path: /metrics + interval: 15s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index ff76e4da..854673fe 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -37,6 +37,7 @@ yaml_paths = [ root / "alertmanager-config.yaml", root / "alertmanager-config-email.example.yaml", root / "backend-service-monitor.yaml", + root / "traefik-service-monitor.yaml", root / "datastore-pod-monitors.yaml", root / "logs" / "loki-values.yaml", root / "logs" / "alloy-values.yaml", @@ -168,6 +169,15 @@ if dashboard_sidecar.get("label") != "grafana_dashboard": raise SystemExit("Grafana dashboard sidecar must select grafana_dashboard label") print("KUBE-PROMETHEUS-STACK VALUES SHAPE OK") +traefik_config = yaml.safe_load((Path("kubernetes/setup/traefik/traefik-config.yaml")).read_text()) +traefik_values = yaml.safe_load(traefik_config.get("spec", {}).get("valuesContent", "")) +traefik_prometheus = traefik_values.get("metrics", {}).get("prometheus", {}) +if traefik_prometheus.get("addServicesLabels") is not True: + raise SystemExit("Traefik Prometheus metrics must include service labels for ingress 5xx alerts") +if traefik_prometheus.get("service", {}).get("enabled") is not True: + raise SystemExit("Traefik Prometheus metrics service must be enabled for ServiceMonitor scraping") +print("TRAEFIK METRICS SHAPE OK") + loki_values = yaml.safe_load((root / "logs" / "loki-values.yaml").read_text()) loki_config = loki_values.get("loki", {}) if loki_values.get("deploymentMode") != "Monolithic": @@ -207,6 +217,18 @@ if endpoint.get("port") != "api" or endpoint.get("path") != "/metrics" or endpoi if backend_sm.get("spec", {}).get("selector", {}).get("matchLabels", {}).get("app") != "backend": raise SystemExit("backend ServiceMonitor must select app=backend") +traefik_sm = yaml.safe_load((root / "traefik-service-monitor.yaml").read_text()) +if traefik_sm.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": + raise SystemExit("traefik ServiceMonitor must keep release=kube-prometheus-stack label") +traefik_namespaces = traefik_sm.get("spec", {}).get("namespaceSelector", {}).get("matchNames", []) +if "kube-system" not in traefik_namespaces: + raise SystemExit("traefik ServiceMonitor must scrape kube-system") +traefik_endpoint = (traefik_sm.get("spec", {}).get("endpoints") or [{}])[0] +if traefik_endpoint.get("port") != "metrics" or traefik_endpoint.get("path") != "/metrics" or traefik_endpoint.get("interval") != "15s": + raise SystemExit("traefik ServiceMonitor must scrape metrics /metrics every 15s") +if traefik_sm.get("spec", {}).get("selector", {}).get("matchLabels", {}).get("app.kubernetes.io/name") != "traefik": + raise SystemExit("traefik ServiceMonitor must select app.kubernetes.io/name=traefik") + pod_monitors = list(yaml.safe_load_all((root / "datastore-pod-monitors.yaml").read_text())) for monitor in pod_monitors: name = monitor.get("metadata", {}).get("name") diff --git a/kubernetes/setup/traefik/traefik-config.yaml b/kubernetes/setup/traefik/traefik-config.yaml index 15094644..39393221 100644 --- a/kubernetes/setup/traefik/traefik-config.yaml +++ b/kubernetes/setup/traefik/traefik-config.yaml @@ -16,6 +16,14 @@ spec: - "--accesslog.fields.headers.names.X-Request-ID=keep" - "--accesslog.fields.headers.names.X-Forwarded-For=keep" + metrics: + prometheus: + addEntryPointsLabels: true + addRoutersLabels: true + addServicesLabels: true + service: + enabled: true + service: enabled: true type: LoadBalancer From 775507f6faa352d6d17dfc02d48f0f5ba562fbf6 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 17:28:06 +0900 Subject: [PATCH 059/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EB=9F=B0?= =?UTF-8?q?=EB=B6=81=20=EC=95=8C=EB=A6=BC=EB=AA=85=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/developer/infra/observability-runbook/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 6714430a..0759b365 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -682,7 +682,7 @@ Grafana Explore: {namespace="", container=~"celery-worker|celery-beat"} ``` -### CeleryTaskFailures / CeleryTaskRetries / CeleryTaskRuntimeHigh / ScheduledTaskStale +### CeleryTaskFailures / CeleryTaskRetries / CeleryTaskRuntimeHigh / CeleryMinuteScheduledTaskStale / CeleryDailyScheduledTaskStale / CeleryWeeklyScheduledTaskStale 확인: From 1c825f2525e3b8c0f69c25db2ed0e951c2883bff Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 18:20:48 +0900 Subject: [PATCH 060/147] =?UTF-8?q?=EC=85=80=EB=9F=AC=EB=A6=AC=20=EA=B4=80?= =?UTF-8?q?=EC=B8=A1=EC=84=B1=20=EB=A9=94=ED=8A=B8=EB=A6=AD=20=EC=88=98?= =?UTF-8?q?=EC=A7=91=20=EC=95=88=EC=A0=95=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_metrics.py | 26 ++++++++++++++++----- backend/utils/tests.py | 31 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 3c6bfc2c..16b9bba4 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -342,7 +342,7 @@ def _celery_task_metrics(self): for field, value in task_totals.items(): task_name, status = self._split_field(field, 2) - total_metric.add_metric([task_name, status], float(value)) + total_metric.add_metric([task_name, status], self._float_or_zero(value)) task_names = set(runtime_counts) | set(runtime_sums) for field in runtime_buckets: @@ -352,22 +352,26 @@ def _celery_task_metrics(self): cumulative_buckets = [] for bucket in CELERY_TASK_RUNTIME_BUCKETS: bucket_label = "+Inf" if bucket == float("inf") else str(bucket) - value = float(runtime_buckets.get(self._join_field(task_name, bucket_label), 0)) + value = self._float_or_zero(runtime_buckets.get(self._join_field(task_name, bucket_label), 0)) cumulative_buckets.append((bucket_label, value)) runtime_metric.add_metric( [task_name], cumulative_buckets, - float(runtime_sums.get(task_name, 0)), + self._float_or_zero(runtime_sums.get(task_name, 0)), ) for field, value in last_runtimes.items(): task_name, status = self._split_field(field, 2) - last_runtime_metric.add_metric([task_name, status], float(value)) + last_runtime_metric.add_metric([task_name, status], self._float_or_zero(value)) now_timestamp = timezone.now().timestamp() for field, value in last_seen_at.items(): task_name, status = self._split_field(field, 2) - age_seconds = max(now_timestamp - float(value), 0) + seen_timestamp = self._float_or_none(value) + if seen_timestamp is None: + logger.warning("Skipping invalid Celery task last_seen_at metric: %s=%s", field, value) + continue + age_seconds = max(now_timestamp - seen_timestamp, 0) last_seen_age_metric.add_metric([task_name, status], age_seconds) if status == "success": last_success_age_metric.add_metric([task_name], age_seconds) @@ -487,6 +491,18 @@ def _split_field(field, expected_parts): def _join_field(*parts): return "|".join(str(part) for part in parts) + @staticmethod + def _float_or_zero(value): + parsed = CodePlaceCollector._float_or_none(value) + return parsed if parsed is not None else 0.0 + + @staticmethod + def _float_or_none(value): + try: + return float(value) + except (TypeError, ValueError): + return None + def register_codeplace_metrics(): if getattr(register_codeplace_metrics, "_registered", False): diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 8a1f9c82..fdd80de1 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -134,6 +134,37 @@ def test_celery_task_metrics_are_collected_from_redis(self): last_runtime = samples["codeplace_celery_task_last_runtime_seconds"][0] self.assertEqual(last_runtime.value, 1.5) + def test_celery_task_metrics_tolerate_malformed_redis_values(self): + hashes = { + celery_observability.CELERY_TASK_TOTAL_KEY: { + b"judge.tasks.judge_task|success": b"bad-count", + }, + celery_observability.CELERY_TASK_RUNTIME_BUCKET_KEY: { + b"judge.tasks.judge_task|1": b"bad-bucket", + }, + celery_observability.CELERY_TASK_RUNTIME_COUNT_KEY: { + b"judge.tasks.judge_task": b"bad-count", + }, + celery_observability.CELERY_TASK_RUNTIME_SUM_KEY: { + b"judge.tasks.judge_task": b"bad-sum", + }, + celery_observability.CELERY_TASK_RUNTIME_LAST_KEY: { + b"judge.tasks.judge_task|success": b"bad-runtime", + }, + celery_observability.CELERY_TASK_LAST_SEEN_AT_KEY: { + b"judge.tasks.judge_task|success": b"bad-timestamp", + }, + } + + with patch("utils.observability_metrics.cache.hgetall", side_effect=lambda key: hashes.get(key, {})): + samples = self._samples_by_metric(list(CodePlaceCollector()._celery_task_metrics())) + + total_sample = samples["codeplace_celery_task_count"][0] + self.assertEqual(total_sample.value, 0) + last_runtime_sample = samples["codeplace_celery_task_last_runtime_seconds"][0] + self.assertEqual(last_runtime_sample.value, 0) + self.assertEqual(samples["codeplace_celery_task_last_seen_age_seconds"], []) + def test_postgres_health_metrics_are_collected(self): class FakeCursor: def __init__(self): From cc7a39e091deeb77793741dcd6ace49ca8a59b2a Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 18:24:01 +0900 Subject: [PATCH 061/147] =?UTF-8?q?=ED=94=84=EB=A1=9C=EB=A9=94=ED=85=8C?= =?UTF-8?q?=EC=9A=B0=EC=8A=A4=20=EB=A3=B0=20=EA=B2=80=EC=A6=9D=20=EB=B0=A9?= =?UTF-8?q?=EC=8B=9D=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/validate.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 854673fe..3179481e 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -378,7 +378,16 @@ PY if optional_cmd promtool; then echo "==> checking Prometheus rules with promtool" - promtool check rules "${MONITORING_DIR}/prometheus-rules.yaml" + python3 - <<'PY' +from pathlib import Path +import yaml + +prometheus_rule = yaml.safe_load(Path("kubernetes/monitoring/prometheus-rules.yaml").read_text()) +Path("/tmp/codeplace-prometheus-rules-groups.yaml").write_text( + yaml.safe_dump({"groups": prometheus_rule["spec"]["groups"]}, sort_keys=False) +) +PY + promtool check rules /tmp/codeplace-prometheus-rules-groups.yaml else echo "SKIP promtool check: promtool not found" fi From 7ced7ec89f9da0946c3b617ee5f4d85a598511b2 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 18:27:08 +0900 Subject: [PATCH 062/147] =?UTF-8?q?=EC=9D=B8=EA=B7=B8=EB=A0=88=EC=8A=A4=20?= =?UTF-8?q?5xx=20=EC=95=8C=EB=A6=BC=20=EB=9D=BC=EB=B2=A8=20=ED=98=B8?= =?UTF-8?q?=ED=99=98=EC=84=B1=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-runbook/_index.md | 3 +++ kubernetes/monitoring/prometheus-rules.yaml | 20 +++++++++++++++++-- kubernetes/monitoring/validate.sh | 13 ++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 0759b365..838bac54 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -227,7 +227,10 @@ Grafana Explore: 확인: ```promql +codeplace:ingress_5xx_rate2m{namespace=""} +codeplace:ingress_request_rate2m{namespace=""} sum by (exported_service, code) (rate(traefik_service_requests_total{exported_service=~"-.*"}[5m])) +sum by (service, code) (rate(traefik_service_requests_total{service=~"-.*"}[5m])) ``` ```sh diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index c9bc9757..0cf1f2ea 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -17,6 +17,22 @@ spec: expr: codeplace:api_5xx_rate5m / clamp_min(codeplace:api_request_rate5m, 0.001) - record: codeplace:api_p95_latency_seconds5m expr: histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~"code-place-(dev|prod)"}[5m]))) + - record: codeplace:ingress_request_rate2m + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-dev-.*"}[2m])) or sum(rate(traefik_service_requests_total{service=~"code-place-dev-.*"}[2m])) + labels: + namespace: code-place-dev + - record: codeplace:ingress_5xx_rate2m + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-dev-.*", code=~"5.."}[2m])) or sum(rate(traefik_service_requests_total{service=~"code-place-dev-.*", code=~"5.."}[2m])) + labels: + namespace: code-place-dev + - record: codeplace:ingress_request_rate2m + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-prod-.*"}[2m])) or sum(rate(traefik_service_requests_total{service=~"code-place-prod-.*"}[2m])) + labels: + namespace: code-place-prod + - record: codeplace:ingress_5xx_rate2m + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-prod-.*", code=~"5.."}[2m])) or sum(rate(traefik_service_requests_total{service=~"code-place-prod-.*", code=~"5.."}[2m])) + labels: + namespace: code-place-prod - record: codeplace:public_endpoint_availability5m expr: avg by (environment, namespace, service, instance) (avg_over_time(probe_success{probe_type="public-http"}[5m])) - record: codeplace:judge_available_total @@ -56,7 +72,7 @@ spec: summary: "Backend API 5xx rate is high" description: "Backend API 5xx responses are above 5% for 2 minutes." - alert: Ingress5xxSpike - expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-dev-.*", code=~"5.."}[2m])) / clamp_min(sum(rate(traefik_service_requests_total{exported_service=~"code-place-dev-.*"}[2m])), 1) > 0.05 + expr: codeplace:ingress_5xx_rate2m{namespace="code-place-dev"} / clamp_min(codeplace:ingress_request_rate2m{namespace="code-place-dev"}, 1) > 0.05 for: 2m labels: namespace: code-place-dev @@ -66,7 +82,7 @@ spec: summary: "Ingress 5xx rate is high in dev" description: "Traefik 5xx responses for code-place-dev are above 5% for 2 minutes." - alert: Ingress5xxSpike - expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-prod-.*", code=~"5.."}[2m])) / clamp_min(sum(rate(traefik_service_requests_total{exported_service=~"code-place-prod-.*"}[2m])), 1) > 0.05 + expr: codeplace:ingress_5xx_rate2m{namespace="code-place-prod"} / clamp_min(codeplace:ingress_request_rate2m{namespace="code-place-prod"}, 1) > 0.05 for: 2m labels: namespace: code-place-prod diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 3179481e..49b502c4 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -124,6 +124,19 @@ required_alerts = { missing_alerts = required_alerts - set(alert_instances) if missing_alerts: raise SystemExit(f"missing required observability alerts: {sorted(missing_alerts)}") +records = { + rule.get("record") + for group in rules["spec"]["groups"] + for rule in group.get("rules", []) + if rule.get("record") +} +required_records = { + "codeplace:ingress_request_rate2m", + "codeplace:ingress_5xx_rate2m", +} +missing_records = required_records - records +if missing_records: + raise SystemExit(f"missing required observability recording rules: {sorted(missing_records)}") print("PROMETHEUS RULE SHAPE OK") alertmanager = list(yaml.safe_load_all((root / "alertmanager-config.yaml").read_text()))[0] From 388618bd8e599195a4cbb981091bc7d9f14f614f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 18:29:01 +0900 Subject: [PATCH 063/147] =?UTF-8?q?=EA=B0=9C=EC=9A=94=20=EB=8C=80=EC=8B=9C?= =?UTF-8?q?=EB=B3=B4=EB=93=9C=20=EC=9D=B8=EA=B7=B8=EB=A0=88=EC=8A=A4=20?= =?UTF-8?q?=EC=A7=80=ED=91=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/grafana-dashboard-codeplace.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index a53f1e93..4e5d70be 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -60,17 +60,18 @@ data: }, { "type": "stat", - "title": "Request Rate", + "title": "API / Ingress Request Rate", "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, "fieldConfig": {"defaults": {"unit": "reqps"}}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace:api_request_rate5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:api_request_rate5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} api"}, + {"expr": "codeplace:ingress_request_rate2m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} ingress"} ] }, { "type": "stat", - "title": "5xx Ratio", + "title": "API / Ingress 5xx Ratio", "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, "fieldConfig": { "defaults": { @@ -87,7 +88,8 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace:api_5xx_ratio5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:api_5xx_ratio5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} api"}, + {"expr": "codeplace:ingress_5xx_rate2m{namespace=~\"$namespace\"} / clamp_min(codeplace:ingress_request_rate2m{namespace=~\"$namespace\"}, 1)", "legendFormat": "{{namespace}} ingress"} ] }, { From 1671407cec65dfd20258155d610438b65c0d0e19 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 18:45:58 +0900 Subject: [PATCH 064/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=EC=8A=A4=ED=83=9D?= =?UTF-8?q?=20=EC=8A=A4=EB=AA=A8=ED=81=AC=20=EC=B2=B4=ED=81=AC=20=ED=95=84?= =?UTF-8?q?=EC=88=98=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 6 +++--- kubernetes/monitoring/logs/README.md | 7 +++++++ kubernetes/monitoring/smoke-check.sh | 9 ++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 29c30df6..1b8d2181 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -222,7 +222,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. - `dcgm-exporter.yaml`: vLLM GPU node용 NVIDIA DCGM exporter DaemonSet/Service/ServiceMonitor. - `validate.sh`: monitoring YAML, Grafana dashboard JSON, monitoring/app kustomize render, app observability env/Service wiring, 선택적 promtool/Helm render 검증 스크립트. -- `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, Traefik metrics scrape 설정, OTel/Tempo/Event exporter, Service endpoint, optional Loki/Alloy/DCGM 상태를 확인하는 smoke 검증 스크립트. +- `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, Traefik metrics scrape 설정, OTel/Tempo/Event exporter, Service endpoint, Loki/Alloy, optional DCGM 상태를 확인하는 smoke 검증 스크립트. 첫 bootstrap에서 logs stack 설치 전만 `REQUIRE_LOGS_STACK=0`으로 건너뜁니다. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. `alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. @@ -404,9 +404,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - scrape resource shape check: ServiceMonitor/PodMonitor selector label, scrape path/port/interval. - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. - Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh` - - Prometheus Operator CRD, monitoring namespace resource, selector label, webhook Secret, kube-prometheus-stack Pod readiness, optional Loki/Alloy, app namespace backend service/port/readiness를 읽기 전용으로 확인합니다. + - Prometheus Operator CRD, monitoring namespace resource, selector label, webhook Secret, kube-prometheus-stack Pod readiness, Loki/Alloy, app namespace backend service/port/readiness를 읽기 전용으로 확인합니다. -`promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다. +`promtool`이 있는 환경에서는 `validate.sh`가 `PrometheusRule.spec.groups`를 임시 rule file로 추출한 뒤 `promtool check rules`를 실행합니다. `helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다. ### Discord Webhook Secret diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 8d24b1c5..901634e2 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -56,6 +56,13 @@ MONITORING_NAMESPACE=monitoring CODEPLACE_NAMESPACES="code-place-dev" \ bash kubernetes/monitoring/smoke-check.sh ``` +The smoke check treats Loki and Alloy as required by default. During the first +bootstrap before installing the logs stack, explicitly opt out: + +```sh +REQUIRE_LOGS_STACK=0 bash kubernetes/monitoring/smoke-check.sh +``` + ```sh kubectl -n monitoring get pod | grep -E 'loki|alloy|grafana' kubectl -n monitoring get pvc | grep loki diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index e141b300..22cc1f5b 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -3,6 +3,7 @@ set -euo pipefail NAMESPACE="${MONITORING_NAMESPACE:-monitoring}" APP_NAMESPACES="${CODEPLACE_NAMESPACES:-code-place-dev code-place-prod}" +REQUIRE_LOGS_STACK="${REQUIRE_LOGS_STACK:-1}" ok() { echo "OK $*" @@ -171,13 +172,17 @@ if kubectl -n "$NAMESPACE" get daemonset dcgm-exporter >/dev/null 2>&1; then require_service_endpoints "$NAMESPACE" dcgm-exporter fi -echo "==> checking optional logs stack" +echo "==> checking logs stack" if [ "$(resource_count "$NAMESPACE" pod app.kubernetes.io/name=loki)" -gt 0 ]; then kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l app.kubernetes.io/name=loki --timeout=30s ok "loki pods ready" if resource_exists "$NAMESPACE" service loki-gateway; then require_service_endpoints "$NAMESPACE" loki-gateway + else + fail "$NAMESPACE service/loki-gateway is missing" fi +elif [ "$REQUIRE_LOGS_STACK" = "1" ]; then + fail "loki pods are missing; set REQUIRE_LOGS_STACK=0 only for pre-Loki bootstrap checks" else echo "SKIP loki pods: app.kubernetes.io/name=loki not found" fi @@ -187,6 +192,8 @@ if kubectl -n "$NAMESPACE" get daemonset alloy >/dev/null 2>&1; then available="$(kubectl -n "$NAMESPACE" get daemonset alloy -o jsonpath='{.status.numberAvailable}')" [ "$desired" = "$available" ] || fail "daemonset/alloy available=$available desired=$desired" ok "daemonset/alloy available=$available desired=$desired" +elif [ "$REQUIRE_LOGS_STACK" = "1" ]; then + fail "daemonset/alloy is missing; set REQUIRE_LOGS_STACK=0 only for pre-Alloy bootstrap checks" else echo "SKIP daemonset/alloy: not found" fi From 7f6c01d1286219afc7bc859cb3e0bc3a4d0e32c3 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 18:53:07 +0900 Subject: [PATCH 065/147] =?UTF-8?q?=EA=B7=B8=EB=9D=BC=ED=8C=8C=EB=82=98=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EC=86=8C=EC=8A=A4=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EA=B0=95=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/validate.sh | 37 ++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 49b502c4..55aa2a2d 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -175,8 +175,14 @@ if alertmanager_selector.get("alertmanagerConfig") != "codeplace": raise SystemExit("AlertmanagerConfig selector must match alertmanagerConfig=codeplace") grafana_values = stack_values.get("grafana", {}) datasources = grafana_values.get("additionalDataSources") or [] -if not any(ds.get("name") == "Loki" and ds.get("type") == "loki" for ds in datasources): +datasource_by_name = {ds.get("name"): ds for ds in datasources if ds.get("name")} +if not any(ds.get("name") == "Loki" and ds.get("uid") == "Loki" and ds.get("type") == "loki" for ds in datasources): raise SystemExit("Grafana values must provision Loki datasource") +if not any(ds.get("name") == "Tempo" and ds.get("uid") == "Tempo" and ds.get("type") == "tempo" for ds in datasources): + raise SystemExit("Grafana values must provision Tempo datasource") +tempo_datasource = datasource_by_name.get("Tempo", {}) +if tempo_datasource.get("jsonData", {}).get("tracesToLogsV2", {}).get("datasourceUid") != "Loki": + raise SystemExit("Grafana Tempo datasource must link tracesToLogsV2 to Loki") dashboard_sidecar = grafana_values.get("sidecar", {}).get("dashboards", {}) if dashboard_sidecar.get("label") != "grafana_dashboard": raise SystemExit("Grafana dashboard sidecar must select grafana_dashboard label") @@ -258,6 +264,8 @@ if "./alertmanager-config-email.example.yaml" in kustomization.get("resources", print("MONITORING KUSTOMIZATION SHAPE OK") all_dashboard_paths = sorted(root.glob("grafana-dashboard-*.yaml")) +provisioned_datasources = {"Prometheus", *datasource_by_name} +dashboard_datasources = {} for dashboard_path in all_dashboard_paths: dashboard_map = yaml.safe_load(dashboard_path.read_text()) for key, value in dashboard_map.get("data", {}).items(): @@ -269,12 +277,39 @@ for dashboard_path in all_dashboard_paths: panels = dashboard.get("panels") or [] if not panels: raise SystemExit(f"{dashboard_path}:{key} must contain panels") + datasource_names = set() for panel in panels: if not panel.get("title") or not panel.get("type") or not panel.get("gridPos"): raise SystemExit(f"{dashboard_path}:{key} has panel missing title/type/gridPos") + datasource = panel.get("datasource") + if isinstance(datasource, dict): + datasource = datasource.get("uid") or datasource.get("type") + if datasource: + datasource_names.add(datasource) + if datasource not in provisioned_datasources: + raise SystemExit(f"{dashboard_path}:{key}:{panel.get('title')} uses unknown datasource {datasource}") for target in panel.get("targets", []): if not target.get("expr"): raise SystemExit(f"{dashboard_path}:{key}:{panel.get('title')} has target missing expr") + dashboard_datasources[dashboard_path.name] = datasource_names + +required_dashboard_datasources = { + "grafana-dashboard-logs.yaml": {"Prometheus", "Loki"}, + "grafana-dashboard-kubernetes-events.yaml": {"Prometheus", "Loki"}, + "grafana-dashboard-traces.yaml": {"Prometheus"}, + "grafana-dashboard-codeplace.yaml": set(), + "grafana-dashboard-ai-inference.yaml": {"Prometheus"}, + "grafana-dashboard-monitoring-stack.yaml": {"Prometheus"}, + "grafana-dashboard-public-endpoints.yaml": {"Prometheus"}, + "grafana-dashboard-storage.yaml": {"Prometheus"}, +} +for dashboard_name, required_datasources in required_dashboard_datasources.items(): + actual_datasources = dashboard_datasources.get(dashboard_name) + if actual_datasources is None: + raise SystemExit(f"missing dashboard {dashboard_name}") + missing_datasources = required_datasources - actual_datasources + if missing_datasources: + raise SystemExit(f"{dashboard_name} must use datasources {sorted(missing_datasources)}") dashboard_requirements = { "grafana-dashboard-codeplace.yaml": { From 32120d30c1695796343963676b5ddccc657d1049 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 18:57:43 +0900 Subject: [PATCH 066/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?=ED=97=AC=EB=A6=84=20=EC=B0=A8=ED=8A=B8=20=EB=B2=84=EC=A0=84=20?= =?UTF-8?q?=EA=B3=A0=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 5 +++-- kubernetes/monitoring/logs/README.md | 10 ++++++++++ kubernetes/monitoring/validate.sh | 3 +++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 1b8d2181..2afcd30c 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -339,11 +339,12 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. ## 5. 운영 확인 절차 1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. + - `helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --version 86.3.1 --values kubernetes/monitoring/kube-prometheus-stack-values.yaml` 2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. Email fallback을 켤 경우 SMTP 정보를 별도로 정한 뒤 `alertmanager-email` Secret과 email AlertmanagerConfig를 추가 적용합니다. 3. Loki와 Alloy를 설치합니다. - - `helm upgrade --install loki grafana-community/loki --namespace monitoring --values kubernetes/monitoring/logs/loki-values.yaml` - - `helm upgrade --install alloy grafana/alloy --namespace monitoring --values kubernetes/monitoring/logs/alloy-values.yaml` + - `helm upgrade --install loki grafana-community/loki --namespace monitoring --version 17.4.7 --values kubernetes/monitoring/logs/loki-values.yaml` + - `helm upgrade --install alloy grafana/alloy --namespace monitoring --version 1.10.0 --values kubernetes/monitoring/logs/alloy-values.yaml` 4. 애플리케이션은 환경별 overlay로 적용합니다. - dev: `kubectl apply -k kubernetes/overlays/dev` - prod: `kubectl apply -k kubernetes/overlays/prod` diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 901634e2..480b9af8 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -10,6 +10,12 @@ This directory contains Helm values for the Kubernetes logs stack. This is the selected baseline while cloud object storage is unavailable. Longhorn PVC is better than local hostPath or emptyDir because it gives Kubernetes-managed persistence and operationally visible volume health without introducing a new storage service. It is still not a durable log archive and should be treated as short-retention troubleshooting storage. +Pinned chart versions: + +- `prometheus-community/kube-prometheus-stack`: `86.3.1` +- `grafana-community/loki`: `17.4.7` +- `grafana/alloy`: `1.10.0` + ## Install or Upgrade Install or upgrade kube-prometheus-stack first so the Prometheus Operator CRDs are present and the Grafana `Loki` datasource is provisioned. Then install Loki and Alloy. @@ -22,14 +28,17 @@ helm repo update helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ --namespace monitoring \ + --version 86.3.1 \ --values kubernetes/monitoring/kube-prometheus-stack-values.yaml helm upgrade --install loki grafana-community/loki \ --namespace monitoring \ + --version 17.4.7 \ --values kubernetes/monitoring/logs/loki-values.yaml helm upgrade --install alloy grafana/alloy \ --namespace monitoring \ + --version 1.10.0 \ --values kubernetes/monitoring/logs/alloy-values.yaml ``` @@ -90,6 +99,7 @@ This is not a horizontally scalable Loki design. It is the selected cloud-free b Keep these invariants unless the storage design changes intentionally: - Loki stays in `Monolithic` deployment mode. +- Loki is installed from `grafana-community/loki` chart `17.4.7`. Do not switch it to the latest `grafana/loki` chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. - The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. - Retention stays short: dev 72h, prod 168h. diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 55aa2a2d..d875038f 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -444,18 +444,21 @@ if optional_cmd helm; then echo "==> rendering Helm charts" helm template kube-prometheus-stack prometheus-community/kube-prometheus-stack \ --namespace monitoring \ + --version 86.3.1 \ --values "${MONITORING_DIR}/kube-prometheus-stack-values.yaml" \ >/tmp/codeplace-kube-prometheus-stack-render.yaml echo "HELM OK kube-prometheus-stack" helm template loki grafana-community/loki \ --namespace monitoring \ + --version 17.4.7 \ --values "${MONITORING_DIR}/logs/loki-values.yaml" \ >/tmp/codeplace-loki-render.yaml echo "HELM OK loki" helm template alloy grafana/alloy \ --namespace monitoring \ + --version 1.10.0 \ --values "${MONITORING_DIR}/logs/alloy-values.yaml" \ >/tmp/codeplace-alloy-render.yaml echo "HELM OK alloy" From aaf8d5e71d8ef14474724695956e9d8fa7abaa0d Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:01:32 +0900 Subject: [PATCH 067/147] =?UTF-8?q?=EC=95=8C=EB=A6=BC=20=EB=9D=BC=EC=9A=B0?= =?UTF-8?q?=ED=8C=85=20=EB=84=A4=EC=9E=84=EC=8A=A4=ED=8E=98=EC=9D=B4?= =?UTF-8?q?=EC=8A=A4=20=EB=A7=A4=EC=B2=98=20=ED=95=B4=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 1 + .../developer/infra/observability-runbook/_index.md | 2 ++ kubernetes/monitoring/kube-prometheus-stack-values.yaml | 2 ++ kubernetes/monitoring/logs/README.md | 1 + kubernetes/monitoring/validate.sh | 8 ++++++++ 5 files changed, 14 insertions(+) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 2afcd30c..322f1a65 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -340,6 +340,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. - `helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --version 86.3.1 --values kubernetes/monitoring/kube-prometheus-stack-values.yaml` + - values는 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None`을 설정합니다. AlertmanagerConfig는 `monitoring` namespace에 있지만 CodePlace alert의 `namespace` label은 `code-place-dev`/`code-place-prod`이므로, 기본 `OnNamespace` matcher를 쓰면 외부 namespace alert가 Discord receiver까지 도달하지 않을 수 있습니다. 2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. Email fallback을 켤 경우 SMTP 정보를 별도로 정한 뒤 `alertmanager-email` Secret과 email AlertmanagerConfig를 추가 적용합니다. 3. Loki와 Alloy를 설치합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 838bac54..e585d30f 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -49,6 +49,8 @@ Alerts -> alertname / namespace / priority Status -> Configuration -> p0-discord / p1-discord ``` +`kube-prometheus-stack-values.yaml`의 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type`은 `None`이어야 합니다. 기본 `OnNamespace` 전략이면 `monitoring` namespace의 AlertmanagerConfig가 `namespace=code-place-dev` 또는 `namespace=code-place-prod` alert를 라우팅하지 못할 수 있습니다. + Email fallback을 켠 환경에서는 `Status -> Configuration`에서 `p0-email` / `p1-email` receiver도 확인합니다. Monitoring stack 확인: diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index 1503d1f1..3101e792 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -9,6 +9,8 @@ prometheus: alertmanager: alertmanagerSpec: + alertmanagerConfigMatcherStrategy: + type: None alertmanagerConfigSelector: matchLabels: alertmanagerConfig: codeplace diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 480b9af8..cfd33bc8 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -19,6 +19,7 @@ Pinned chart versions: ## Install or Upgrade Install or upgrade kube-prometheus-stack first so the Prometheus Operator CRDs are present and the Grafana `Loki` datasource is provisioned. Then install Loki and Alloy. +The kube-prometheus-stack values set `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None` so the `monitoring` namespace AlertmanagerConfig can route CodePlace alerts whose `namespace` labels are `code-place-dev` or `code-place-prod`. ```sh helm repo add grafana-community https://grafana-community.github.io/helm-charts diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index d875038f..6e9b6199 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -173,6 +173,14 @@ alertmanager_selector = ( ) if alertmanager_selector.get("alertmanagerConfig") != "codeplace": raise SystemExit("AlertmanagerConfig selector must match alertmanagerConfig=codeplace") +alertmanager_matcher_strategy = ( + stack_values.get("alertmanager", {}) + .get("alertmanagerSpec", {}) + .get("alertmanagerConfigMatcherStrategy", {}) + .get("type") +) +if alertmanager_matcher_strategy != "None": + raise SystemExit("AlertmanagerConfig matcher strategy must be None for cross-namespace CodePlace alerts") grafana_values = stack_values.get("grafana", {}) datasources = grafana_values.get("additionalDataSources") or [] datasource_by_name = {ds.get("name"): ds for ds in datasources if ds.get("name")} From 83e866c7c0684e7406660ba690f5cf971bddebd3 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:03:20 +0900 Subject: [PATCH 068/147] =?UTF-8?q?=EC=95=8C=EB=A6=BC=20=EB=A7=A4=EC=B2=98?= =?UTF-8?q?=20=EC=8A=A4=EB=AA=A8=ED=81=AC=20=EC=B2=B4=ED=81=AC=20=EB=B3=B4?= =?UTF-8?q?=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-runbook/_index.md | 1 + kubernetes/monitoring/smoke-check.sh | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index e585d30f..8a358c38 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -59,6 +59,7 @@ Monitoring stack 확인: kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|operator' kubectl -n monitoring get prometheusrule codeplace-fast-alerts -o yaml kubectl -n monitoring get alertmanagerconfig codeplace-alert-routing -o yaml +kubectl -n monitoring get alertmanager kube-prometheus-stack-alertmanager -o jsonpath='{.spec.alertmanagerConfigMatcherStrategy.type}' ``` ```promql diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index 22cc1f5b..dbaa5e10 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -52,6 +52,18 @@ require_label() { ok "$resource/$name label $key=$expected" } +require_jsonpath_value() { + local namespace="$1" + local resource="$2" + local name="$3" + local jsonpath="$4" + local expected="$5" + local actual + actual="$(kubectl -n "$namespace" get "$resource" "$name" -o "jsonpath=$jsonpath")" + [ "$actual" = "$expected" ] || fail "$resource/$name $jsonpath expected=$expected actual=$actual" + ok "$resource/$name $jsonpath=$expected" +} + require_cmd kubectl echo "==> checking Prometheus Operator CRDs" @@ -126,6 +138,8 @@ require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-monitoring-stac require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-public-endpoints grafana_dashboard 1 require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-storage grafana_dashboard 1 require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-traces grafana_dashboard 1 +require_jsonpath_value "$NAMESPACE" alertmanager kube-prometheus-stack-alertmanager \ + '{.spec.alertmanagerConfigMatcherStrategy.type}' None if resource_exists "$NAMESPACE" secret alertmanager-contact-points; then kubectl -n "$NAMESPACE" get secret alertmanager-contact-points \ From 991616769c97d39950342d5d531df9e48b602738 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:05:42 +0900 Subject: [PATCH 069/147] =?UTF-8?q?=EA=B7=B8=EB=9D=BC=ED=8C=8C=EB=82=98=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EC=86=8C=EC=8A=A4=20=EC=8A=A4?= =?UTF-8?q?=EB=AA=A8=ED=81=AC=20=EC=B2=B4=ED=81=AC=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-runbook/_index.md | 1 + kubernetes/monitoring/smoke-check.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 8a358c38..53233449 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -57,6 +57,7 @@ Monitoring stack 확인: ```sh kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|operator' +kubectl -n monitoring get configmap kube-prometheus-stack-grafana-datasource -o yaml kubectl -n monitoring get prometheusrule codeplace-fast-alerts -o yaml kubectl -n monitoring get alertmanagerconfig codeplace-alert-routing -o yaml kubectl -n monitoring get alertmanager kube-prometheus-stack-alertmanager -o jsonpath='{.spec.alertmanagerConfigMatcherStrategy.type}' diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index dbaa5e10..b76938e1 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -64,6 +64,18 @@ require_jsonpath_value() { ok "$resource/$name $jsonpath=$expected" } +require_configmap_data_contains() { + local namespace="$1" + local name="$2" + local key="$3" + local expected="$4" + local value + value="$(kubectl -n "$namespace" get configmap "$name" -o "jsonpath={.data.${key//./\\.}}")" + printf '%s' "$value" | grep -Fq "$expected" \ + || fail "configmap/$name $key missing expected content: $expected" + ok "configmap/$name $key contains $expected" +} + require_cmd kubectl echo "==> checking Prometheus Operator CRDs" @@ -103,6 +115,7 @@ for resource in \ "configmap/grafana-dashboard-codeplace-public-endpoints" \ "configmap/grafana-dashboard-codeplace-storage" \ "configmap/grafana-dashboard-codeplace-traces" \ + "configmap/kube-prometheus-stack-grafana-datasource" \ "configmap/kubernetes-event-exporter-config" \ "configmap/otel-collector-config" \ "configmap/tempo-config" \ @@ -138,6 +151,11 @@ require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-monitoring-stac require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-public-endpoints grafana_dashboard 1 require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-storage grafana_dashboard 1 require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-traces grafana_dashboard 1 +require_label "$NAMESPACE" configmap kube-prometheus-stack-grafana-datasource grafana_datasource 1 +require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'name: Loki' +require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'uid: Loki' +require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'name: Tempo' +require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'uid: Tempo' require_jsonpath_value "$NAMESPACE" alertmanager kube-prometheus-stack-alertmanager \ '{.spec.alertmanagerConfigMatcherStrategy.type}' None From a707e0d8c5a8216b3070f3fd2f0f961d39862019 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:08:50 +0900 Subject: [PATCH 070/147] =?UTF-8?q?Alloy=20=ED=8C=8C=EB=93=9C=20=EB=A1=9C?= =?UTF-8?q?=EA=B7=B8=20=EA=B2=BD=EB=A1=9C=20=EB=A7=88=EC=9A=B4=ED=8A=B8=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/logs/README.md | 3 ++- kubernetes/monitoring/logs/alloy-values.yaml | 2 ++ kubernetes/monitoring/smoke-check.sh | 4 ++++ kubernetes/monitoring/validate.sh | 10 ++++++++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index cfd33bc8..9145e34f 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -3,7 +3,7 @@ This directory contains Helm values for the Kubernetes logs stack. - Loki: monolithic mode, filesystem storage, Longhorn PVC. -- Alloy: DaemonSet log collector for `code-place-dev`, `code-place-prod`, `monitoring`, and Traefik pods in `kube-system`. +- Alloy: DaemonSet log collector for `code-place-dev`, `code-place-prod`, `monitoring`, and Traefik pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. - Kubernetes Event Exporter: Warning events are written to stdout as JSON and collected by Alloy from the `monitoring` namespace. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. - Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. @@ -102,6 +102,7 @@ Keep these invariants unless the storage design changes intentionally: - Loki stays in `Monolithic` deployment mode. - Loki is installed from `grafana-community/loki` chart `17.4.7`. Do not switch it to the latest `grafana/loki` chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. +- Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. - The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. - Retention stays short: dev 72h, prod 168h. - The Loki chart's MinIO subchart stays disabled. If object storage becomes available, use a separately operated S3-compatible service and migrate Loki storage deliberately. diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml index 93644695..8a7240f3 100644 --- a/kubernetes/monitoring/logs/alloy-values.yaml +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -5,6 +5,8 @@ controller: type: daemonset alloy: + mounts: + varlog: true extraEnv: - name: NODE_NAME valueFrom: diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index b76938e1..db4539be 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -224,6 +224,10 @@ if kubectl -n "$NAMESPACE" get daemonset alloy >/dev/null 2>&1; then available="$(kubectl -n "$NAMESPACE" get daemonset alloy -o jsonpath='{.status.numberAvailable}')" [ "$desired" = "$available" ] || fail "daemonset/alloy available=$available desired=$desired" ok "daemonset/alloy available=$available desired=$desired" + require_jsonpath_value "$NAMESPACE" daemonset alloy \ + '{.spec.template.spec.volumes[?(@.name=="varlog")].hostPath.path}' /var/log + require_jsonpath_value "$NAMESPACE" daemonset alloy \ + '{.spec.template.spec.containers[?(@.name=="alloy")].volumeMounts[?(@.name=="varlog")].mountPath}' /var/log elif [ "$REQUIRE_LOGS_STACK" = "1" ]; then fail "daemonset/alloy is missing; set REQUIRE_LOGS_STACK=0 only for pre-Alloy bootstrap checks" else diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 6e9b6199..52efa382 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -235,6 +235,16 @@ if loki_values.get("minio", {}).get("enabled") is not False: raise SystemExit("Loki chart MinIO subchart must stay disabled") print("LOKI ON-PREM STORAGE SHAPE OK") +alloy_values = yaml.safe_load((root / "logs" / "alloy-values.yaml").read_text()) +if alloy_values.get("alloy", {}).get("mounts", {}).get("varlog") is not True: + raise SystemExit("Alloy must mount host /var/log for /var/log/pods log collection") +alloy_config = alloy_values.get("alloy", {}).get("configMap", {}).get("content", "") +if "/var/log/pods/" not in alloy_config: + raise SystemExit("Alloy config must read Kubernetes pod logs from /var/log/pods") +if "loki-gateway.monitoring.svc.cluster.local" not in alloy_config: + raise SystemExit("Alloy config must write to the in-cluster Loki gateway") +print("ALLOY LOG COLLECTION SHAPE OK") + backend_sm = yaml.safe_load((root / "backend-service-monitor.yaml").read_text()) if backend_sm.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": raise SystemExit("backend ServiceMonitor must keep release=kube-prometheus-stack label") From 9fe84903828d47c05a6f3a83f17ffd6ae9a6d8f1 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:11:30 +0900 Subject: [PATCH 071/147] =?UTF-8?q?=ED=94=84=EB=A1=A0=ED=8A=B8=EC=97=94?= =?UTF-8?q?=EB=93=9C=20nginx=20=EB=A1=9C=EA=B7=B8=20=EA=B2=80=EC=A6=9D=20?= =?UTF-8?q?=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/validate.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 52efa382..7f2ce8f1 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -245,6 +245,27 @@ if "loki-gateway.monitoring.svc.cluster.local" not in alloy_config: raise SystemExit("Alloy config must write to the in-cluster Loki gateway") print("ALLOY LOG COLLECTION SHAPE OK") +frontend_nginx = Path("frontend/deploy/kube_nginx/nginx.conf").read_text() +frontend_proxy = Path("frontend/deploy/kube_nginx/api_proxy.conf").read_text() +frontend_entrypoint = Path("frontend/deploy/entrypoint.sh").read_text() +required_frontend_nginx_parts = [ + "error_log /dev/stderr", + "log_format main escape=json", + "\"logger\":\"frontend.nginx\"", + "\"request_id\":\"$codeplace_request_id\"", + "\"status_code\":$status", + "access_log /dev/stdout main", + "add_header X-Request-ID $codeplace_request_id always", +] +for text in required_frontend_nginx_parts: + if text not in frontend_nginx: + raise SystemExit(f"frontend Kubernetes nginx config missing {text}") +if "proxy_set_header X-Request-ID $codeplace_request_id;" not in frontend_proxy: + raise SystemExit("frontend Kubernetes nginx proxy must propagate X-Request-ID") +if "exec nginx -c /app/deploy/kube_nginx/nginx.conf" not in frontend_entrypoint: + raise SystemExit("frontend entrypoint must use Kubernetes nginx config") +print("FRONTEND NGINX LOGGING SHAPE OK") + backend_sm = yaml.safe_load((root / "backend-service-monitor.yaml").read_text()) if backend_sm.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": raise SystemExit("backend ServiceMonitor must keep release=kube-prometheus-stack label") @@ -425,6 +446,11 @@ for overlay in ("dev", "prod"): if "api" not in port_names: raise SystemExit(f"{overlay} backend Service must expose port name api for ServiceMonitor") + frontend_deployment = find_one(docs, "Deployment", "frontend", path) + frontend_env = env_map(frontend_deployment, "frontend") + if frontend_env.get("FORCE_HTTPS") != "0": + raise SystemExit(f"{overlay} frontend must keep FORCE_HTTPS=0 because Traefik terminates TLS") + for deployment_name, container_name in ( ("backend", "backend"), ("celery-worker", "celery-worker"), From 0cf92d472c0cb4cc2299c60e006932fadd118f56 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:13:54 +0900 Subject: [PATCH 072/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?selector=20=EA=B2=80=EC=A6=9D=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/validate.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 7f2ce8f1..dd47a242 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -438,19 +438,49 @@ def env_map(deployment, container_name): raise SystemExit(f"{deployment['metadata']['name']} missing container {container_name}") +def require_labels(name, actual, expected): + for key, value in expected.items(): + if actual.get(key) != value: + raise SystemExit(f"{name} label {key} expected={value} actual={actual.get(key)}") + + +def require_service_targets_deployment(service, deployment, source): + service_name = service["metadata"]["name"] + service_selector = service.get("spec", {}).get("selector", {}) + pod_labels = deployment.get("spec", {}).get("template", {}).get("metadata", {}).get("labels", {}) + require_labels(f"{source} Service/{service_name} selector target", pod_labels, service_selector) + + +backend_service_monitor = yaml.safe_load(Path("kubernetes/monitoring/backend-service-monitor.yaml").read_text()) +backend_service_monitor_selector = backend_service_monitor.get("spec", {}).get("selector", {}).get("matchLabels", {}) +vllm_service_monitor = yaml.safe_load(Path("kubernetes/monitoring/vllm-service-monitor.yaml").read_text()) +vllm_service_monitor_selector = vllm_service_monitor.get("spec", {}).get("selector", {}).get("matchLabels", {}) + + for overlay in ("dev", "prod"): path = f"/tmp/codeplace-{overlay}-render.yaml" docs = docs_for(path) backend_service = find_one(docs, "Service", "backend", path) + backend_deployment = find_one(docs, "Deployment", "backend", path) port_names = {port.get("name") for port in backend_service.get("spec", {}).get("ports", [])} if "api" not in port_names: raise SystemExit(f"{overlay} backend Service must expose port name api for ServiceMonitor") + require_labels(f"{overlay} backend ServiceMonitor selector", backend_service.get("metadata", {}).get("labels", {}), backend_service_monitor_selector) + require_service_targets_deployment(backend_service, backend_deployment, overlay) frontend_deployment = find_one(docs, "Deployment", "frontend", path) + frontend_service = find_one(docs, "Service", "frontend", path) + require_service_targets_deployment(frontend_service, frontend_deployment, overlay) frontend_env = env_map(frontend_deployment, "frontend") if frontend_env.get("FORCE_HTTPS") != "0": raise SystemExit(f"{overlay} frontend must keep FORCE_HTTPS=0 because Traefik terminates TLS") + if overlay == "prod": + vllm_service = find_one(docs, "Service", "vllm", path) + vllm_deployment = find_one(docs, "Deployment", "vllm", path) + require_labels(f"{overlay} vLLM ServiceMonitor selector", vllm_service.get("metadata", {}).get("labels", {}), vllm_service_monitor_selector) + require_service_targets_deployment(vllm_service, vllm_deployment, overlay) + for deployment_name, container_name in ( ("backend", "backend"), ("celery-worker", "celery-worker"), From e9410caa2717bc274ee8bab5c32cabf1129c8ceb Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:18:25 +0900 Subject: [PATCH 073/147] =?UTF-8?q?Loki=20=EC=B0=A8=ED=8A=B8=20=EB=B0=B0?= =?UTF-8?q?=ED=8F=AC=20=EB=AA=A8=EB=93=9C=20=EC=A0=95=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/logs/README.md | 11 +++++------ kubernetes/monitoring/logs/loki-values.yaml | 6 +++--- kubernetes/monitoring/validate.sh | 9 +++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 9145e34f..4e1cd0ef 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -13,7 +13,7 @@ This is the selected baseline while cloud object storage is unavailable. Longhor Pinned chart versions: - `prometheus-community/kube-prometheus-stack`: `86.3.1` -- `grafana-community/loki`: `17.4.7` +- `grafana/loki`: `6.55.0` - `grafana/alloy`: `1.10.0` ## Install or Upgrade @@ -22,7 +22,6 @@ Install or upgrade kube-prometheus-stack first so the Prometheus Operator CRDs a The kube-prometheus-stack values set `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None` so the `monitoring` namespace AlertmanagerConfig can route CodePlace alerts whose `namespace` labels are `code-place-dev` or `code-place-prod`. ```sh -helm repo add grafana-community https://grafana-community.github.io/helm-charts helm repo add grafana https://grafana.github.io/helm-charts helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update @@ -32,9 +31,9 @@ helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheu --version 86.3.1 \ --values kubernetes/monitoring/kube-prometheus-stack-values.yaml -helm upgrade --install loki grafana-community/loki \ +helm upgrade --install loki grafana/loki \ --namespace monitoring \ - --version 17.4.7 \ + --version 6.55.0 \ --values kubernetes/monitoring/logs/loki-values.yaml helm upgrade --install alloy grafana/alloy \ @@ -99,8 +98,8 @@ This is not a horizontally scalable Loki design. It is the selected cloud-free b Keep these invariants unless the storage design changes intentionally: -- Loki stays in `Monolithic` deployment mode. -- Loki is installed from `grafana-community/loki` chart `17.4.7`. Do not switch it to the latest `grafana/loki` chart without revalidating rendered workloads and values compatibility. +- Loki stays in `SingleBinary` deployment mode. +- Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. - Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. - The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. diff --git a/kubernetes/monitoring/logs/loki-values.yaml b/kubernetes/monitoring/logs/loki-values.yaml index a98e4817..28c1678f 100644 --- a/kubernetes/monitoring/logs/loki-values.yaml +++ b/kubernetes/monitoring/logs/loki-values.yaml @@ -1,6 +1,6 @@ -# Helm values for grafana-community/loki. +# Helm values for grafana/loki. # This is an on-premises, cloud-free Loki deployment. It intentionally uses -# Monolithic mode with a Longhorn-backed filesystem PVC instead of object storage. +# SingleBinary mode with a Longhorn-backed filesystem PVC instead of object storage. loki: auth_enabled: false @@ -45,7 +45,7 @@ loki: ruler: enable_api: true -deploymentMode: Monolithic +deploymentMode: SingleBinary singleBinary: replicas: 1 diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index dd47a242..12ad9a61 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -161,6 +161,7 @@ prometheus_spec = stack_values.get("prometheus", {}).get("prometheusSpec", {}) for key in ( "serviceMonitorSelectorNilUsesHelmValues", "podMonitorSelectorNilUsesHelmValues", + "probeSelectorNilUsesHelmValues", "ruleSelectorNilUsesHelmValues", ): if prometheus_spec.get(key) is not False: @@ -207,8 +208,8 @@ print("TRAEFIK METRICS SHAPE OK") loki_values = yaml.safe_load((root / "logs" / "loki-values.yaml").read_text()) loki_config = loki_values.get("loki", {}) -if loki_values.get("deploymentMode") != "Monolithic": - raise SystemExit("Loki must stay in Monolithic mode for the on-prem baseline") +if loki_values.get("deploymentMode") != "SingleBinary": + raise SystemExit("Loki must stay in SingleBinary mode for the on-prem baseline") if loki_config.get("storage", {}).get("type") != "filesystem": raise SystemExit("Loki must use filesystem storage while cloud object storage is unavailable") if loki_config.get("commonConfig", {}).get("replication_factor") != 1: @@ -523,9 +524,9 @@ if optional_cmd helm; then >/tmp/codeplace-kube-prometheus-stack-render.yaml echo "HELM OK kube-prometheus-stack" - helm template loki grafana-community/loki \ + helm template loki grafana/loki \ --namespace monitoring \ - --version 17.4.7 \ + --version 6.55.0 \ --values "${MONITORING_DIR}/logs/loki-values.yaml" \ >/tmp/codeplace-loki-render.yaml echo "HELM OK loki" From 95adf0b289262c8462ee69bc3915530584d90a1b Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:20:46 +0900 Subject: [PATCH 074/147] =?UTF-8?q?Alloy=20=EB=A1=9C=EA=B7=B8=20=EC=88=98?= =?UTF-8?q?=EC=A7=91=20=EB=B2=94=EC=9C=84=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/logs/README.md | 5 +++-- kubernetes/monitoring/logs/alloy-values.yaml | 4 ++-- kubernetes/monitoring/validate.sh | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 4e1cd0ef..33810acf 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -2,8 +2,8 @@ This directory contains Helm values for the Kubernetes logs stack. -- Loki: monolithic mode, filesystem storage, Longhorn PVC. -- Alloy: DaemonSet log collector for `code-place-dev`, `code-place-prod`, `monitoring`, and Traefik pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. +- Loki: SingleBinary mode, filesystem storage, Longhorn PVC. +- Alloy: DaemonSet log collector for every pod in `code-place-dev`, `code-place-prod`, and `monitoring`, plus Traefik pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. - Kubernetes Event Exporter: Warning events are written to stdout as JSON and collected by Alloy from the `monitoring` namespace. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. - Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. @@ -101,6 +101,7 @@ Keep these invariants unless the storage design changes intentionally: - Loki stays in `SingleBinary` deployment mode. - Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. +- Alloy keeps namespace-based collection for `code-place-dev`, `code-place-prod`, and `monitoring`; do not depend on `app.kubernetes.io/name` for CodePlace app logs because the application manifests primarily use `app`. - Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. - The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. - Retention stays short: dev 72h, prod 168h. diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml index 8a7240f3..2007bf8b 100644 --- a/kubernetes/monitoring/logs/alloy-values.yaml +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -28,9 +28,9 @@ alloy: targets = discovery.kubernetes.pod.targets rule { - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name"] + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app", "__meta_kubernetes_pod_label_app_kubernetes_io_name"] separator = "/" - regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/traefik" + regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik" action = "keep" } diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 12ad9a61..10d00aa6 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -244,6 +244,10 @@ if "/var/log/pods/" not in alloy_config: raise SystemExit("Alloy config must read Kubernetes pod logs from /var/log/pods") if "loki-gateway.monitoring.svc.cluster.local" not in alloy_config: raise SystemExit("Alloy config must write to the in-cluster Loki gateway") +if '"__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app", "__meta_kubernetes_pod_label_app_kubernetes_io_name"' not in alloy_config: + raise SystemExit("Alloy keep rule must consider namespace, app, and app.kubernetes.io/name labels") +if "regex = \"(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik\"" not in alloy_config: + raise SystemExit("Alloy keep rule must collect CodePlace/monitoring namespaces and kube-system Traefik logs") print("ALLOY LOG COLLECTION SHAPE OK") frontend_nginx = Path("frontend/deploy/kube_nginx/nginx.conf").read_text() From d33b003bb851cb96cafd13a91e25feefc18728b4 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:22:52 +0900 Subject: [PATCH 075/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=EC=8A=A4=ED=83=9D?= =?UTF-8?q?=20=EC=8A=A4=EB=AA=A8=ED=81=AC=20=EC=B2=B4=ED=81=AC=20=EB=B3=B4?= =?UTF-8?q?=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/smoke-check.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index db4539be..a17fa549 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -206,6 +206,12 @@ fi echo "==> checking logs stack" if [ "$(resource_count "$NAMESPACE" pod app.kubernetes.io/name=loki)" -gt 0 ]; then + kubectl -n "$NAMESPACE" get statefulset loki >/dev/null + ok "statefulset/loki" + require_jsonpath_value "$NAMESPACE" statefulset loki \ + '{.spec.volumeClaimTemplates[?(@.metadata.name=="storage")].spec.storageClassName}' longhorn + require_jsonpath_value "$NAMESPACE" statefulset loki \ + '{.spec.volumeClaimTemplates[?(@.metadata.name=="storage")].spec.resources.requests.storage}' 50Gi kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l app.kubernetes.io/name=loki --timeout=30s ok "loki pods ready" if resource_exists "$NAMESPACE" service loki-gateway; then @@ -213,6 +219,9 @@ if [ "$(resource_count "$NAMESPACE" pod app.kubernetes.io/name=loki)" -gt 0 ]; t else fail "$NAMESPACE service/loki-gateway is missing" fi + kubectl -n "$NAMESPACE" get persistentvolumeclaim storage-loki-0 >/dev/null + ok "persistentvolumeclaim/storage-loki-0" + require_label "$NAMESPACE" servicemonitor loki release kube-prometheus-stack elif [ "$REQUIRE_LOGS_STACK" = "1" ]; then fail "loki pods are missing; set REQUIRE_LOGS_STACK=0 only for pre-Loki bootstrap checks" else @@ -228,6 +237,8 @@ if kubectl -n "$NAMESPACE" get daemonset alloy >/dev/null 2>&1; then '{.spec.template.spec.volumes[?(@.name=="varlog")].hostPath.path}' /var/log require_jsonpath_value "$NAMESPACE" daemonset alloy \ '{.spec.template.spec.containers[?(@.name=="alloy")].volumeMounts[?(@.name=="varlog")].mountPath}' /var/log + require_label "$NAMESPACE" servicemonitor alloy release kube-prometheus-stack + require_configmap_data_contains "$NAMESPACE" alloy config.alloy 'regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik"' elif [ "$REQUIRE_LOGS_STACK" = "1" ]; then fail "daemonset/alloy is missing; set REQUIRE_LOGS_STACK=0 only for pre-Alloy bootstrap checks" else From fcef10933d5818e65ab262a445d25f6055579980 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:26:10 +0900 Subject: [PATCH 076/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=EB=8C=80=EC=8B=9C?= =?UTF-8?q?=EB=B3=B4=EB=93=9C=20=EB=84=A4=EC=9E=84=EC=8A=A4=ED=8E=98?= =?UTF-8?q?=EC=9D=B4=EC=8A=A4=20=EB=B2=94=EC=9C=84=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/grafana-dashboard-logs.yaml | 2 +- kubernetes/monitoring/validate.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index 2946b09f..ab6be07e 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -26,7 +26,7 @@ data: "query": "code-place-dev,code-place-prod,monitoring", "multi": true, "includeAll": true, - "allValue": "code-place-dev|code-place-prod", + "allValue": "code-place-dev|code-place-prod|monitoring", "current": { "text": "code-place-dev", "value": "code-place-dev" diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 10d00aa6..6e3553f9 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -372,6 +372,19 @@ dashboard_requirements = { }, } dashboard_by_name = {path.name: yaml.safe_load(path.read_text()) for path in all_dashboard_paths} +logs_dashboard = [ + json.loads(value) + for value in dashboard_by_name["grafana-dashboard-logs.yaml"].get("data", {}).values() +][0] +logs_namespace_variables = [ + variable + for variable in logs_dashboard.get("templating", {}).get("list", []) + if variable.get("name") == "namespace" +] +if len(logs_namespace_variables) != 1: + raise SystemExit("grafana-dashboard-logs.yaml must define exactly one namespace variable") +if logs_namespace_variables[0].get("allValue") != "code-place-dev|code-place-prod|monitoring": + raise SystemExit("CodePlace Logs dashboard namespace All value must include monitoring logs") for dashboard_name, panel_requirements in dashboard_requirements.items(): dashboard_map = dashboard_by_name.get(dashboard_name) if not dashboard_map: From 6e084197944df4d5a75c1d0c1667366d7252ef82 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:28:47 +0900 Subject: [PATCH 077/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=EB=8C=80=EC=8B=9C?= =?UTF-8?q?=EB=B3=B4=EB=93=9C=20=EB=B9=88=20=ED=8C=A8=EB=84=90=20=EB=B0=A9?= =?UTF-8?q?=EC=A7=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/grafana-dashboard-logs.yaml | 8 ++++---- kubernetes/monitoring/validate.sh | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index ab6be07e..5a9a5d61 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -63,7 +63,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-[0-9]+\"})", "legendFormat": "loki"} + {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-[0-9]+\"}) or vector(0)", "legendFormat": "loki"} ] }, { @@ -74,8 +74,8 @@ data: "fieldConfig": {"defaults": {"unit": "short"}}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "kube_daemonset_status_number_available{namespace=\"monitoring\", daemonset=\"alloy\"}", "legendFormat": "available"}, - {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"monitoring\", daemonset=\"alloy\"}", "legendFormat": "desired"} + {"expr": "kube_daemonset_status_number_available{namespace=\"monitoring\", daemonset=\"alloy\"} or vector(0)", "legendFormat": "available"}, + {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"monitoring\", daemonset=\"alloy\"} or vector(0)", "legendFormat": "desired"} ] }, { @@ -98,7 +98,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}), 1)", "legendFormat": "usage"} + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}), 1) or vector(0)", "legendFormat": "usage"} ] }, { diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 6e3553f9..8780d227 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -385,6 +385,17 @@ if len(logs_namespace_variables) != 1: raise SystemExit("grafana-dashboard-logs.yaml must define exactly one namespace variable") if logs_namespace_variables[0].get("allValue") != "code-place-dev|code-place-prod|monitoring": raise SystemExit("CodePlace Logs dashboard namespace All value must include monitoring logs") +logs_panels = { + panel.get("title"): panel + for panel in logs_dashboard.get("panels", []) +} +for title in ("Loki Ready", "Alloy Nodes", "Loki PVC Usage"): + panel = logs_panels.get(title) + if not panel: + raise SystemExit(f"grafana-dashboard-logs.yaml missing panel {title}") + exprs = "\n".join(target.get("expr", "") for target in panel.get("targets", [])) + if "or vector(0)" not in exprs: + raise SystemExit(f"grafana-dashboard-logs.yaml:{title} must avoid empty stat panels with or vector(0)") for dashboard_name, panel_requirements in dashboard_requirements.items(): dashboard_map = dashboard_by_name.get(dashboard_name) if not dashboard_map: From d839034323f2ef539a806ca6853b257da9038e45 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:31:19 +0900 Subject: [PATCH 078/147] =?UTF-8?q?=EC=9A=94=EC=B2=AD=20ID=20=ED=97=A4?= =?UTF-8?q?=EB=8D=94=20=EC=95=88=EC=A0=84=EC=84=B1=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/account/middleware.py | 5 +++++ backend/account/tests.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/backend/account/middleware.py b/backend/account/middleware.py index 0cbac0e5..3208d973 100644 --- a/backend/account/middleware.py +++ b/backend/account/middleware.py @@ -1,4 +1,5 @@ import logging +import re import time import uuid @@ -16,6 +17,7 @@ request_logger = logging.getLogger("codeplace.request") MAX_REQUEST_ID_LENGTH = 128 +REQUEST_ID_ALLOWED_CHARS = re.compile(r"[^A-Za-z0-9_.:-]+") class RequestIDMiddleware: @@ -37,6 +39,9 @@ def __call__(self, request): @staticmethod def _request_id(request): request_id = request.META.get("HTTP_X_REQUEST_ID") + if not request_id: + return uuid.uuid4().hex + request_id = REQUEST_ID_ALLOWED_CHARS.sub("-", request_id).strip("-") if not request_id: return uuid.uuid4().hex return request_id[:MAX_REQUEST_ID_LENGTH] diff --git a/backend/account/tests.py b/backend/account/tests.py index 43cab62d..df046a5d 100644 --- a/backend/account/tests.py +++ b/backend/account/tests.py @@ -46,6 +46,24 @@ def test_accepts_existing_request_id_and_truncates_it(self): self.assertEqual(request.request_id, "a" * 128) self.assertEqual(response["X-Request-ID"], request.request_id) + def test_sanitizes_request_id_before_returning_response_header(self): + middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True})) + request = self.factory.get("/", HTTP_X_REQUEST_ID="trace-1\nSet-Cookie:bad") + + response = middleware(request) + + self.assertEqual(request.request_id, "trace-1-Set-Cookie:bad") + self.assertEqual(response["X-Request-ID"], request.request_id) + + def test_generates_request_id_when_header_has_no_safe_characters(self): + middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True})) + request = self.factory.get("/", HTTP_X_REQUEST_ID="\n\t") + + response = middleware(request) + + self.assertEqual(len(request.request_id), 32) + self.assertEqual(response["X-Request-ID"], request.request_id) + class PermissionDecoratorTest(APITestCase): """ From 3bf7860dd59ede5a67d02d517760d0e13e9ff27e Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 19:34:16 +0900 Subject: [PATCH 079/147] =?UTF-8?q?=ED=94=84=EB=A1=A0=ED=8A=B8=EC=97=94?= =?UTF-8?q?=EB=93=9C=20=EB=A1=9C=EA=B7=B8=20=EB=AF=BC=EA=B0=90=20URL=20?= =?UTF-8?q?=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/deploy/kube_nginx/nginx.conf | 1 - kubernetes/monitoring/validate.sh | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/deploy/kube_nginx/nginx.conf b/frontend/deploy/kube_nginx/nginx.conf index 110a502a..0f10c1fb 100644 --- a/frontend/deploy/kube_nginx/nginx.conf +++ b/frontend/deploy/kube_nginx/nginx.conf @@ -39,7 +39,6 @@ http { '"duration_seconds":$request_time,' '"remote_addr":"$remote_addr",' '"body_bytes_sent":$body_bytes_sent,' - '"referer":"$http_referer",' '"user_agent":"$http_user_agent",' '"x_forwarded_for":"$http_x_forwarded_for"' '}'; diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 8780d227..74b5cd5e 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -265,6 +265,8 @@ required_frontend_nginx_parts = [ for text in required_frontend_nginx_parts: if text not in frontend_nginx: raise SystemExit(f"frontend Kubernetes nginx config missing {text}") +if "$http_referer" in frontend_nginx: + raise SystemExit("frontend Kubernetes nginx access log must not include Referer because it can contain tokens") if "proxy_set_header X-Request-ID $codeplace_request_id;" not in frontend_proxy: raise SystemExit("frontend Kubernetes nginx proxy must propagate X-Request-ID") if "exec nginx -c /app/deploy/kube_nginx/nginx.conf" not in frontend_entrypoint: From 12bcad04f6c1c0003ab4999e6f23c9d7f76c3517 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:09:59 +0900 Subject: [PATCH 080/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=9A=B4?= =?UTF-8?q?=EC=98=81=20=EB=AC=B8=EC=84=9C=20=EC=B5=9C=EC=8B=A0=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 322f1a65..9457934c 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -140,7 +140,7 @@ Traefik은 JSON access log를 출력하고, `X-Request-ID`, `X-Forwarded-For` he frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. access log는 Loki에서 파싱 가능한 JSON 형식이며, `request_id`, `method`, `path`, `status_code`, `duration_seconds`, `remote_addr`, `user_agent`, `x_forwarded_for`를 포함합니다. nginx는 들어온 `X-Request-ID`를 유지하고, 없으면 nginx `$request_id`를 생성해 응답 header와 backend proxy header로 전달합니다. -Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 운영 조건에서는 Loki를 Monolithic mode로 배포하고 Longhorn PVC 기반 filesystem storage를 사용합니다. +Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 운영 조건에서는 Loki를 SingleBinary mode로 배포하고 Longhorn PVC 기반 filesystem storage를 사용합니다. - Loki release: `loki`, namespace: `monitoring`. - Alloy release: `alloy`, namespace: `monitoring`. @@ -153,7 +153,7 @@ Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라 현재 유지해야 하는 로그 저장소 불변 조건은 다음과 같습니다. -- Loki deployment mode는 `Monolithic`입니다. +- Loki deployment mode는 `SingleBinary`입니다. - Loki storage type은 `filesystem`입니다. - Loki PVC storageClass는 `longhorn`, size는 `50Gi`입니다. - Loki retention은 dev 72h, prod 168h입니다. @@ -211,7 +211,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. - `kubernetes-event-exporter.yaml`: Kubernetes Warning event를 stdout JSON으로 export해서 Alloy/Loki가 수집하도록 하는 exporter, ServiceMonitor, 최소 event read RBAC. - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. -- `logs/loki-values.yaml`: Loki Monolithic, Longhorn PVC, dev/prod retention 설정. +- `logs/loki-values.yaml`: Loki SingleBinary, Longhorn PVC, dev/prod retention 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. - `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, Loki ingest/error/canary, Alloy write backpressure, 최근 backend error, judge/celery log 조회 dashboard. - `otel-collector.yaml`: backend/celery OTLP trace를 받아 Tempo로 전송하는 OpenTelemetry Collector. @@ -225,7 +225,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, Traefik metrics scrape 설정, OTel/Tempo/Event exporter, Service endpoint, Loki/Alloy, optional DCGM 상태를 확인하는 smoke 검증 스크립트. 첫 bootstrap에서 logs stack 설치 전만 `REQUIRE_LOGS_STACK=0`으로 건너뜁니다. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. -`alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. +`alertmanager-contact-points` Secret은 repo에 평문으로 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자는 SealedSecrets로 `alertmanager-contact-points` Secret을 생성합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. Email fallback은 SMTP host/from/to/auth가 필요하므로 기본 kustomization에는 포함하지 않습니다. SMTP 값이 정해지면 `kubernetes/monitoring/alertmanager-config-email.example.yaml`을 실제 값으로 복사하거나 패치해서 적용합니다. Prometheus Operator의 `AlertmanagerConfig`는 receiver에 `emailConfigs`를 지원하며, SMTP password는 같은 namespace의 Secret key로 참조합니다. @@ -341,10 +341,10 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. - `helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --version 86.3.1 --values kubernetes/monitoring/kube-prometheus-stack-values.yaml` - values는 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None`을 설정합니다. AlertmanagerConfig는 `monitoring` namespace에 있지만 CodePlace alert의 `namespace` label은 `code-place-dev`/`code-place-prod`이므로, 기본 `OnNamespace` matcher를 쓰면 외부 namespace alert가 Discord receiver까지 도달하지 않을 수 있습니다. -2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. +2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 SealedSecret으로 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. Email fallback을 켤 경우 SMTP 정보를 별도로 정한 뒤 `alertmanager-email` Secret과 email AlertmanagerConfig를 추가 적용합니다. 3. Loki와 Alloy를 설치합니다. - - `helm upgrade --install loki grafana-community/loki --namespace monitoring --version 17.4.7 --values kubernetes/monitoring/logs/loki-values.yaml` + - `helm upgrade --install loki grafana/loki --namespace monitoring --version 6.55.0 --values kubernetes/monitoring/logs/loki-values.yaml` - `helm upgrade --install alloy grafana/alloy --namespace monitoring --version 1.10.0 --values kubernetes/monitoring/logs/alloy-values.yaml` 4. 애플리케이션은 환경별 overlay로 적용합니다. - dev: `kubectl apply -k kubernetes/overlays/dev` @@ -370,10 +370,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `/metrics`는 cluster 내부 scrape 전용이며 외부 Ingress에 노출하지 않습니다. - 공개 URL synthetic probe는 monitoring namespace 내부에서 외부 도메인으로 나가는 HTTPS 요청입니다. DNS, outbound network, TLS, Traefik/frontend/hub-auth 경로를 함께 검증하지만 브라우저 JS runtime error를 대체하지는 않습니다. -- Discord webhook URL은 Git에 저장하지 않습니다. +- Discord webhook URL은 Git에 평문으로 저장하지 않습니다. 운영 Secret은 SealedSecret으로 관리합니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. - OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지하고 dev overlay에서 먼저 활성화합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. +- Metrics backend는 kube-prometheus-stack의 Prometheus입니다. Mimir는 현재 온프렘 단기 운영 기준선에 포함하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - Kubernetes event는 Kubernetes Event Exporter가 Warning event만 stdout JSON으로 내보내고, Alloy가 일반 Pod log와 같은 경로로 Loki에 적재합니다. - PostgreSQL은 CNPG instance exporter와 PodMonitor로 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor로 `redis_*` 지표를 수집합니다. @@ -402,7 +403,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - PrometheusRule shape check: P0/P1 interval, alert priority/severity, summary/description, duplicate alert namespace label. - AlertmanagerConfig shape check: groupBy, P0/P1 Discord receivers, webhook Secret reference. - kube-prometheus-stack values shape check: Prometheus selector policy, AlertmanagerConfig selector, Loki datasource, dashboard sidecar label. - - Loki storage shape check: Monolithic mode, filesystem storage, Longhorn PVC, 50Gi size, dev/prod retention, MinIO disabled. + - Loki storage shape check: SingleBinary mode, filesystem storage, Longhorn PVC, 50Gi size, dev/prod retention, MinIO disabled. - scrape resource shape check: ServiceMonitor/PodMonitor selector label, scrape path/port/interval. - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. - Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh` @@ -413,10 +414,18 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. ### Discord Webhook Secret -Discord webhook URL은 repo에 커밋하지 않습니다. 클러스터별로 다음 명령으로 생성합니다. +Discord webhook URL은 repo에 평문으로 커밋하지 않습니다. SealedSecrets controller가 설치된 클러스터에서 운영자가 다음 흐름으로 SealedSecret을 생성합니다. ```sh +mkdir -p kubernetes/monitoring/secrets + kubectl -n monitoring create secret generic alertmanager-contact-points \ --from-literal=webhook-url="$ALERT_WEBHOOK_URL" \ - --dry-run=client -o yaml | kubectl apply -f - + --dry-run=client -o yaml \ + | kubeseal --controller-namespace kube-system --format yaml \ + > kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml + +kubectl apply -f kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml ``` + +SealedSecret 파일은 cluster 공개키로 암호화된 값만 포함합니다. controller namespace가 다르면 `--controller-namespace`를 실제 설치 namespace로 바꿉니다. From 816dcf08b1e71e74ab0b8749ee0a6b0220c728db Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:12:22 +0900 Subject: [PATCH 081/147] =?UTF-8?q?=EC=95=8C=EB=A6=BC=20SealedSecret=20?= =?UTF-8?q?=EC=9A=B4=EC=98=81=20=EC=A0=88=EC=B0=A8=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-runbook/_index.md | 25 +++++++++++++++++++ kubernetes/monitoring/secrets/README.md | 23 +++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 kubernetes/monitoring/secrets/README.md diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 53233449..b5409beb 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -49,6 +49,17 @@ Alerts -> alertname / namespace / priority Status -> Configuration -> p0-discord / p1-discord ``` +Discord webhook은 SealedSecret으로 관리합니다. AlertmanagerConfig는 최종적으로 `monitoring` namespace의 `alertmanager-contact-points` Secret key `webhook-url`을 참조하므로, 알림이 오지 않으면 SealedSecret controller와 복호화된 Secret을 함께 확인합니다. + +```sh +kubectl get crd sealedsecrets.bitnami.com +kubectl get pod -A | grep -i sealed +kubectl -n monitoring get sealedsecret,secret | grep alertmanager-contact-points +kubectl -n monitoring get secret alertmanager-contact-points -o jsonpath='{.data.webhook-url}' | wc -c +kubectl -n monitoring get alertmanagerconfig codeplace-alert-routing -o yaml +kubectl -n monitoring logs alertmanager-kube-prometheus-stack-alertmanager-0 -c alertmanager --tail=100 +``` + `kube-prometheus-stack-values.yaml`의 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type`은 `None`이어야 합니다. 기본 `OnNamespace` 전략이면 `monitoring` namespace의 AlertmanagerConfig가 `namespace=code-place-dev` 또는 `namespace=code-place-prod` alert를 라우팅하지 못할 수 있습니다. Email fallback을 켠 환경에서는 `Status -> Configuration`에서 `p0-email` / `p1-email` receiver도 확인합니다. @@ -1088,6 +1099,20 @@ node pressure는 앱 문제가 아니라 cluster resource 문제일 가능성이 Discord webhook 자체는 curl로 확인할 수 있습니다. Alertmanager routing은 임시 `PrometheusRule`로 확인합니다. Email fallback은 SMTP 정보가 정해진 환경에서만 켭니다. `kubernetes/monitoring/alertmanager-config-email.example.yaml`의 placeholder를 실제 `smarthost`, `from`, `to`, `authUsername`으로 바꾸고 `alertmanager-email` Secret을 만든 뒤 적용합니다. +SealedSecret으로 webhook을 새로 만들 때는 운영자가 실제 cluster 공개키로 암호화합니다. + +```sh +mkdir -p kubernetes/monitoring/secrets + +kubectl -n monitoring create secret generic alertmanager-contact-points \ + --from-literal=webhook-url="$ALERT_WEBHOOK_URL" \ + --dry-run=client -o yaml \ + | kubeseal --controller-namespace kube-system --format yaml \ + > kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml + +kubectl apply -f kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml +``` + ```yaml apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule diff --git a/kubernetes/monitoring/secrets/README.md b/kubernetes/monitoring/secrets/README.md new file mode 100644 index 00000000..6d81f949 --- /dev/null +++ b/kubernetes/monitoring/secrets/README.md @@ -0,0 +1,23 @@ +# Monitoring Secrets + +This directory is reserved for SealedSecret manifests generated per cluster. + +Do not commit plaintext Kubernetes Secret manifests or raw Discord webhook URLs. +The AlertmanagerConfig expects the decrypted Secret below to exist in the +`monitoring` namespace: + +- Secret: `alertmanager-contact-points` +- Key: `webhook-url` + +Generate the SealedSecret with the target cluster public key: + +```sh +kubectl -n monitoring create secret generic alertmanager-contact-points \ + --from-literal=webhook-url="$ALERT_WEBHOOK_URL" \ + --dry-run=client -o yaml \ + | kubeseal --controller-namespace kube-system --format yaml \ + > kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml +``` + +If the SealedSecrets controller runs outside `kube-system`, change +`--controller-namespace` to the actual namespace. From 3edf6d8402411917c7147cf0cad6ab78d26ee7a4 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:15:31 +0900 Subject: [PATCH 082/147] =?UTF-8?q?=ED=8A=B8=EB=A0=88=EC=9D=B4=EC=8A=A4=20?= =?UTF-8?q?=EB=8C=80=EC=8B=9C=EB=B3=B4=EB=93=9C=20=EA=B2=80=EC=A6=9D=20?= =?UTF-8?q?=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../monitoring/grafana-dashboard-traces.yaml | 2 +- kubernetes/monitoring/smoke-check.sh | 4 ++++ kubernetes/monitoring/validate.sh | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/kubernetes/monitoring/grafana-dashboard-traces.yaml b/kubernetes/monitoring/grafana-dashboard-traces.yaml index 40e59040..9b7467fc 100644 --- a/kubernetes/monitoring/grafana-dashboard-traces.yaml +++ b/kubernetes/monitoring/grafana-dashboard-traces.yaml @@ -117,7 +117,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}), 1)", "legendFormat": "usage"} + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}), 1) or vector(0)", "legendFormat": "usage"} ] }, { diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index a17fa549..92b2b648 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -158,6 +158,10 @@ require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datas require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'uid: Tempo' require_jsonpath_value "$NAMESPACE" alertmanager kube-prometheus-stack-alertmanager \ '{.spec.alertmanagerConfigMatcherStrategy.type}' None +require_jsonpath_value "$NAMESPACE" persistentvolumeclaim tempo-data \ + '{.spec.storageClassName}' longhorn +require_jsonpath_value "$NAMESPACE" persistentvolumeclaim tempo-data \ + '{.spec.resources.requests.storage}' 20Gi if resource_exists "$NAMESPACE" secret alertmanager-contact-points; then kubectl -n "$NAMESPACE" get secret alertmanager-contact-points \ diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index 74b5cd5e..bcde1de4 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -398,6 +398,21 @@ for title in ("Loki Ready", "Alloy Nodes", "Loki PVC Usage"): exprs = "\n".join(target.get("expr", "") for target in panel.get("targets", [])) if "or vector(0)" not in exprs: raise SystemExit(f"grafana-dashboard-logs.yaml:{title} must avoid empty stat panels with or vector(0)") +traces_dashboard = [ + json.loads(value) + for value in dashboard_by_name["grafana-dashboard-traces.yaml"].get("data", {}).values() +][0] +traces_panels = { + panel.get("title"): panel + for panel in traces_dashboard.get("panels", []) +} +for title in ("Tempo Ready", "OTel Collector Ready", "Trace Receive Rate", "Trace Export Failures", "Tempo PVC Usage"): + panel = traces_panels.get(title) + if not panel: + raise SystemExit(f"grafana-dashboard-traces.yaml missing panel {title}") + exprs = "\n".join(target.get("expr", "") for target in panel.get("targets", [])) + if "or vector(0)" not in exprs: + raise SystemExit(f"grafana-dashboard-traces.yaml:{title} must avoid empty stat panels with or vector(0)") for dashboard_name, panel_requirements in dashboard_requirements.items(): dashboard_map = dashboard_by_name.get(dashboard_name) if not dashboard_map: From 802e974e09a9def023bcd99f8a19d03493f70d58 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:19:16 +0900 Subject: [PATCH 083/147] =?UTF-8?q?=ED=8A=B8=EB=A0=88=EC=9D=B4=EC=8A=A4=20?= =?UTF-8?q?=EC=BF=A0=EB=B2=84=EB=84=A4=ED=8B=B0=EC=8A=A4=20=EB=A9=94?= =?UTF-8?q?=ED=83=80=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/otel-collector.yaml | 51 +++++++++++++++++++++++ kubernetes/monitoring/smoke-check.sh | 8 ++++ kubernetes/monitoring/validate.sh | 43 +++++++++++++++++++ 3 files changed, 102 insertions(+) diff --git a/kubernetes/monitoring/otel-collector.yaml b/kubernetes/monitoring/otel-collector.yaml index 3975b52e..cf957d78 100644 --- a/kubernetes/monitoring/otel-collector.yaml +++ b/kubernetes/monitoring/otel-collector.yaml @@ -1,4 +1,45 @@ apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-collector +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-collector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: otel-collector +subjects: + - kind: ServiceAccount + name: otel-collector + namespace: monitoring +--- +apiVersion: v1 kind: ConfigMap metadata: name: otel-collector-config @@ -17,6 +58,14 @@ data: check_interval: 5s limit_mib: 384 spike_limit_mib: 128 + k8sattributes: + auth_type: serviceAccount + extract: + metadata: + - k8s.namespace.name + - k8s.pod.name + - k8s.deployment.name + - k8s.node.name batch: timeout: 5s send_batch_size: 512 @@ -40,6 +89,7 @@ data: - otlp processors: - memory_limiter + - k8sattributes - batch exporters: - otlp/tempo @@ -60,6 +110,7 @@ spec: labels: app: otel-collector spec: + serviceAccountName: otel-collector containers: - name: otel-collector image: otel/opentelemetry-collector-contrib:0.128.0 diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh index 92b2b648..de6585fa 100755 --- a/kubernetes/monitoring/smoke-check.sh +++ b/kubernetes/monitoring/smoke-check.sh @@ -119,6 +119,9 @@ for resource in \ "configmap/kubernetes-event-exporter-config" \ "configmap/otel-collector-config" \ "configmap/tempo-config" \ + "serviceaccount/otel-collector" \ + "clusterrole/otel-collector" \ + "clusterrolebinding/otel-collector" \ "deployment/blackbox-exporter" \ "deployment/kubernetes-event-exporter" \ "deployment/otel-collector" \ @@ -158,6 +161,11 @@ require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datas require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'uid: Tempo' require_jsonpath_value "$NAMESPACE" alertmanager kube-prometheus-stack-alertmanager \ '{.spec.alertmanagerConfigMatcherStrategy.type}' None +require_jsonpath_value "$NAMESPACE" deployment otel-collector \ + '{.spec.template.spec.serviceAccountName}' otel-collector +require_configmap_data_contains "$NAMESPACE" otel-collector-config config.yaml 'k8sattributes:' +require_configmap_data_contains "$NAMESPACE" otel-collector-config config.yaml 'k8s.namespace.name' +require_configmap_data_contains "$NAMESPACE" otel-collector-config config.yaml 'k8s.deployment.name' require_jsonpath_value "$NAMESPACE" persistentvolumeclaim tempo-data \ '{.spec.storageClassName}' longhorn require_jsonpath_value "$NAMESPACE" persistentvolumeclaim tempo-data \ diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh index bcde1de4..1db524dd 100755 --- a/kubernetes/monitoring/validate.sh +++ b/kubernetes/monitoring/validate.sh @@ -250,6 +250,49 @@ if "regex = \"(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/trae raise SystemExit("Alloy keep rule must collect CodePlace/monitoring namespaces and kube-system Traefik logs") print("ALLOY LOG COLLECTION SHAPE OK") +otel_documents = list(yaml.safe_load_all((root / "otel-collector.yaml").read_text())) +otel_by_kind_name = { + (document.get("kind"), document.get("metadata", {}).get("name")): document + for document in otel_documents + if document +} +if ("ServiceAccount", "otel-collector") not in otel_by_kind_name: + raise SystemExit("otel-collector must run with a dedicated ServiceAccount") +cluster_role = otel_by_kind_name.get(("ClusterRole", "otel-collector"), {}) +rules_by_group = { + tuple(rule.get("apiGroups", [])): set(rule.get("resources", [])) + for rule in cluster_role.get("rules", []) +} +if not {"pods", "namespaces"}.issubset(rules_by_group.get(("",), set())): + raise SystemExit("otel-collector ClusterRole must read pods and namespaces for k8s trace attributes") +if "replicasets" not in rules_by_group.get(("apps",), set()): + raise SystemExit("otel-collector ClusterRole must read replicasets for deployment trace attributes") +cluster_role_binding = otel_by_kind_name.get(("ClusterRoleBinding", "otel-collector"), {}) +subjects = cluster_role_binding.get("subjects", []) +if not any(subject.get("kind") == "ServiceAccount" and subject.get("name") == "otel-collector" for subject in subjects): + raise SystemExit("otel-collector ClusterRoleBinding must bind the otel-collector ServiceAccount") +otel_deployment = otel_by_kind_name.get(("Deployment", "otel-collector"), {}) +if otel_deployment.get("spec", {}).get("template", {}).get("spec", {}).get("serviceAccountName") != "otel-collector": + raise SystemExit("otel-collector Deployment must use the otel-collector ServiceAccount") +otel_config = otel_by_kind_name.get(("ConfigMap", "otel-collector-config"), {}).get("data", {}).get("config.yaml", "") +otel_config_yaml = yaml.safe_load(otel_config) +processors = otel_config_yaml.get("processors", {}) +if "k8sattributes" not in processors: + raise SystemExit("otel-collector must enrich traces with k8sattributes") +k8s_metadata = set(processors["k8sattributes"].get("extract", {}).get("metadata", [])) +required_k8s_metadata = {"k8s.namespace.name", "k8s.pod.name", "k8s.deployment.name", "k8s.node.name"} +if not required_k8s_metadata.issubset(k8s_metadata): + raise SystemExit("otel-collector k8sattributes must extract namespace, pod, deployment, and node") +trace_processors = ( + otel_config_yaml.get("service", {}) + .get("pipelines", {}) + .get("traces", {}) + .get("processors", []) +) +if "k8sattributes" not in trace_processors: + raise SystemExit("otel-collector traces pipeline must include k8sattributes") +print("OTEL TRACE ENRICHMENT SHAPE OK") + frontend_nginx = Path("frontend/deploy/kube_nginx/nginx.conf").read_text() frontend_proxy = Path("frontend/deploy/kube_nginx/api_proxy.conf").read_text() frontend_entrypoint = Path("frontend/deploy/entrypoint.sh").read_text() From 27eef1c61297b0f62eb5372e6ac3bc08817fd35f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:26:59 +0900 Subject: [PATCH 084/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0=EC=99=80=20=EC=BB=AC=EB=A0=89=ED=84=B0=20=EB=B6=84?= =?UTF-8?q?=EC=82=B0=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 19 +- .../infra/observability-runbook/_index.md | 1 - .../monitoring/grafana-dashboard-traces.yaml | 8 +- kubernetes/monitoring/logs/README.md | 30 +- kubernetes/monitoring/otel-collector.yaml | 19 +- kubernetes/monitoring/prometheus-rules.yaml | 10 + kubernetes/monitoring/smoke-check.sh | 274 -------- kubernetes/monitoring/validate.sh | 632 ------------------ 8 files changed, 39 insertions(+), 954 deletions(-) delete mode 100755 kubernetes/monitoring/smoke-check.sh delete mode 100755 kubernetes/monitoring/validate.sh diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 9457934c..aed8c12e 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -221,8 +221,6 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `grafana-dashboard-storage.yaml`: Longhorn volume/node/disk 상태와 storage capacity dashboard. - `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. - `dcgm-exporter.yaml`: vLLM GPU node용 NVIDIA DCGM exporter DaemonSet/Service/ServiceMonitor. -- `validate.sh`: monitoring YAML, Grafana dashboard JSON, monitoring/app kustomize render, app observability env/Service wiring, 선택적 promtool/Helm render 검증 스크립트. -- `smoke-check.sh`: 운영 클러스터 적용 후 CRD, 핵심 monitoring 리소스, dashboard ConfigMap, Probe, Traefik metrics scrape 설정, OTel/Tempo/Event exporter, Service endpoint, Loki/Alloy, optional DCGM 상태를 확인하는 smoke 검증 스크립트. 첫 bootstrap에서 logs stack 설치 전만 `REQUIRE_LOGS_STACK=0`으로 건너뜁니다. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. `alertmanager-contact-points` Secret은 repo에 평문으로 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자는 SealedSecrets로 `alertmanager-contact-points` Secret을 생성합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. @@ -397,20 +395,9 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Kubernetes monitoring render: `kubectl kustomize kubernetes/monitoring` - Logs values YAML parse: `kubernetes/monitoring/logs/loki-values.yaml`, `kubernetes/monitoring/logs/alloy-values.yaml` - Grafana dashboard JSON parse: `grafana-dashboard-codeplace.yaml`, `grafana-dashboard-logs.yaml` -- Monitoring bundle validation: `bash kubernetes/monitoring/validate.sh` - - YAML parse: kube-prometheus-stack values, PrometheusRule, AlertmanagerConfig, ServiceMonitor, PodMonitor, Loki/Alloy values. - - Grafana dashboard JSON parse와 dashboard shape check: uid/title/refresh/panel/target expr. - - PrometheusRule shape check: P0/P1 interval, alert priority/severity, summary/description, duplicate alert namespace label. - - AlertmanagerConfig shape check: groupBy, P0/P1 Discord receivers, webhook Secret reference. - - kube-prometheus-stack values shape check: Prometheus selector policy, AlertmanagerConfig selector, Loki datasource, dashboard sidecar label. - - Loki storage shape check: SingleBinary mode, filesystem storage, Longhorn PVC, 50Gi size, dev/prod retention, MinIO disabled. - - scrape resource shape check: ServiceMonitor/PodMonitor selector label, scrape path/port/interval. - - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인. -- Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh` - - Prometheus Operator CRD, monitoring namespace resource, selector label, webhook Secret, kube-prometheus-stack Pod readiness, Loki/Alloy, app namespace backend service/port/readiness를 읽기 전용으로 확인합니다. - -`promtool`이 있는 환경에서는 `validate.sh`가 `PrometheusRule.spec.groups`를 임시 rule file로 추출한 뒤 `promtool check rules`를 실행합니다. -`helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다. +- Prometheus rule syntax: `promtool check rules`에 `PrometheusRule.spec.groups`를 추출해서 확인 +- Helm render: kube-prometheus-stack, Loki, Alloy chart를 pinned version과 values로 렌더링 +- Live cluster check: CRD, monitoring namespace resource, dashboard ConfigMap, webhook Secret, kube-prometheus-stack Pod readiness, Loki/Alloy, app namespace backend service/port/readiness를 `kubectl`로 확인 ### Discord Webhook Secret diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index b5409beb..15d3a7f2 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -14,7 +14,6 @@ weight: 6 알림을 받으면 먼저 scope를 확인합니다. ```sh -bash kubernetes/monitoring/validate.sh kubectl get ns kubectl -n monitoring get prometheusrule,alertmanagerconfig,servicemonitor kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy|otel|tempo|blackbox|event-exporter' diff --git a/kubernetes/monitoring/grafana-dashboard-traces.yaml b/kubernetes/monitoring/grafana-dashboard-traces.yaml index 9b7467fc..c8fa91af 100644 --- a/kubernetes/monitoring/grafana-dashboard-traces.yaml +++ b/kubernetes/monitoring/grafana-dashboard-traces.yaml @@ -43,7 +43,7 @@ data: }, { "type": "stat", - "title": "OTel Collector Ready", + "title": "OTel Collector Replicas", "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -53,14 +53,16 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} ] } } }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"otel-collector-.*\"}) or vector(0)", "legendFormat": "otel-collector"} + {"expr": "kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"otel-collector\"} or vector(0)", "legendFormat": "available"}, + {"expr": "kube_deployment_spec_replicas{namespace=\"monitoring\", deployment=\"otel-collector\"} or vector(0)", "legendFormat": "desired"} ] }, { diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 33810acf..3ebbee50 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -44,37 +44,12 @@ helm upgrade --install alloy grafana/alloy \ ## Verify -Run the repository validation first: - -```sh -bash kubernetes/monitoring/validate.sh -``` - -If `helm` is installed and the chart repos are already added, the script also renders kube-prometheus-stack, Loki, and Alloy. Without `helm`, it still validates YAML, Grafana dashboard JSON, and kustomize output. - -After applying the monitoring resources to a live cluster, run the read-only smoke check: - -```sh -bash kubernetes/monitoring/smoke-check.sh -``` - -Use `MONITORING_NAMESPACE` and `CODEPLACE_NAMESPACES` to override the defaults: - -```sh -MONITORING_NAMESPACE=monitoring CODEPLACE_NAMESPACES="code-place-dev" \ - bash kubernetes/monitoring/smoke-check.sh -``` - -The smoke check treats Loki and Alloy as required by default. During the first -bootstrap before installing the logs stack, explicitly opt out: - -```sh -REQUIRE_LOGS_STACK=0 bash kubernetes/monitoring/smoke-check.sh -``` +After applying the monitoring resources, check the live resources directly: ```sh kubectl -n monitoring get pod | grep -E 'loki|alloy|grafana' kubectl -n monitoring get pvc | grep loki +kubectl -n monitoring get servicemonitor loki alloy ``` Grafana should show a `Loki` datasource. Useful Explore queries: @@ -101,6 +76,7 @@ Keep these invariants unless the storage design changes intentionally: - Loki stays in `SingleBinary` deployment mode. - Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. +- In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors can be replicated across nodes. - Alloy keeps namespace-based collection for `code-place-dev`, `code-place-prod`, and `monitoring`; do not depend on `app.kubernetes.io/name` for CodePlace app logs because the application manifests primarily use `app`. - Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. - The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. diff --git a/kubernetes/monitoring/otel-collector.yaml b/kubernetes/monitoring/otel-collector.yaml index cf957d78..b7afdbb1 100644 --- a/kubernetes/monitoring/otel-collector.yaml +++ b/kubernetes/monitoring/otel-collector.yaml @@ -101,7 +101,7 @@ metadata: labels: app: otel-collector spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: otel-collector @@ -111,6 +111,13 @@ spec: app: otel-collector spec: serviceAccountName: otel-collector + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: otel-collector containers: - name: otel-collector image: otel/opentelemetry-collector-contrib:0.128.0 @@ -154,6 +161,16 @@ spec: configMap: name: otel-collector-config --- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: otel-collector +spec: + minAvailable: 1 + selector: + matchLabels: + app: otel-collector +--- apiVersion: v1 kind: Service metadata: diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 0cf1f2ea..a1b876ea 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -813,6 +813,16 @@ spec: annotations: summary: "OpenTelemetry Collector is not ready" description: "The OpenTelemetry Collector has not been ready for 2 minutes. Enabled services cannot export traces." + - alert: OTelCollectorReplicaUnavailable + expr: kube_deployment_status_replicas_available{namespace="monitoring", deployment="otel-collector"} < kube_deployment_spec_replicas{namespace="monitoring", deployment="otel-collector"} + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "OpenTelemetry Collector replica is unavailable" + description: "At least one OpenTelemetry Collector replica is unavailable for 5 minutes. Trace ingestion is still degraded even if another replica is serving traffic." - alert: OTelCollectorRefusedSpans expr: sum(increase(otelcol_receiver_refused_spans_total{namespace="monitoring"}[5m])) > 0 for: 2m diff --git a/kubernetes/monitoring/smoke-check.sh b/kubernetes/monitoring/smoke-check.sh deleted file mode 100755 index de6585fa..00000000 --- a/kubernetes/monitoring/smoke-check.sh +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -NAMESPACE="${MONITORING_NAMESPACE:-monitoring}" -APP_NAMESPACES="${CODEPLACE_NAMESPACES:-code-place-dev code-place-prod}" -REQUIRE_LOGS_STACK="${REQUIRE_LOGS_STACK:-1}" - -ok() { - echo "OK $*" -} - -fail() { - echo "FAIL $*" >&2 - exit 1 -} - -require_cmd() { - command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" -} - -resource_exists() { - kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1 -} - -resource_count() { - kubectl -n "$1" get "$2" -l "$3" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | wc -w | tr -d ' ' -} - -require_service_endpoints() { - local namespace="$1" - local service="$2" - local addresses - addresses="$(kubectl -n "$namespace" get endpoints "$service" -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)" - if [ -z "$addresses" ]; then - addresses="$(kubectl -n "$namespace" get endpointslice \ - -l "kubernetes.io/service-name=$service" \ - -o jsonpath='{.items[*].endpoints[*].addresses[*]}' 2>/dev/null || true)" - fi - [ -n "$addresses" ] || fail "$namespace service/$service has no ready endpoints" - ok "$namespace service/$service endpoints" -} - -require_label() { - local namespace="$1" - local resource="$2" - local name="$3" - local key="$4" - local expected="$5" - local actual - actual="$(kubectl -n "$namespace" get "$resource" "$name" -o "jsonpath={.metadata.labels['${key//./\\.}']}")" - [ "$actual" = "$expected" ] || fail "$resource/$name label $key expected=$expected actual=$actual" - ok "$resource/$name label $key=$expected" -} - -require_jsonpath_value() { - local namespace="$1" - local resource="$2" - local name="$3" - local jsonpath="$4" - local expected="$5" - local actual - actual="$(kubectl -n "$namespace" get "$resource" "$name" -o "jsonpath=$jsonpath")" - [ "$actual" = "$expected" ] || fail "$resource/$name $jsonpath expected=$expected actual=$actual" - ok "$resource/$name $jsonpath=$expected" -} - -require_configmap_data_contains() { - local namespace="$1" - local name="$2" - local key="$3" - local expected="$4" - local value - value="$(kubectl -n "$namespace" get configmap "$name" -o "jsonpath={.data.${key//./\\.}}")" - printf '%s' "$value" | grep -Fq "$expected" \ - || fail "configmap/$name $key missing expected content: $expected" - ok "configmap/$name $key contains $expected" -} - -require_cmd kubectl - -echo "==> checking Prometheus Operator CRDs" -for crd in \ - prometheusrules.monitoring.coreos.com \ - probes.monitoring.coreos.com \ - servicemonitors.monitoring.coreos.com \ - podmonitors.monitoring.coreos.com \ - alertmanagerconfigs.monitoring.coreos.com; do - kubectl get crd "$crd" >/dev/null - ok "CRD $crd" -done - -echo "==> checking monitoring namespace resources" -kubectl get namespace "$NAMESPACE" >/dev/null -ok "namespace $NAMESPACE" - -for resource in \ - "prometheusrule/codeplace-fast-alerts" \ - "servicemonitor/backend" \ - "servicemonitor/blackbox-exporter" \ - "servicemonitor/dcgm-exporter" \ - "servicemonitor/kubernetes-event-exporter" \ - "servicemonitor/longhorn" \ - "servicemonitor/otel-collector" \ - "servicemonitor/tempo" \ - "servicemonitor/traefik" \ - "servicemonitor/vllm" \ - "podmonitor/postgres" \ - "podmonitor/redis" \ - "alertmanagerconfig/codeplace-alert-routing" \ - "configmap/grafana-dashboard-codeplace" \ - "configmap/grafana-dashboard-codeplace-ai-inference" \ - "configmap/grafana-dashboard-codeplace-kubernetes-events" \ - "configmap/grafana-dashboard-codeplace-logs" \ - "configmap/grafana-dashboard-codeplace-monitoring-stack" \ - "configmap/grafana-dashboard-codeplace-public-endpoints" \ - "configmap/grafana-dashboard-codeplace-storage" \ - "configmap/grafana-dashboard-codeplace-traces" \ - "configmap/kube-prometheus-stack-grafana-datasource" \ - "configmap/kubernetes-event-exporter-config" \ - "configmap/otel-collector-config" \ - "configmap/tempo-config" \ - "serviceaccount/otel-collector" \ - "clusterrole/otel-collector" \ - "clusterrolebinding/otel-collector" \ - "deployment/blackbox-exporter" \ - "deployment/kubernetes-event-exporter" \ - "deployment/otel-collector" \ - "deployment/tempo" \ - "service/blackbox-exporter" \ - "service/kubernetes-event-exporter" \ - "service/otel-collector" \ - "service/tempo" \ - "persistentvolumeclaim/tempo-data" \ - "probe/codeplace-public-dev-http" \ - "probe/codeplace-public-prod-http" \ - "probe/codeplace-grafana-http" \ - "probe/codeplace-hub-auth-dev-http" \ - "probe/codeplace-hub-auth-prod-http"; do - kubectl -n "$NAMESPACE" get "$resource" >/dev/null - ok "$resource" -done - -require_label "$NAMESPACE" prometheusrule codeplace-fast-alerts release kube-prometheus-stack -require_label "$NAMESPACE" servicemonitor backend release kube-prometheus-stack -require_label "$NAMESPACE" servicemonitor traefik release kube-prometheus-stack -require_label "$NAMESPACE" podmonitor postgres release kube-prometheus-stack -require_label "$NAMESPACE" podmonitor redis release kube-prometheus-stack -require_label "$NAMESPACE" alertmanagerconfig codeplace-alert-routing alertmanagerConfig codeplace -require_label "$NAMESPACE" configmap grafana-dashboard-codeplace grafana_dashboard 1 -require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-ai-inference grafana_dashboard 1 -require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-kubernetes-events grafana_dashboard 1 -require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-logs grafana_dashboard 1 -require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-monitoring-stack grafana_dashboard 1 -require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-public-endpoints grafana_dashboard 1 -require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-storage grafana_dashboard 1 -require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-traces grafana_dashboard 1 -require_label "$NAMESPACE" configmap kube-prometheus-stack-grafana-datasource grafana_datasource 1 -require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'name: Loki' -require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'uid: Loki' -require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'name: Tempo' -require_configmap_data_contains "$NAMESPACE" kube-prometheus-stack-grafana-datasource datasource.yaml 'uid: Tempo' -require_jsonpath_value "$NAMESPACE" alertmanager kube-prometheus-stack-alertmanager \ - '{.spec.alertmanagerConfigMatcherStrategy.type}' None -require_jsonpath_value "$NAMESPACE" deployment otel-collector \ - '{.spec.template.spec.serviceAccountName}' otel-collector -require_configmap_data_contains "$NAMESPACE" otel-collector-config config.yaml 'k8sattributes:' -require_configmap_data_contains "$NAMESPACE" otel-collector-config config.yaml 'k8s.namespace.name' -require_configmap_data_contains "$NAMESPACE" otel-collector-config config.yaml 'k8s.deployment.name' -require_jsonpath_value "$NAMESPACE" persistentvolumeclaim tempo-data \ - '{.spec.storageClassName}' longhorn -require_jsonpath_value "$NAMESPACE" persistentvolumeclaim tempo-data \ - '{.spec.resources.requests.storage}' 20Gi - -if resource_exists "$NAMESPACE" secret alertmanager-contact-points; then - kubectl -n "$NAMESPACE" get secret alertmanager-contact-points \ - -o jsonpath='{.data.webhook-url}' | grep -q . \ - || fail "secret/alertmanager-contact-points missing webhook-url key" - ok "secret/alertmanager-contact-points webhook-url" -else - fail "secret/alertmanager-contact-points is missing" -fi - -echo "==> checking kube-prometheus-stack pods" -for selector in \ - "app.kubernetes.io/name=grafana" \ - "app.kubernetes.io/name=prometheus" \ - "app.kubernetes.io/name=alertmanager" \ - "app.kubernetes.io/name=kube-prometheus-stack-operator"; do - kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l "$selector" --timeout=30s - ok "pods ready for $selector" -done - -echo "==> checking CodePlace monitoring pods" -for selector in \ - "app=blackbox-exporter" \ - "app.kubernetes.io/name=kubernetes-event-exporter" \ - "app=otel-collector" \ - "app=tempo"; do - kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l "$selector" --timeout=30s - ok "pods ready for $selector" -done - -for service in \ - blackbox-exporter \ - kubernetes-event-exporter \ - otel-collector \ - tempo; do - require_service_endpoints "$NAMESPACE" "$service" -done - -if kubectl -n "$NAMESPACE" get daemonset dcgm-exporter >/dev/null 2>&1; then - desired="$(kubectl -n "$NAMESPACE" get daemonset dcgm-exporter -o jsonpath='{.status.desiredNumberScheduled}')" - available="$(kubectl -n "$NAMESPACE" get daemonset dcgm-exporter -o jsonpath='{.status.numberAvailable}')" - [ "$desired" = "$available" ] || fail "daemonset/dcgm-exporter available=$available desired=$desired" - ok "daemonset/dcgm-exporter available=$available desired=$desired" - require_service_endpoints "$NAMESPACE" dcgm-exporter -fi - -echo "==> checking logs stack" -if [ "$(resource_count "$NAMESPACE" pod app.kubernetes.io/name=loki)" -gt 0 ]; then - kubectl -n "$NAMESPACE" get statefulset loki >/dev/null - ok "statefulset/loki" - require_jsonpath_value "$NAMESPACE" statefulset loki \ - '{.spec.volumeClaimTemplates[?(@.metadata.name=="storage")].spec.storageClassName}' longhorn - require_jsonpath_value "$NAMESPACE" statefulset loki \ - '{.spec.volumeClaimTemplates[?(@.metadata.name=="storage")].spec.resources.requests.storage}' 50Gi - kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l app.kubernetes.io/name=loki --timeout=30s - ok "loki pods ready" - if resource_exists "$NAMESPACE" service loki-gateway; then - require_service_endpoints "$NAMESPACE" loki-gateway - else - fail "$NAMESPACE service/loki-gateway is missing" - fi - kubectl -n "$NAMESPACE" get persistentvolumeclaim storage-loki-0 >/dev/null - ok "persistentvolumeclaim/storage-loki-0" - require_label "$NAMESPACE" servicemonitor loki release kube-prometheus-stack -elif [ "$REQUIRE_LOGS_STACK" = "1" ]; then - fail "loki pods are missing; set REQUIRE_LOGS_STACK=0 only for pre-Loki bootstrap checks" -else - echo "SKIP loki pods: app.kubernetes.io/name=loki not found" -fi - -if kubectl -n "$NAMESPACE" get daemonset alloy >/dev/null 2>&1; then - desired="$(kubectl -n "$NAMESPACE" get daemonset alloy -o jsonpath='{.status.desiredNumberScheduled}')" - available="$(kubectl -n "$NAMESPACE" get daemonset alloy -o jsonpath='{.status.numberAvailable}')" - [ "$desired" = "$available" ] || fail "daemonset/alloy available=$available desired=$desired" - ok "daemonset/alloy available=$available desired=$desired" - require_jsonpath_value "$NAMESPACE" daemonset alloy \ - '{.spec.template.spec.volumes[?(@.name=="varlog")].hostPath.path}' /var/log - require_jsonpath_value "$NAMESPACE" daemonset alloy \ - '{.spec.template.spec.containers[?(@.name=="alloy")].volumeMounts[?(@.name=="varlog")].mountPath}' /var/log - require_label "$NAMESPACE" servicemonitor alloy release kube-prometheus-stack - require_configmap_data_contains "$NAMESPACE" alloy config.alloy 'regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik"' -elif [ "$REQUIRE_LOGS_STACK" = "1" ]; then - fail "daemonset/alloy is missing; set REQUIRE_LOGS_STACK=0 only for pre-Alloy bootstrap checks" -else - echo "SKIP daemonset/alloy: not found" -fi - -echo "==> checking application namespaces" -for app_namespace in $APP_NAMESPACES; do - kubectl get namespace "$app_namespace" >/dev/null - ok "namespace $app_namespace" - kubectl -n "$app_namespace" get svc backend >/dev/null - ok "$app_namespace service/backend" - kubectl -n "$app_namespace" get svc backend -o jsonpath='{.spec.ports[*].name}' | grep -qw api \ - || fail "$app_namespace service/backend missing api port" - ok "$app_namespace service/backend api port" - kubectl -n "$app_namespace" wait --for=condition=Ready pod -l app=backend --timeout=30s - ok "$app_namespace backend pods ready" - require_service_endpoints "$app_namespace" backend -done - -echo "observability smoke check completed" diff --git a/kubernetes/monitoring/validate.sh b/kubernetes/monitoring/validate.sh deleted file mode 100755 index 1db524dd..00000000 --- a/kubernetes/monitoring/validate.sh +++ /dev/null @@ -1,632 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -MONITORING_DIR="${ROOT_DIR}/kubernetes/monitoring" - -require_cmd() { - local cmd="$1" - if ! command -v "${cmd}" >/dev/null 2>&1; then - echo "missing required command: ${cmd}" >&2 - exit 1 - fi -} - -optional_cmd() { - command -v "$1" >/dev/null 2>&1 -} - -require_cmd python3 -require_cmd kubectl - -echo "==> validating monitoring YAML and Grafana dashboard JSON" -python3 - <<'PY' -from pathlib import Path -import json -import sys - -try: - import yaml -except ImportError: - yaml = None - -root = Path("kubernetes/monitoring") -yaml_paths = [ - root / "kube-prometheus-stack-values.yaml", - root / "prometheus-rules.yaml", - root / "alertmanager-config.yaml", - root / "alertmanager-config-email.example.yaml", - root / "backend-service-monitor.yaml", - root / "traefik-service-monitor.yaml", - root / "datastore-pod-monitors.yaml", - root / "logs" / "loki-values.yaml", - root / "logs" / "alloy-values.yaml", - root / "grafana-dashboard-codeplace.yaml", - root / "grafana-dashboard-ai-inference.yaml", - root / "grafana-dashboard-kubernetes-events.yaml", - root / "grafana-dashboard-logs.yaml", - root / "grafana-dashboard-monitoring-stack.yaml", - root / "grafana-dashboard-public-endpoints.yaml", - root / "grafana-dashboard-storage.yaml", - root / "grafana-dashboard-traces.yaml", -] - -for path in yaml_paths: - text = path.read_text() - if yaml is None: - if path.name.startswith("grafana-dashboard-"): - print(f"missing Python module yaml; cannot extract dashboard JSON from {path}", file=sys.stderr) - sys.exit(1) - print(f"SKIP YAML parse {path}: Python module yaml not found") - continue - - documents = list(yaml.safe_load_all(text)) - print(f"YAML OK {path}") - if path.name.startswith("grafana-dashboard-"): - data = documents[0] - for key, value in data.get("data", {}).items(): - json.loads(value) - print(f"JSON OK {path}:{key}") - -if yaml is None: - sys.exit(0) - -rules = list(yaml.safe_load_all((root / "prometheus-rules.yaml").read_text()))[0] -rule_groups = {group["name"]: group for group in rules["spec"]["groups"]} -expected_intervals = { - "codeplace.p0": "15s", - "codeplace.p1": "30s", -} -for group_name, interval in expected_intervals.items(): - actual = rule_groups.get(group_name, {}).get("interval") - if actual != interval: - raise SystemExit(f"{group_name} interval must be {interval}, got {actual}") - -alert_instances = {} -for group in rules["spec"]["groups"]: - expected_priority = "P0" if group["name"].endswith(".p0") else "P1" if group["name"].endswith(".p1") else None - for rule in group.get("rules", []): - if rule.get("record"): - if not rule.get("expr"): - raise SystemExit(f"{rule['record']} is missing expr") - continue - alert_name = rule.get("alert") - if not alert_name: - raise SystemExit(f"rule in {group['name']} is missing alert name") - if not rule.get("expr"): - raise SystemExit(f"{alert_name} is missing expr") - labels = rule.get("labels", {}) - annotations = rule.get("annotations", {}) - if labels.get("severity") not in {"critical", "warning"}: - raise SystemExit(f"{alert_name} is missing valid severity") - if labels.get("priority") not in {"P0", "P1"}: - raise SystemExit(f"{alert_name} is missing valid priority") - if expected_priority and labels.get("priority") != expected_priority: - raise SystemExit(f"{alert_name} priority must match {group['name']}") - for key in ("summary", "description"): - if not annotations.get(key): - raise SystemExit(f"{alert_name} is missing annotation {key}") - alert_instances.setdefault(alert_name, []).append(labels) - -for alert_name, labels_list in alert_instances.items(): - if len(labels_list) > 1 and any("namespace" not in labels for labels in labels_list): - raise SystemExit(f"duplicate alert {alert_name} must include static namespace labels") -required_alerts = { - "SubmissionCreateSystemFailures", - "JudgeTaskFailures", - "CeleryMinuteScheduledTaskStale", - "CeleryDailyScheduledTaskStale", - "CeleryWeeklyScheduledTaskStale", - "AIHintAPIFailures", - "AIHintBackendErrors", - "AIHintBackendLatencyHigh", -} -missing_alerts = required_alerts - set(alert_instances) -if missing_alerts: - raise SystemExit(f"missing required observability alerts: {sorted(missing_alerts)}") -records = { - rule.get("record") - for group in rules["spec"]["groups"] - for rule in group.get("rules", []) - if rule.get("record") -} -required_records = { - "codeplace:ingress_request_rate2m", - "codeplace:ingress_5xx_rate2m", -} -missing_records = required_records - records -if missing_records: - raise SystemExit(f"missing required observability recording rules: {sorted(missing_records)}") -print("PROMETHEUS RULE SHAPE OK") - -alertmanager = list(yaml.safe_load_all((root / "alertmanager-config.yaml").read_text()))[0] -route = alertmanager["spec"]["route"] -if set(route.get("groupBy", [])) != {"alertname", "priority", "namespace"}: - raise SystemExit("Alertmanager groupBy must be alertname, priority, namespace") -receivers = {receiver["name"]: receiver for receiver in alertmanager["spec"]["receivers"]} -for receiver_name in ("p0-discord", "p1-discord"): - receiver = receivers.get(receiver_name) - if not receiver: - raise SystemExit(f"missing receiver {receiver_name}") - configs = receiver.get("discordConfigs") or [] - if not configs: - raise SystemExit(f"{receiver_name} must define discordConfigs") - api_url = configs[0].get("apiURL", {}) - if api_url.get("name") != "alertmanager-contact-points" or api_url.get("key") != "webhook-url": - raise SystemExit(f"{receiver_name} must use alertmanager-contact-points/webhook-url") -print("ALERTMANAGER SHAPE OK") - -stack_values = yaml.safe_load((root / "kube-prometheus-stack-values.yaml").read_text()) -prometheus_spec = stack_values.get("prometheus", {}).get("prometheusSpec", {}) -for key in ( - "serviceMonitorSelectorNilUsesHelmValues", - "podMonitorSelectorNilUsesHelmValues", - "probeSelectorNilUsesHelmValues", - "ruleSelectorNilUsesHelmValues", -): - if prometheus_spec.get(key) is not False: - raise SystemExit(f"kube-prometheus-stack prometheusSpec.{key} must be false") -alertmanager_selector = ( - stack_values.get("alertmanager", {}) - .get("alertmanagerSpec", {}) - .get("alertmanagerConfigSelector", {}) - .get("matchLabels", {}) -) -if alertmanager_selector.get("alertmanagerConfig") != "codeplace": - raise SystemExit("AlertmanagerConfig selector must match alertmanagerConfig=codeplace") -alertmanager_matcher_strategy = ( - stack_values.get("alertmanager", {}) - .get("alertmanagerSpec", {}) - .get("alertmanagerConfigMatcherStrategy", {}) - .get("type") -) -if alertmanager_matcher_strategy != "None": - raise SystemExit("AlertmanagerConfig matcher strategy must be None for cross-namespace CodePlace alerts") -grafana_values = stack_values.get("grafana", {}) -datasources = grafana_values.get("additionalDataSources") or [] -datasource_by_name = {ds.get("name"): ds for ds in datasources if ds.get("name")} -if not any(ds.get("name") == "Loki" and ds.get("uid") == "Loki" and ds.get("type") == "loki" for ds in datasources): - raise SystemExit("Grafana values must provision Loki datasource") -if not any(ds.get("name") == "Tempo" and ds.get("uid") == "Tempo" and ds.get("type") == "tempo" for ds in datasources): - raise SystemExit("Grafana values must provision Tempo datasource") -tempo_datasource = datasource_by_name.get("Tempo", {}) -if tempo_datasource.get("jsonData", {}).get("tracesToLogsV2", {}).get("datasourceUid") != "Loki": - raise SystemExit("Grafana Tempo datasource must link tracesToLogsV2 to Loki") -dashboard_sidecar = grafana_values.get("sidecar", {}).get("dashboards", {}) -if dashboard_sidecar.get("label") != "grafana_dashboard": - raise SystemExit("Grafana dashboard sidecar must select grafana_dashboard label") -print("KUBE-PROMETHEUS-STACK VALUES SHAPE OK") - -traefik_config = yaml.safe_load((Path("kubernetes/setup/traefik/traefik-config.yaml")).read_text()) -traefik_values = yaml.safe_load(traefik_config.get("spec", {}).get("valuesContent", "")) -traefik_prometheus = traefik_values.get("metrics", {}).get("prometheus", {}) -if traefik_prometheus.get("addServicesLabels") is not True: - raise SystemExit("Traefik Prometheus metrics must include service labels for ingress 5xx alerts") -if traefik_prometheus.get("service", {}).get("enabled") is not True: - raise SystemExit("Traefik Prometheus metrics service must be enabled for ServiceMonitor scraping") -print("TRAEFIK METRICS SHAPE OK") - -loki_values = yaml.safe_load((root / "logs" / "loki-values.yaml").read_text()) -loki_config = loki_values.get("loki", {}) -if loki_values.get("deploymentMode") != "SingleBinary": - raise SystemExit("Loki must stay in SingleBinary mode for the on-prem baseline") -if loki_config.get("storage", {}).get("type") != "filesystem": - raise SystemExit("Loki must use filesystem storage while cloud object storage is unavailable") -if loki_config.get("commonConfig", {}).get("replication_factor") != 1: - raise SystemExit("Loki single-binary filesystem storage must keep replication_factor=1") -persistence = loki_values.get("singleBinary", {}).get("persistence", {}) -if not persistence.get("enabled"): - raise SystemExit("Loki singleBinary persistence must be enabled") -if persistence.get("storageClass") != "longhorn": - raise SystemExit("Loki persistence must use Longhorn storageClass") -if persistence.get("size") != "50Gi": - raise SystemExit("Loki PVC size must stay explicit at 50Gi") -limits = loki_config.get("limits_config", {}) -if limits.get("retention_period") != "168h": - raise SystemExit("Loki default retention must be 168h") -retention_by_selector = { - stream.get("selector"): stream.get("period") - for stream in limits.get("retention_stream", []) -} -if retention_by_selector.get('{namespace="code-place-dev"}') != "72h": - raise SystemExit("Loki dev retention must be 72h") -if retention_by_selector.get('{namespace="code-place-prod"}') != "168h": - raise SystemExit("Loki prod retention must be 168h") -if loki_values.get("minio", {}).get("enabled") is not False: - raise SystemExit("Loki chart MinIO subchart must stay disabled") -print("LOKI ON-PREM STORAGE SHAPE OK") - -alloy_values = yaml.safe_load((root / "logs" / "alloy-values.yaml").read_text()) -if alloy_values.get("alloy", {}).get("mounts", {}).get("varlog") is not True: - raise SystemExit("Alloy must mount host /var/log for /var/log/pods log collection") -alloy_config = alloy_values.get("alloy", {}).get("configMap", {}).get("content", "") -if "/var/log/pods/" not in alloy_config: - raise SystemExit("Alloy config must read Kubernetes pod logs from /var/log/pods") -if "loki-gateway.monitoring.svc.cluster.local" not in alloy_config: - raise SystemExit("Alloy config must write to the in-cluster Loki gateway") -if '"__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app", "__meta_kubernetes_pod_label_app_kubernetes_io_name"' not in alloy_config: - raise SystemExit("Alloy keep rule must consider namespace, app, and app.kubernetes.io/name labels") -if "regex = \"(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik\"" not in alloy_config: - raise SystemExit("Alloy keep rule must collect CodePlace/monitoring namespaces and kube-system Traefik logs") -print("ALLOY LOG COLLECTION SHAPE OK") - -otel_documents = list(yaml.safe_load_all((root / "otel-collector.yaml").read_text())) -otel_by_kind_name = { - (document.get("kind"), document.get("metadata", {}).get("name")): document - for document in otel_documents - if document -} -if ("ServiceAccount", "otel-collector") not in otel_by_kind_name: - raise SystemExit("otel-collector must run with a dedicated ServiceAccount") -cluster_role = otel_by_kind_name.get(("ClusterRole", "otel-collector"), {}) -rules_by_group = { - tuple(rule.get("apiGroups", [])): set(rule.get("resources", [])) - for rule in cluster_role.get("rules", []) -} -if not {"pods", "namespaces"}.issubset(rules_by_group.get(("",), set())): - raise SystemExit("otel-collector ClusterRole must read pods and namespaces for k8s trace attributes") -if "replicasets" not in rules_by_group.get(("apps",), set()): - raise SystemExit("otel-collector ClusterRole must read replicasets for deployment trace attributes") -cluster_role_binding = otel_by_kind_name.get(("ClusterRoleBinding", "otel-collector"), {}) -subjects = cluster_role_binding.get("subjects", []) -if not any(subject.get("kind") == "ServiceAccount" and subject.get("name") == "otel-collector" for subject in subjects): - raise SystemExit("otel-collector ClusterRoleBinding must bind the otel-collector ServiceAccount") -otel_deployment = otel_by_kind_name.get(("Deployment", "otel-collector"), {}) -if otel_deployment.get("spec", {}).get("template", {}).get("spec", {}).get("serviceAccountName") != "otel-collector": - raise SystemExit("otel-collector Deployment must use the otel-collector ServiceAccount") -otel_config = otel_by_kind_name.get(("ConfigMap", "otel-collector-config"), {}).get("data", {}).get("config.yaml", "") -otel_config_yaml = yaml.safe_load(otel_config) -processors = otel_config_yaml.get("processors", {}) -if "k8sattributes" not in processors: - raise SystemExit("otel-collector must enrich traces with k8sattributes") -k8s_metadata = set(processors["k8sattributes"].get("extract", {}).get("metadata", [])) -required_k8s_metadata = {"k8s.namespace.name", "k8s.pod.name", "k8s.deployment.name", "k8s.node.name"} -if not required_k8s_metadata.issubset(k8s_metadata): - raise SystemExit("otel-collector k8sattributes must extract namespace, pod, deployment, and node") -trace_processors = ( - otel_config_yaml.get("service", {}) - .get("pipelines", {}) - .get("traces", {}) - .get("processors", []) -) -if "k8sattributes" not in trace_processors: - raise SystemExit("otel-collector traces pipeline must include k8sattributes") -print("OTEL TRACE ENRICHMENT SHAPE OK") - -frontend_nginx = Path("frontend/deploy/kube_nginx/nginx.conf").read_text() -frontend_proxy = Path("frontend/deploy/kube_nginx/api_proxy.conf").read_text() -frontend_entrypoint = Path("frontend/deploy/entrypoint.sh").read_text() -required_frontend_nginx_parts = [ - "error_log /dev/stderr", - "log_format main escape=json", - "\"logger\":\"frontend.nginx\"", - "\"request_id\":\"$codeplace_request_id\"", - "\"status_code\":$status", - "access_log /dev/stdout main", - "add_header X-Request-ID $codeplace_request_id always", -] -for text in required_frontend_nginx_parts: - if text not in frontend_nginx: - raise SystemExit(f"frontend Kubernetes nginx config missing {text}") -if "$http_referer" in frontend_nginx: - raise SystemExit("frontend Kubernetes nginx access log must not include Referer because it can contain tokens") -if "proxy_set_header X-Request-ID $codeplace_request_id;" not in frontend_proxy: - raise SystemExit("frontend Kubernetes nginx proxy must propagate X-Request-ID") -if "exec nginx -c /app/deploy/kube_nginx/nginx.conf" not in frontend_entrypoint: - raise SystemExit("frontend entrypoint must use Kubernetes nginx config") -print("FRONTEND NGINX LOGGING SHAPE OK") - -backend_sm = yaml.safe_load((root / "backend-service-monitor.yaml").read_text()) -if backend_sm.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": - raise SystemExit("backend ServiceMonitor must keep release=kube-prometheus-stack label") -endpoint = (backend_sm.get("spec", {}).get("endpoints") or [{}])[0] -if endpoint.get("port") != "api" or endpoint.get("path") != "/metrics" or endpoint.get("interval") != "15s": - raise SystemExit("backend ServiceMonitor must scrape api /metrics every 15s") -if backend_sm.get("spec", {}).get("selector", {}).get("matchLabels", {}).get("app") != "backend": - raise SystemExit("backend ServiceMonitor must select app=backend") - -traefik_sm = yaml.safe_load((root / "traefik-service-monitor.yaml").read_text()) -if traefik_sm.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": - raise SystemExit("traefik ServiceMonitor must keep release=kube-prometheus-stack label") -traefik_namespaces = traefik_sm.get("spec", {}).get("namespaceSelector", {}).get("matchNames", []) -if "kube-system" not in traefik_namespaces: - raise SystemExit("traefik ServiceMonitor must scrape kube-system") -traefik_endpoint = (traefik_sm.get("spec", {}).get("endpoints") or [{}])[0] -if traefik_endpoint.get("port") != "metrics" or traefik_endpoint.get("path") != "/metrics" or traefik_endpoint.get("interval") != "15s": - raise SystemExit("traefik ServiceMonitor must scrape metrics /metrics every 15s") -if traefik_sm.get("spec", {}).get("selector", {}).get("matchLabels", {}).get("app.kubernetes.io/name") != "traefik": - raise SystemExit("traefik ServiceMonitor must select app.kubernetes.io/name=traefik") - -pod_monitors = list(yaml.safe_load_all((root / "datastore-pod-monitors.yaml").read_text())) -for monitor in pod_monitors: - name = monitor.get("metadata", {}).get("name") - if monitor.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack": - raise SystemExit(f"{name} PodMonitor must keep release=kube-prometheus-stack label") - endpoint = (monitor.get("spec", {}).get("podMetricsEndpoints") or [{}])[0] - if endpoint.get("interval") != "30s": - raise SystemExit(f"{name} PodMonitor must scrape every 30s") -print("SCRAPE RESOURCE SHAPE OK") - -kustomization = yaml.safe_load((root / "kustomization.yaml").read_text()) -if "./alertmanager-config-email.example.yaml" in kustomization.get("resources", []): - raise SystemExit("email fallback example must not be included in default kustomization") -print("MONITORING KUSTOMIZATION SHAPE OK") - -all_dashboard_paths = sorted(root.glob("grafana-dashboard-*.yaml")) -provisioned_datasources = {"Prometheus", *datasource_by_name} -dashboard_datasources = {} -for dashboard_path in all_dashboard_paths: - dashboard_map = yaml.safe_load(dashboard_path.read_text()) - for key, value in dashboard_map.get("data", {}).items(): - dashboard = json.loads(value) - if not dashboard.get("uid") or not dashboard.get("title"): - raise SystemExit(f"{dashboard_path}:{key} must define uid and title") - if dashboard.get("refresh") != "30s": - raise SystemExit(f"{dashboard_path}:{key} refresh must be 30s") - panels = dashboard.get("panels") or [] - if not panels: - raise SystemExit(f"{dashboard_path}:{key} must contain panels") - datasource_names = set() - for panel in panels: - if not panel.get("title") or not panel.get("type") or not panel.get("gridPos"): - raise SystemExit(f"{dashboard_path}:{key} has panel missing title/type/gridPos") - datasource = panel.get("datasource") - if isinstance(datasource, dict): - datasource = datasource.get("uid") or datasource.get("type") - if datasource: - datasource_names.add(datasource) - if datasource not in provisioned_datasources: - raise SystemExit(f"{dashboard_path}:{key}:{panel.get('title')} uses unknown datasource {datasource}") - for target in panel.get("targets", []): - if not target.get("expr"): - raise SystemExit(f"{dashboard_path}:{key}:{panel.get('title')} has target missing expr") - dashboard_datasources[dashboard_path.name] = datasource_names - -required_dashboard_datasources = { - "grafana-dashboard-logs.yaml": {"Prometheus", "Loki"}, - "grafana-dashboard-kubernetes-events.yaml": {"Prometheus", "Loki"}, - "grafana-dashboard-traces.yaml": {"Prometheus"}, - "grafana-dashboard-codeplace.yaml": set(), - "grafana-dashboard-ai-inference.yaml": {"Prometheus"}, - "grafana-dashboard-monitoring-stack.yaml": {"Prometheus"}, - "grafana-dashboard-public-endpoints.yaml": {"Prometheus"}, - "grafana-dashboard-storage.yaml": {"Prometheus"}, -} -for dashboard_name, required_datasources in required_dashboard_datasources.items(): - actual_datasources = dashboard_datasources.get(dashboard_name) - if actual_datasources is None: - raise SystemExit(f"missing dashboard {dashboard_name}") - missing_datasources = required_datasources - actual_datasources - if missing_datasources: - raise SystemExit(f"{dashboard_name} must use datasources {sorted(missing_datasources)}") - -dashboard_requirements = { - "grafana-dashboard-codeplace.yaml": { - "Submission / Judge Outcomes": [ - "codeplace_submission_create_outcome_total", - "codeplace_judge_task_outcome_total", - ], - "Celery Task Runtime / Freshness": [ - "codeplace_celery_task_last_success_age_seconds", - ], - }, - "grafana-dashboard-ai-inference.yaml": { - "AI Hint API Outcomes": ["codeplace_ai_hint_api_outcome_total"], - "Backend AI Hint Streams": ["codeplace_ai_hint_requests_total"], - "Backend AI Hint Duration": ["codeplace_ai_hint_duration_seconds_bucket"], - }, -} -dashboard_by_name = {path.name: yaml.safe_load(path.read_text()) for path in all_dashboard_paths} -logs_dashboard = [ - json.loads(value) - for value in dashboard_by_name["grafana-dashboard-logs.yaml"].get("data", {}).values() -][0] -logs_namespace_variables = [ - variable - for variable in logs_dashboard.get("templating", {}).get("list", []) - if variable.get("name") == "namespace" -] -if len(logs_namespace_variables) != 1: - raise SystemExit("grafana-dashboard-logs.yaml must define exactly one namespace variable") -if logs_namespace_variables[0].get("allValue") != "code-place-dev|code-place-prod|monitoring": - raise SystemExit("CodePlace Logs dashboard namespace All value must include monitoring logs") -logs_panels = { - panel.get("title"): panel - for panel in logs_dashboard.get("panels", []) -} -for title in ("Loki Ready", "Alloy Nodes", "Loki PVC Usage"): - panel = logs_panels.get(title) - if not panel: - raise SystemExit(f"grafana-dashboard-logs.yaml missing panel {title}") - exprs = "\n".join(target.get("expr", "") for target in panel.get("targets", [])) - if "or vector(0)" not in exprs: - raise SystemExit(f"grafana-dashboard-logs.yaml:{title} must avoid empty stat panels with or vector(0)") -traces_dashboard = [ - json.loads(value) - for value in dashboard_by_name["grafana-dashboard-traces.yaml"].get("data", {}).values() -][0] -traces_panels = { - panel.get("title"): panel - for panel in traces_dashboard.get("panels", []) -} -for title in ("Tempo Ready", "OTel Collector Ready", "Trace Receive Rate", "Trace Export Failures", "Tempo PVC Usage"): - panel = traces_panels.get(title) - if not panel: - raise SystemExit(f"grafana-dashboard-traces.yaml missing panel {title}") - exprs = "\n".join(target.get("expr", "") for target in panel.get("targets", [])) - if "or vector(0)" not in exprs: - raise SystemExit(f"grafana-dashboard-traces.yaml:{title} must avoid empty stat panels with or vector(0)") -for dashboard_name, panel_requirements in dashboard_requirements.items(): - dashboard_map = dashboard_by_name.get(dashboard_name) - if not dashboard_map: - raise SystemExit(f"missing dashboard {dashboard_name}") - dashboards = [json.loads(value) for value in dashboard_map.get("data", {}).values()] - panels = { - panel.get("title"): panel - for dashboard in dashboards - for panel in dashboard.get("panels", []) - } - for title, required_expr_parts in panel_requirements.items(): - panel = panels.get(title) - if not panel: - raise SystemExit(f"{dashboard_name} missing panel {title}") - exprs = "\n".join(target.get("expr", "") for target in panel.get("targets", [])) - for expr_part in required_expr_parts: - if expr_part not in exprs: - raise SystemExit(f"{dashboard_name}:{title} missing expr containing {expr_part}") -print("GRAFANA DASHBOARD SHAPE OK") -PY - -echo "==> rendering kustomize monitoring manifests" -kubectl kustomize "${MONITORING_DIR}" >/tmp/codeplace-monitoring-render.yaml -echo "KUSTOMIZE OK kubernetes/monitoring" - -echo "==> rendering application overlays for observability wiring" -for overlay in dev prod; do - kubectl kustomize "${ROOT_DIR}/kubernetes/overlays/${overlay}" >"/tmp/codeplace-${overlay}-render.yaml" - echo "KUSTOMIZE OK kubernetes/overlays/${overlay}" -done - -python3 - <<'PY' -from pathlib import Path -import sys - -try: - import yaml -except ImportError: - print("missing Python module yaml; cannot validate rendered application overlays", file=sys.stderr) - sys.exit(1) - - -def docs_for(path): - return [ - doc for doc in yaml.safe_load_all(Path(path).read_text()) - if isinstance(doc, dict) and doc.get("kind") - ] - - -def find_one(docs, kind, name, source): - matches = [ - doc for doc in docs - if doc.get("kind") == kind and doc.get("metadata", {}).get("name") == name - ] - if len(matches) != 1: - raise SystemExit(f"{source} expected one {kind}/{name}, got {len(matches)}") - return matches[0] - - -def env_map(deployment, container_name): - containers = deployment["spec"]["template"]["spec"].get("containers", []) - for container in containers: - if container.get("name") == container_name: - return { - env.get("name"): env.get("value") - for env in container.get("env", []) - if env.get("name") - } - raise SystemExit(f"{deployment['metadata']['name']} missing container {container_name}") - - -def require_labels(name, actual, expected): - for key, value in expected.items(): - if actual.get(key) != value: - raise SystemExit(f"{name} label {key} expected={value} actual={actual.get(key)}") - - -def require_service_targets_deployment(service, deployment, source): - service_name = service["metadata"]["name"] - service_selector = service.get("spec", {}).get("selector", {}) - pod_labels = deployment.get("spec", {}).get("template", {}).get("metadata", {}).get("labels", {}) - require_labels(f"{source} Service/{service_name} selector target", pod_labels, service_selector) - - -backend_service_monitor = yaml.safe_load(Path("kubernetes/monitoring/backend-service-monitor.yaml").read_text()) -backend_service_monitor_selector = backend_service_monitor.get("spec", {}).get("selector", {}).get("matchLabels", {}) -vllm_service_monitor = yaml.safe_load(Path("kubernetes/monitoring/vllm-service-monitor.yaml").read_text()) -vllm_service_monitor_selector = vllm_service_monitor.get("spec", {}).get("selector", {}).get("matchLabels", {}) - - -for overlay in ("dev", "prod"): - path = f"/tmp/codeplace-{overlay}-render.yaml" - docs = docs_for(path) - backend_service = find_one(docs, "Service", "backend", path) - backend_deployment = find_one(docs, "Deployment", "backend", path) - port_names = {port.get("name") for port in backend_service.get("spec", {}).get("ports", [])} - if "api" not in port_names: - raise SystemExit(f"{overlay} backend Service must expose port name api for ServiceMonitor") - require_labels(f"{overlay} backend ServiceMonitor selector", backend_service.get("metadata", {}).get("labels", {}), backend_service_monitor_selector) - require_service_targets_deployment(backend_service, backend_deployment, overlay) - - frontend_deployment = find_one(docs, "Deployment", "frontend", path) - frontend_service = find_one(docs, "Service", "frontend", path) - require_service_targets_deployment(frontend_service, frontend_deployment, overlay) - frontend_env = env_map(frontend_deployment, "frontend") - if frontend_env.get("FORCE_HTTPS") != "0": - raise SystemExit(f"{overlay} frontend must keep FORCE_HTTPS=0 because Traefik terminates TLS") - - if overlay == "prod": - vllm_service = find_one(docs, "Service", "vllm", path) - vllm_deployment = find_one(docs, "Deployment", "vllm", path) - require_labels(f"{overlay} vLLM ServiceMonitor selector", vllm_service.get("metadata", {}).get("labels", {}), vllm_service_monitor_selector) - require_service_targets_deployment(vllm_service, vllm_deployment, overlay) - - for deployment_name, container_name in ( - ("backend", "backend"), - ("celery-worker", "celery-worker"), - ("celery-beat", "celery-beat"), - ): - deployment = find_one(docs, "Deployment", deployment_name, path) - env = env_map(deployment, container_name) - if env.get("JSON_LOGGING") != "1": - raise SystemExit(f"{overlay} {deployment_name} must keep JSON_LOGGING=1") - if env.get("OTEL_ENABLED") is None: - raise SystemExit(f"{overlay} {deployment_name} must define OTEL_ENABLED") - if env.get("OTEL_EXPORTER_OTLP_ENDPOINT") != "http://otel-collector.monitoring.svc.cluster.local:4317": - raise SystemExit(f"{overlay} {deployment_name} must send OTLP to monitoring otel-collector") - -print("APPLICATION OBSERVABILITY WIRING OK") -PY - -if optional_cmd promtool; then - echo "==> checking Prometheus rules with promtool" - python3 - <<'PY' -from pathlib import Path -import yaml - -prometheus_rule = yaml.safe_load(Path("kubernetes/monitoring/prometheus-rules.yaml").read_text()) -Path("/tmp/codeplace-prometheus-rules-groups.yaml").write_text( - yaml.safe_dump({"groups": prometheus_rule["spec"]["groups"]}, sort_keys=False) -) -PY - promtool check rules /tmp/codeplace-prometheus-rules-groups.yaml -else - echo "SKIP promtool check: promtool not found" -fi - -if optional_cmd helm; then - echo "==> rendering Helm charts" - helm template kube-prometheus-stack prometheus-community/kube-prometheus-stack \ - --namespace monitoring \ - --version 86.3.1 \ - --values "${MONITORING_DIR}/kube-prometheus-stack-values.yaml" \ - >/tmp/codeplace-kube-prometheus-stack-render.yaml - echo "HELM OK kube-prometheus-stack" - - helm template loki grafana/loki \ - --namespace monitoring \ - --version 6.55.0 \ - --values "${MONITORING_DIR}/logs/loki-values.yaml" \ - >/tmp/codeplace-loki-render.yaml - echo "HELM OK loki" - - helm template alloy grafana/alloy \ - --namespace monitoring \ - --version 1.10.0 \ - --values "${MONITORING_DIR}/logs/alloy-values.yaml" \ - >/tmp/codeplace-alloy-render.yaml - echo "HELM OK alloy" -else - echo "SKIP helm template: helm not found" -fi - -echo "observability validation completed" From 64539fe2e10e27dd3ba9149c5da63f3a8959bd02 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:29:57 +0900 Subject: [PATCH 085/147] =?UTF-8?q?=EA=B3=B5=EA=B0=9C=20=EC=97=94=EB=93=9C?= =?UTF-8?q?=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=ED=94=84=EB=A1=9C=EB=B8=8C=20?= =?UTF-8?q?=EB=B6=84=EC=82=B0=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/blackbox-exporter.yaml | 19 ++++++++++++++++++- .../grafana-dashboard-public-endpoints.yaml | 8 +++++--- kubernetes/monitoring/logs/README.md | 2 +- kubernetes/monitoring/prometheus-rules.yaml | 10 ++++++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/kubernetes/monitoring/blackbox-exporter.yaml b/kubernetes/monitoring/blackbox-exporter.yaml index c013525c..8b7d773f 100644 --- a/kubernetes/monitoring/blackbox-exporter.yaml +++ b/kubernetes/monitoring/blackbox-exporter.yaml @@ -27,7 +27,7 @@ metadata: labels: app: blackbox-exporter spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: blackbox-exporter @@ -36,6 +36,13 @@ spec: labels: app: blackbox-exporter spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: blackbox-exporter containers: - name: blackbox-exporter image: quay.io/prometheus/blackbox-exporter:v0.27.0 @@ -72,6 +79,16 @@ spec: configMap: name: blackbox-exporter-config --- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: blackbox-exporter +spec: + minAvailable: 1 + selector: + matchLabels: + app: blackbox-exporter +--- apiVersion: v1 kind: Service metadata: diff --git a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml index 87519b57..6ab3c8d3 100644 --- a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml +++ b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml @@ -133,7 +133,7 @@ data: }, { "type": "stat", - "title": "Blackbox Exporter Ready", + "title": "Blackbox Exporter Replicas", "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -143,14 +143,16 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} ] } } }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"blackbox-exporter-.*\"})", "legendFormat": "ready"} + {"expr": "kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"blackbox-exporter\"} or vector(0)", "legendFormat": "available"}, + {"expr": "kube_deployment_spec_replicas{namespace=\"monitoring\", deployment=\"blackbox-exporter\"} or vector(0)", "legendFormat": "desired"} ] }, { diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 3ebbee50..bd10ed20 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -76,7 +76,7 @@ Keep these invariants unless the storage design changes intentionally: - Loki stays in `SingleBinary` deployment mode. - Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. -- In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors can be replicated across nodes. +- In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors and probe exporters can be replicated across nodes. - Alloy keeps namespace-based collection for `code-place-dev`, `code-place-prod`, and `monitoring`; do not depend on `app.kubernetes.io/name` for CodePlace app logs because the application manifests primarily use `app`. - Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. - The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index a1b876ea..70f45165 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -347,6 +347,16 @@ spec: annotations: summary: "Blackbox exporter is not ready" description: "The synthetic public endpoint probe exporter has not been ready for 2 minutes." + - alert: BlackboxExporterReplicaUnavailable + expr: kube_deployment_status_replicas_available{namespace="monitoring", deployment="blackbox-exporter"} < kube_deployment_spec_replicas{namespace="monitoring", deployment="blackbox-exporter"} + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Blackbox exporter replica is unavailable" + description: "At least one synthetic public endpoint probe exporter replica is unavailable for 5 minutes. Public endpoint probing is degraded." - alert: GrafanaPublicEndpointDown expr: probe_success{environment="monitoring", service="grafana", probe_type="public-http"} == 0 for: 2m From bbc40eeae3b18357d609e59e3d6306e9d050b2c4 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:34:21 +0900 Subject: [PATCH 086/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?=EC=8A=A4=ED=83=9D=20=EC=83=81=ED=83=9C=20=EC=A0=80=EC=9E=A5=20?= =?UTF-8?q?=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grafana-dashboard-monitoring-stack.yaml | 22 +++++++++++++++ .../kube-prometheus-stack-values.yaml | 27 +++++++++++++++++++ kubernetes/monitoring/logs/README.md | 1 + kubernetes/monitoring/prometheus-rules.yaml | 10 +++++++ 4 files changed, 60 insertions(+) diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index af4c53db..5ba518ec 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -245,6 +245,28 @@ data: {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*|grafana-.*\", container!=\"\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*|grafana-.*\", container!=\"\"})", "legendFormat": "{{pod}} memory bytes"} ] + }, + { + "type": "timeseries", + "title": "Monitoring PVC Usage", + "gridPos": {"x": 0, "y": 28, "w": 24, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "max by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim!~\".*loki.*|tempo-data\"}) / clamp_min(max by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim!~\".*loki.*|tempo-data\"}), 1) or vector(0)", "legendFormat": "{{persistentvolumeclaim}}"} + ] } ] } diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index 3101e792..d1dd5a08 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -2,10 +2,21 @@ prometheus: prometheusSpec: evaluationInterval: 15s scrapeInterval: 15s + retention: 15d + retentionSize: 45GB serviceMonitorSelectorNilUsesHelmValues: false podMonitorSelectorNilUsesHelmValues: false probeSelectorNilUsesHelmValues: false ruleSelectorNilUsesHelmValues: false + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi alertmanager: alertmanagerSpec: @@ -17,8 +28,24 @@ alertmanager: alertmanagerConfigNamespaceSelector: matchLabels: kubernetes.io/metadata.name: monitoring + storage: + volumeClaimTemplate: + spec: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi grafana: + persistence: + enabled: true + type: pvc + storageClassName: longhorn + accessModes: + - ReadWriteOnce + size: 10Gi additionalDataSources: - name: Loki uid: Loki diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index bd10ed20..1bcc8d23 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -77,6 +77,7 @@ Keep these invariants unless the storage design changes intentionally: - Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. - In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors and probe exporters can be replicated across nodes. +- kube-prometheus-stack Prometheus, Alertmanager, and Grafana also use Longhorn PVCs so metric data, silences, and UI state survive pod rescheduling. - Alloy keeps namespace-based collection for `code-place-dev`, `code-place-prod`, and `monitoring`; do not depend on `app.kubernetes.io/name` for CodePlace app logs because the application manifests primarily use `app`. - Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. - The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 70f45165..700065f1 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -743,6 +743,16 @@ spec: annotations: summary: "Loki PVC usage is above 85%" description: "The Loki Longhorn PVC has been above 85% usage for 10 minutes. Log retention or PVC size must be adjusted before ingestion stops." + - alert: MonitoringPVCAlmostFull + expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim!~".*loki.*|tempo-data"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim!~".*loki.*|tempo-data"}), 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Monitoring PVC usage is above 85%" + description: "A Prometheus, Alertmanager, Grafana, or monitoring support PVC has been above 85% usage for 10 minutes. Check retention, WAL growth, and Longhorn capacity." - alert: AlloyDaemonSetUnavailable expr: (kube_daemonset_status_number_available{namespace="monitoring", daemonset="alloy"} < kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="alloy"}) or absent(kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="alloy"}) for: 5m From b832c3f746279e4248f11453e8a3ad209a4561b8 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:36:39 +0900 Subject: [PATCH 087/147] =?UTF-8?q?Longhorn=20=EB=A7=A4=EB=8B=88=EC=A0=80?= =?UTF-8?q?=20=EB=B6=80=EB=B6=84=20=EC=9E=A5=EC=95=A0=20=EA=B4=80=EC=B8=A1?= =?UTF-8?q?=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/grafana-dashboard-storage.yaml | 8 +++++--- kubernetes/monitoring/prometheus-rules.yaml | 10 ++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/kubernetes/monitoring/grafana-dashboard-storage.yaml b/kubernetes/monitoring/grafana-dashboard-storage.yaml index e12e121e..43269d39 100644 --- a/kubernetes/monitoring/grafana-dashboard-storage.yaml +++ b/kubernetes/monitoring/grafana-dashboard-storage.yaml @@ -21,7 +21,7 @@ data: "panels": [ { "type": "stat", - "title": "Longhorn Managers Ready", + "title": "Longhorn Manager Replicas", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -31,14 +31,16 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "orange", "value": 1}, + {"color": "green", "value": 3} ] } } }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(kube_pod_status_ready{namespace=\"longhorn-system\", condition=\"true\", pod=~\"longhorn-manager-.*\"})", "legendFormat": "ready"} + {"expr": "kube_daemonset_status_number_available{namespace=\"longhorn-system\", daemonset=\"longhorn-manager\"} or vector(0)", "legendFormat": "available"}, + {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"longhorn-system\", daemonset=\"longhorn-manager\"} or vector(0)", "legendFormat": "desired"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 700065f1..67d4d7ff 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -228,6 +228,16 @@ spec: annotations: summary: "Longhorn managers are not ready" description: "No Longhorn manager pod has been ready for 2 minutes. PVC health and Longhorn metrics may be unavailable." + - alert: LonghornManagerReplicaUnavailable + expr: kube_daemonset_status_number_available{namespace="longhorn-system", daemonset="longhorn-manager"} < kube_daemonset_status_desired_number_scheduled{namespace="longhorn-system", daemonset="longhorn-manager"} + for: 5m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn manager replica is unavailable" + description: "At least one Longhorn manager replica is unavailable for 5 minutes. Storage health visibility and volume operations are degraded." - alert: LonghornVolumeFaulted expr: longhorn_volume_robustness == 3 for: 1m From 0e03428878afaf6b2fbe920dc181c9d6b73aed0e Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:40:12 +0900 Subject: [PATCH 088/147] =?UTF-8?q?=EB=88=84=EC=A0=81=20=EB=A9=94=ED=8A=B8?= =?UTF-8?q?=EB=A6=AD=20=EC=B9=B4=EC=9A=B4=ED=84=B0=20=ED=83=80=EC=9E=85=20?= =?UTF-8?q?=EC=A0=95=ED=95=A9=EC=84=B1=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_metrics.py | 14 +++++++------- backend/utils/tests.py | 10 +++++++--- .../infra/observability-improvement-plan/_index.md | 2 +- .../infra/observability-runbook/_index.md | 2 +- .../monitoring/grafana-dashboard-codeplace.yaml | 2 +- kubernetes/monitoring/prometheus-rules.yaml | 4 ++-- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 16b9bba4..20866120 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -5,7 +5,7 @@ from django.db.models import Count, Min from django.utils import timezone from prometheus_client import Counter, Histogram, REGISTRY -from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily +from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, HistogramMetricFamily from submission.models import JudgeStatus, Submission from utils.cache import cache @@ -117,8 +117,8 @@ def describe(self): "codeplace_submission_judge_duration_seconds", "Judge duration for submissions completed in the last 10 minutes.", ) - yield GaugeMetricFamily( - "codeplace_celery_task_count", + yield CounterMetricFamily( + "codeplace_celery_task_total", "Total Celery tasks grouped by task name and status.", labels=["task_name", "status"], ) @@ -156,7 +156,7 @@ def describe(self): "codeplace_redis_max_clients", "Configured Redis maxclients from CONFIG GET maxclients.", ) - yield GaugeMetricFamily( + yield CounterMetricFamily( "codeplace_redis_rejected_connections_total", "Total Redis rejected connections from INFO stats.", ) @@ -299,8 +299,8 @@ def _judge_duration_histogram(self): return metric def _celery_task_metrics(self): - total_metric = GaugeMetricFamily( - "codeplace_celery_task_count", + total_metric = CounterMetricFamily( + "codeplace_celery_task_total", "Total Celery tasks grouped by task name and status.", labels=["task_name", "status"], ) @@ -440,7 +440,7 @@ def _redis_health_metrics(self): "codeplace_redis_max_clients", "Configured Redis maxclients from CONFIG GET maxclients.", ) - rejected_connections_metric = GaugeMetricFamily( + rejected_connections_metric = CounterMetricFamily( "codeplace_redis_rejected_connections_total", "Total Redis rejected connections from INFO stats.", ) diff --git a/backend/utils/tests.py b/backend/utils/tests.py index fdd80de1..07603dc8 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -112,7 +112,8 @@ def test_celery_task_metrics_are_collected_from_redis(self): with patch("utils.observability_metrics.cache.hgetall", side_effect=lambda key: hashes.get(key, {})): samples = self._samples_by_metric(list(CodePlaceCollector()._celery_task_metrics())) - total_samples = samples["codeplace_celery_task_count"] + total_samples = samples["codeplace_celery_task"] + self.assertEqual(total_samples[0].name, "codeplace_celery_task_total") self.assertEqual(total_samples[0].labels, { "task_name": "judge.tasks.judge_task", "status": "success", @@ -159,7 +160,8 @@ def test_celery_task_metrics_tolerate_malformed_redis_values(self): with patch("utils.observability_metrics.cache.hgetall", side_effect=lambda key: hashes.get(key, {})): samples = self._samples_by_metric(list(CodePlaceCollector()._celery_task_metrics())) - total_sample = samples["codeplace_celery_task_count"][0] + total_sample = samples["codeplace_celery_task"][0] + self.assertEqual(total_sample.name, "codeplace_celery_task_total") self.assertEqual(total_sample.value, 0) last_runtime_sample = samples["codeplace_celery_task_last_runtime_seconds"][0] self.assertEqual(last_runtime_sample.value, 0) @@ -205,7 +207,9 @@ def test_redis_health_metrics_are_collected(self): self.assertEqual(samples["codeplace_redis_connected_clients"][0].value, 25) self.assertEqual(samples["codeplace_redis_max_clients"][0].value, 1000) - self.assertEqual(samples["codeplace_redis_rejected_connections_total"][0].value, 3) + rejected_sample = samples["codeplace_redis_rejected_connections"][0] + self.assertEqual(rejected_sample.name, "codeplace_redis_rejected_connections_total") + self.assertEqual(rejected_sample.value, 3) class CeleryObservabilityTest(SimpleTestCase): diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index aed8c12e..78967928 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -48,7 +48,7 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_judge_server_available{hostname}` - `codeplace_judge_server_last_heartbeat_age_seconds{hostname}` - `codeplace_judge_server_task_number{hostname}` -- `codeplace_celery_task_count{task_name,status}` +- `codeplace_celery_task_total{task_name,status}` - `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_celery_task_last_runtime_seconds{task_name,status}` - `codeplace_celery_task_last_seen_age_seconds{task_name,status}` diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 15d3a7f2..9c37fa99 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -704,7 +704,7 @@ Grafana Explore: 확인: ```promql -sum by (task_name, status) (increase(codeplace_celery_task_count{namespace=""}[10m])) +sum by (task_name, status) (increase(codeplace_celery_task_total{namespace=""}[10m])) histogram_quantile(0.95, sum by (task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=""}[10m]))) codeplace_celery_task_last_runtime_seconds{namespace=""} codeplace_celery_task_last_seen_age_seconds{namespace=""} diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 4e5d70be..5fee850a 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -221,7 +221,7 @@ data: "gridPos": {"x": 0, "y": 13, "w": 8, "h": 7}, "fieldConfig": {"defaults": {"unit": "ops"}}, "targets": [ - {"expr": "sum by (namespace, task_name, status) (rate(codeplace_celery_task_count{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{task_name}} {{status}}"} + {"expr": "sum by (namespace, task_name, status) (rate(codeplace_celery_task_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{task_name}} {{status}}"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 67d4d7ff..f6d1394d 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -500,7 +500,7 @@ spec: summary: "Celery beat is not ready" description: "The celery-beat pod has not been ready for 2 minutes." - alert: CeleryTaskFailures - expr: sum by (namespace, task_name) (increase(codeplace_celery_task_count{namespace=~"code-place-(dev|prod)", status="failure"}[10m])) > 0 + expr: sum by (namespace, task_name) (increase(codeplace_celery_task_total{namespace=~"code-place-(dev|prod)", status="failure"}[10m])) > 0 for: 2m labels: severity: warning @@ -509,7 +509,7 @@ spec: summary: "Celery task failures detected" description: "A Celery task has failed in the last 10 minutes." - alert: CeleryTaskRetries - expr: sum by (namespace, task_name) (increase(codeplace_celery_task_count{namespace=~"code-place-(dev|prod)", status="retry"}[10m])) > 3 + expr: sum by (namespace, task_name) (increase(codeplace_celery_task_total{namespace=~"code-place-(dev|prod)", status="retry"}[10m])) > 3 for: 5m labels: severity: warning From 0611776e2c3f412f688e0c81f9c98929aafebc01 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:43:29 +0900 Subject: [PATCH 089/147] =?UTF-8?q?=EA=B3=B5=EA=B0=9C=20=EC=97=94=EB=93=9C?= =?UTF-8?q?=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=ED=94=84=EB=A1=9C=EB=B8=8C=20?= =?UTF-8?q?=EB=88=84=EB=9D=BD=20=EC=95=8C=EB=A6=BC=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/prometheus-rules.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index f6d1394d..b70811ed 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -92,7 +92,7 @@ spec: summary: "Ingress 5xx rate is high in prod" description: "Traefik 5xx responses for code-place-prod are above 5% for 2 minutes." - alert: PublicEndpointDown - expr: probe_success{environment="prod", probe_type="public-http"} == 0 + expr: (probe_success{environment="prod", probe_type="public-http"} == 0) or absent(probe_success{environment="prod", service="frontend", probe_type="public-http"}) or absent(probe_success{environment="prod", service="hub-auth", probe_type="public-http"}) for: 1m labels: severity: critical @@ -311,7 +311,7 @@ spec: summary: "Backend API p95 latency is high" description: "Backend API p95 latency is above 2 seconds for 5 minutes." - alert: PublicEndpointDown - expr: probe_success{environment="dev", probe_type="public-http"} == 0 + expr: (probe_success{environment="dev", probe_type="public-http"} == 0) or absent(probe_success{environment="dev", service="frontend", probe_type="public-http"}) or absent(probe_success{environment="dev", service="hub-auth", probe_type="public-http"}) for: 2m labels: severity: warning @@ -368,7 +368,7 @@ spec: summary: "Blackbox exporter replica is unavailable" description: "At least one synthetic public endpoint probe exporter replica is unavailable for 5 minutes. Public endpoint probing is degraded." - alert: GrafanaPublicEndpointDown - expr: probe_success{environment="monitoring", service="grafana", probe_type="public-http"} == 0 + expr: (probe_success{environment="monitoring", service="grafana", probe_type="public-http"} == 0) or absent(probe_success{environment="monitoring", service="grafana", probe_type="public-http"}) for: 2m labels: severity: warning From 8ea437f50927f8074f53b5cc0d8337e6ab698358 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:47:37 +0900 Subject: [PATCH 090/147] =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EC=8A=A4?= =?UTF-8?q?=ED=86=A0=EC=96=B4=20HA=20=EC=95=8C=EB=A6=BC=20=EB=93=B1?= =?UTF-8?q?=EA=B8=89=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grafana-dashboard-codeplace.yaml | 14 ++++---- kubernetes/monitoring/prometheus-rules.yaml | 34 ++++++++++++++----- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 5fee850a..829e5211 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -417,7 +417,7 @@ data: }, { "type": "stat", - "title": "PostgreSQL Ready", + "title": "PostgreSQL Ready Replicas", "gridPos": {"x": 0, "y": 55, "w": 6, "h": 5}, "fieldConfig": { "defaults": { @@ -426,19 +426,20 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "orange", "value": 1}, + {"color": "green", "value": 3} ] } } }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"postgres-[0-9]+\"})", "legendFormat": "{{namespace}}"} + {"expr": "sum by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"postgres-[0-9]+\"}) or vector(0)", "legendFormat": "{{namespace}} ready"} ] }, { "type": "stat", - "title": "Redis Ready", + "title": "Redis/Sentinel Ready Replicas", "gridPos": {"x": 6, "y": 55, "w": 6, "h": 5}, "fieldConfig": { "defaults": { @@ -447,14 +448,15 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "orange", "value": 3}, + {"color": "green", "value": 6} ] } } }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"redis-.*|redis-replication-.*\"})", "legendFormat": "{{namespace}}"} + {"expr": "sum by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"redis-.*|redis-replication-.*\"}) or vector(0)", "legendFormat": "{{namespace}} ready"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index b70811ed..e74d8244 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -129,7 +129,7 @@ spec: summary: "Judge-server heartbeat is stale" description: "A judge-server heartbeat is older than 180 seconds." - alert: PostgresUnavailable - expr: (min(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) + expr: (sum(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) for: 1m labels: namespace: code-place-dev @@ -137,9 +137,9 @@ spec: priority: P0 annotations: summary: "PostgreSQL pod is not ready in dev" - description: "At least one dev PostgreSQL pod has not been ready, or PostgreSQL pod readiness metrics are missing, for 1 minute." + description: "No dev PostgreSQL pod has been ready, or PostgreSQL pod readiness metrics are missing, for 1 minute." - alert: PostgresUnavailable - expr: (min(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"postgres-[0-9]+"}) + expr: (sum(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"postgres-[0-9]+"}) for: 1m labels: namespace: code-place-prod @@ -147,9 +147,9 @@ spec: priority: P0 annotations: summary: "PostgreSQL pod is not ready in prod" - description: "At least one prod PostgreSQL pod has not been ready, or PostgreSQL pod readiness metrics are missing, for 1 minute." + description: "No prod PostgreSQL pod has been ready, or PostgreSQL pod readiness metrics are missing, for 1 minute." - alert: RedisUnavailable - expr: (min(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) + expr: (sum(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) for: 1m labels: namespace: code-place-dev @@ -157,9 +157,9 @@ spec: priority: P0 annotations: summary: "Redis pod is not ready in dev" - description: "At least one dev Redis pod has not been ready, or Redis pod readiness metrics are missing, for 1 minute." + description: "No dev Redis/Sentinel pod has been ready, or Redis pod readiness metrics are missing, for 1 minute." - alert: RedisUnavailable - expr: (min(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"redis-.*|redis-replication-.*"}) + expr: (sum(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"redis-.*|redis-replication-.*"}) for: 1m labels: namespace: code-place-prod @@ -167,7 +167,7 @@ spec: priority: P0 annotations: summary: "Redis pod is not ready in prod" - description: "At least one prod Redis pod has not been ready, or Redis pod readiness metrics are missing, for 1 minute." + description: "No prod Redis/Sentinel pod has been ready, or Redis pod readiness metrics are missing, for 1 minute." - alert: PostgresExporterDown expr: (min(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) == 0) or absent(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) for: 2m @@ -680,6 +680,24 @@ spec: annotations: summary: "PostgreSQL HA placement is degraded" description: "CNPG reports fewer than 3 distinct nodes for PostgreSQL instances for 10 minutes." + - alert: PostgresReplicaUnavailable + expr: sum by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"postgres-[0-9]+"}) < 3 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL ready replica count is degraded" + description: "Fewer than 3 PostgreSQL pods have been ready for 5 minutes. Failover capacity or quorum may be reduced." + - alert: RedisReplicaUnavailable + expr: sum by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"redis-.*|redis-replication-.*"}) < 6 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Redis/Sentinel ready replica count is degraded" + description: "Fewer than 6 Redis replication and Sentinel pods have been ready for 5 minutes. Redis failover capacity may be reduced." - alert: RedisMemoryHigh expr: (max by (namespace, pod) (redis_memory_used_bytes{namespace=~"code-place-(dev|prod)"}) / max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}) > 0.85) and on (namespace, pod) max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}) > 0 for: 10m From 57e6ebd583bec92caeb369db37a8e3f8892e839e Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:52:13 +0900 Subject: [PATCH 091/147] =?UTF-8?q?=EC=95=8C=EB=A6=BC=20=EA=B2=BD=EB=A1=9C?= =?UTF-8?q?=EC=99=80=20Loki=20=EA=B2=8C=EC=9D=B4=ED=8A=B8=EC=9B=A8?= =?UTF-8?q?=EC=9D=B4=20HA=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 8 +++++--- .../monitoring/grafana-dashboard-logs.yaml | 5 +++-- .../grafana-dashboard-monitoring-stack.yaml | 7 ++++--- .../kube-prometheus-stack-values.yaml | 3 +++ kubernetes/monitoring/logs/README.md | 3 ++- kubernetes/monitoring/logs/loki-values.yaml | 10 +++++++++- kubernetes/monitoring/prometheus-rules.yaml | 20 +++++++++++++++++++ 7 files changed, 46 insertions(+), 10 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 78967928..3855e499 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -210,8 +210,8 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `grafana-dashboard-monitoring-stack.yaml`: Prometheus, Alertmanager, Grafana, Prometheus Operator readiness와 rule/notification 상태 dashboard. - `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. - `kubernetes-event-exporter.yaml`: Kubernetes Warning event를 stdout JSON으로 export해서 Alloy/Loki가 수집하도록 하는 exporter, ServiceMonitor, 최소 event read RBAC. -- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. -- `logs/loki-values.yaml`: Loki SingleBinary, Longhorn PVC, dev/prod retention 설정. +- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, Alertmanager 2 replicas, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. +- `logs/loki-values.yaml`: Loki SingleBinary, Longhorn PVC, dev/prod retention, Loki gateway 2 replicas 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. - `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, Loki ingest/error/canary, Alloy write backpressure, 최근 backend error, judge/celery log 조회 dashboard. - `otel-collector.yaml`: backend/celery OTLP trace를 받아 Tempo로 전송하는 OpenTelemetry Collector. @@ -269,6 +269,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. - `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. - `PrometheusAlertmanagerDiscoveryFailed`: Prometheus가 Alertmanager target을 0개 discovery 2분 지속. +- `AlertmanagerReplicaUnavailable`: Alertmanager ready replica 2개 미만 5분 지속. - `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `SubmissionCreateSystemFailures`: 제출 생성 API에서 DB 또는 judge enqueue 실패 발생 2분 지속. @@ -307,6 +308,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `LokiPVCAlmostFull`: Loki Longhorn PVC 사용률 85% 초과 10분 지속. - `AlloyDaemonSetUnavailable`: Alloy log collector가 모든 node에서 available하지 않음 5분 지속. - `LokiGatewayUnavailable`: Loki gateway Pod not ready 2분 지속. +- `LokiGatewayReplicaUnavailable`: Loki gateway ready replica 2개 미만 5분 지속. - `LokiIngestionStalled`: Loki log line 수신량 0이 10분 지속. - `LokiRequestErrors`: Loki 5xx response 발생 5분 지속. - `LokiCanaryMissingEntries`: Loki canary write/readback 누락 발생 5분 지속. @@ -336,7 +338,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. ## 5. 운영 확인 절차 -1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. +1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 Alertmanager 2 replicas, `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. - `helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --version 86.3.1 --values kubernetes/monitoring/kube-prometheus-stack-values.yaml` - values는 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None`을 설정합니다. AlertmanagerConfig는 `monitoring` namespace에 있지만 CodePlace alert의 `namespace` label은 `code-place-dev`/`code-place-prod`이므로, 기본 `OnNamespace` matcher를 쓰면 외부 namespace alert가 Discord receiver까지 도달하지 않을 수 있습니다. 2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 SealedSecret으로 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index 5a9a5d61..5adcb9f1 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -46,7 +46,7 @@ data: "panels": [ { "type": "stat", - "title": "Loki Ready", + "title": "Loki Ready Replicas", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -63,7 +63,8 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-[0-9]+\"}) or vector(0)", "legendFormat": "loki"} + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-[0-9]+\"}) or vector(0)", "legendFormat": "single-binary"}, + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-gateway-.*\"}) or vector(0)", "legendFormat": "gateway"} ] }, { diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index 5ba518ec..383e3d16 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -43,7 +43,7 @@ data: }, { "type": "stat", - "title": "Alertmanager Ready", + "title": "Alertmanager Ready Replicas", "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -53,14 +53,15 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} ] } } }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"alertmanager-kube-prometheus-stack-alertmanager-.*\"}) or vector(0)", "legendFormat": "alertmanager"} + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"alertmanager-kube-prometheus-stack-alertmanager-.*\"}) or vector(0)", "legendFormat": "alertmanager ready"} ] }, { diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index d1dd5a08..11c3f64f 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -20,6 +20,9 @@ prometheus: alertmanager: alertmanagerSpec: + replicas: 2 + podAntiAffinity: soft + podAntiAffinityTopologyKey: kubernetes.io/hostname alertmanagerConfigMatcherStrategy: type: None alertmanagerConfigSelector: diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 1bcc8d23..918e3919 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -3,6 +3,7 @@ This directory contains Helm values for the Kubernetes logs stack. - Loki: SingleBinary mode, filesystem storage, Longhorn PVC. +- Loki gateway: 2 replicas for the stateless write/query entrypoint. - Alloy: DaemonSet log collector for every pod in `code-place-dev`, `code-place-prod`, and `monitoring`, plus Traefik pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. - Kubernetes Event Exporter: Warning events are written to stdout as JSON and collected by Alloy from the `monitoring` namespace. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. @@ -76,7 +77,7 @@ Keep these invariants unless the storage design changes intentionally: - Loki stays in `SingleBinary` deployment mode. - Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. -- In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors and probe exporters can be replicated across nodes. +- In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors, Loki gateway, and probe exporters can be replicated across nodes. - kube-prometheus-stack Prometheus, Alertmanager, and Grafana also use Longhorn PVCs so metric data, silences, and UI state survive pod rescheduling. - Alloy keeps namespace-based collection for `code-place-dev`, `code-place-prod`, and `monitoring`; do not depend on `app.kubernetes.io/name` for CodePlace app logs because the application manifests primarily use `app`. - Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. diff --git a/kubernetes/monitoring/logs/loki-values.yaml b/kubernetes/monitoring/logs/loki-values.yaml index 28c1678f..e735cff6 100644 --- a/kubernetes/monitoring/logs/loki-values.yaml +++ b/kubernetes/monitoring/logs/loki-values.yaml @@ -64,7 +64,15 @@ singleBinary: gateway: enabled: true - replicas: 1 + replicas: 2 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/component: gateway + app.kubernetes.io/instance: loki lokiCanary: enabled: true diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index e74d8244..fcf53d84 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -427,6 +427,16 @@ spec: annotations: summary: "Prometheus has not discovered Alertmanager" description: "Prometheus has discovered zero Alertmanager targets for 2 minutes. Alert delivery may be broken even if alert rules are firing." + - alert: AlertmanagerReplicaUnavailable + expr: sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) < 2 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Alertmanager replica count is degraded" + description: "Fewer than 2 Alertmanager replicas have been ready for 5 minutes. Alert delivery still works, but the notification path has lost HA." - alert: AlertmanagerNotificationFailures expr: sum(increase(alertmanager_notifications_failed_total[5m])) > 0 for: 2m @@ -801,6 +811,16 @@ spec: annotations: summary: "Loki gateway is not ready" description: "The Loki gateway has not been ready for 2 minutes. Alloy writes and Grafana Loki queries may fail." + - alert: LokiGatewayReplicaUnavailable + expr: sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) < 2 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki gateway replica count is degraded" + description: "Fewer than 2 Loki gateway replicas have been ready for 5 minutes. Loki writes and queries still work, but the gateway path has lost HA." - alert: LokiIngestionStalled expr: (sum(rate(loki_distributor_lines_received_total{namespace="monitoring"}[5m])) == 0) or absent(loki_distributor_lines_received_total{namespace="monitoring"}) for: 10m From 24454c7d962022f8587efc62015dfe7ac50f0403 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:55:47 +0900 Subject: [PATCH 092/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=20=EC=8A=A4=ED=83=9D?= =?UTF-8?q?=20unavailable=20=EC=95=8C=EB=A6=BC=20=EA=B8=B0=EC=A4=80=20?= =?UTF-8?q?=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../monitoring/grafana-dashboard-traces.yaml | 2 +- kubernetes/monitoring/prometheus-rules.yaml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kubernetes/monitoring/grafana-dashboard-traces.yaml b/kubernetes/monitoring/grafana-dashboard-traces.yaml index c8fa91af..f333003a 100644 --- a/kubernetes/monitoring/grafana-dashboard-traces.yaml +++ b/kubernetes/monitoring/grafana-dashboard-traces.yaml @@ -38,7 +38,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"tempo-.*\"}) or vector(0)", "legendFormat": "tempo"} + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"tempo-.*\"}) or vector(0)", "legendFormat": "tempo"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index fcf53d84..9744d3b4 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -209,7 +209,7 @@ spec: summary: "Redis metrics exporter is down" description: "Prometheus cannot collect Redis exporter metrics for code-place-prod for 2 minutes." - alert: LokiUnavailable - expr: (min(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) + expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) for: 1m labels: severity: critical @@ -217,7 +217,7 @@ spec: namespace: monitoring annotations: summary: "Loki is not ready" - description: "The Loki single-binary pod has not been ready for 1 minute. Log ingestion and log queries may be unavailable." + description: "No Loki single-binary pod has been ready for 1 minute. Log ingestion and log queries may be unavailable." - alert: LonghornManagerDown expr: (sum(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) == 0) or absent(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) for: 2m @@ -802,7 +802,7 @@ spec: summary: "Alloy log collector is not running on every node" description: "At least one node is missing an available Alloy DaemonSet pod for 5 minutes. Logs from that node may be missing." - alert: LokiGatewayUnavailable - expr: (max by (namespace, pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) + expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) for: 2m labels: severity: warning @@ -810,7 +810,7 @@ spec: namespace: monitoring annotations: summary: "Loki gateway is not ready" - description: "The Loki gateway has not been ready for 2 minutes. Alloy writes and Grafana Loki queries may fail." + description: "No Loki gateway replica has been ready for 2 minutes. Alloy writes and Grafana Loki queries may fail." - alert: LokiGatewayReplicaUnavailable expr: sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) < 2 for: 5m @@ -872,7 +872,7 @@ spec: summary: "Alloy is retrying Loki writes" description: "Alloy has retried Loki write batches for 5 minutes. Loki writes may be slow or intermittently failing." - alert: OTelCollectorUnavailable - expr: (max by (namespace, pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) + expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) for: 2m labels: severity: warning @@ -880,7 +880,7 @@ spec: namespace: monitoring annotations: summary: "OpenTelemetry Collector is not ready" - description: "The OpenTelemetry Collector has not been ready for 2 minutes. Enabled services cannot export traces." + description: "No OpenTelemetry Collector replica has been ready for 2 minutes. Enabled services cannot export traces." - alert: OTelCollectorReplicaUnavailable expr: kube_deployment_status_replicas_available{namespace="monitoring", deployment="otel-collector"} < kube_deployment_spec_replicas{namespace="monitoring", deployment="otel-collector"} for: 5m @@ -912,7 +912,7 @@ spec: summary: "OpenTelemetry Collector cannot export spans" description: "The OpenTelemetry Collector has failed to export spans to Tempo in the last 5 minutes. Check Tempo readiness, collector exporter logs, and network path." - alert: TempoUnavailable - expr: (max by (namespace, pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) + expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) for: 2m labels: severity: warning @@ -920,7 +920,7 @@ spec: namespace: monitoring annotations: summary: "Tempo is not ready" - description: "Tempo has not been ready for 2 minutes. Trace queries and trace ingestion may fail." + description: "No Tempo pod has been ready for 2 minutes. Trace queries and trace ingestion may fail." - alert: TempoPVCAlmostFull expr: max(kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}), 1) > 0.85 for: 10m From 049d0fcc4273434d5f7cca87693d96a496b06b6b Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:58:00 +0900 Subject: [PATCH 093/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?=EC=84=A4=EC=A0=95=20reload=20=EC=8B=A4=ED=8C=A8=20=EA=B0=90?= =?UTF-8?q?=EC=A7=80=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 2 ++ .../grafana-dashboard-monitoring-stack.yaml | 10 ++++++---- kubernetes/monitoring/prometheus-rules.yaml | 20 +++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 3855e499..f3c69a98 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -268,8 +268,10 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `GrafanaUnavailable`: Grafana Pod not ready 2분 지속. - `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. - `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. +- `PrometheusConfigReloadFailed`: Prometheus generated config reload 실패 또는 reload metric 누락 2분 지속. - `PrometheusAlertmanagerDiscoveryFailed`: Prometheus가 Alertmanager target을 0개 discovery 2분 지속. - `AlertmanagerReplicaUnavailable`: Alertmanager ready replica 2개 미만 5분 지속. +- `AlertmanagerConfigReloadFailed`: Alertmanager generated config reload 실패 또는 reload metric 누락 2분 지속. - `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `SubmissionCreateSystemFailures`: 제출 생성 API에서 DB 또는 judge enqueue 실패 발생 2분 지속. diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index 383e3d16..0ce5484f 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -206,23 +206,25 @@ data: }, { "type": "timeseries", - "title": "Rule Evaluation Failures", + "title": "Prometheus Rule and Config Health", "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "{{rule_group}}"} + {"expr": "sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "{{rule_group}} failures"}, + {"expr": "max(prometheus_config_last_reload_successful{namespace=\"monitoring\"}) or vector(0)", "legendFormat": "config reload successful"} ] }, { "type": "timeseries", - "title": "Alertmanager Notifications", + "title": "Alertmanager Notifications and Config", "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "ops"}}, "targets": [ {"expr": "sum by (integration) (rate(alertmanager_notifications_total[5m]))", "legendFormat": "{{integration}} sent"}, - {"expr": "sum by (integration) (rate(alertmanager_notifications_failed_total[5m])) or vector(0)", "legendFormat": "{{integration}} failed"} + {"expr": "sum by (integration) (rate(alertmanager_notifications_failed_total[5m])) or vector(0)", "legendFormat": "{{integration}} failed"}, + {"expr": "max(alertmanager_config_last_reload_successful{namespace=\"monitoring\"}) or vector(0)", "legendFormat": "config reload successful"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 9744d3b4..a607a506 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -417,6 +417,16 @@ spec: annotations: summary: "Prometheus rule evaluation is failing" description: "Prometheus has reported rule evaluation failures for 5 minutes. Alert and recording rule expressions should be checked." + - alert: PrometheusConfigReloadFailed + expr: (max(prometheus_config_last_reload_successful{namespace="monitoring"}) == 0) or absent(prometheus_config_last_reload_successful{namespace="monitoring"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Prometheus config reload is failing" + description: "Prometheus has not successfully reloaded its generated configuration for 2 minutes, or the reload metric is missing. ServiceMonitor, Probe, and rule changes may not be active." - alert: PrometheusAlertmanagerDiscoveryFailed expr: (max(prometheus_notifications_alertmanagers_discovered) < 1) or absent(prometheus_notifications_alertmanagers_discovered) for: 2m @@ -437,6 +447,16 @@ spec: annotations: summary: "Alertmanager replica count is degraded" description: "Fewer than 2 Alertmanager replicas have been ready for 5 minutes. Alert delivery still works, but the notification path has lost HA." + - alert: AlertmanagerConfigReloadFailed + expr: (max(alertmanager_config_last_reload_successful{namespace="monitoring"}) == 0) or absent(alertmanager_config_last_reload_successful{namespace="monitoring"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Alertmanager config reload is failing" + description: "Alertmanager has not successfully reloaded its generated configuration for 2 minutes, or the reload metric is missing. Check AlertmanagerConfig, webhook Secret references, and operator reconciliation." - alert: AlertmanagerNotificationFailures expr: sum(increase(alertmanager_notifications_failed_total[5m])) > 0 for: 2m From d6aa4598206642af54d40cee2b1cb07f9747ba70 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 20:59:31 +0900 Subject: [PATCH 094/147] =?UTF-8?q?dev=20P1=20=EC=99=B8=EB=B6=80=20?= =?UTF-8?q?=EC=95=8C=EB=A6=BC=20=EB=9D=BC=EC=9A=B0=ED=8C=85=20=EB=B6=84?= =?UTF-8?q?=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 4 ++-- kubernetes/monitoring/alertmanager-config.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index f3c69a98..b1bc5973 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -255,7 +255,7 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 ### P1 -P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. +P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev` namespace의 P1은 외부 Discord로 보내지 않고 Alertmanager/Grafana UI에만 남깁니다. `code-place-prod`, `monitoring`, `longhorn-system` 등 운영 영향이 있는 P1은 Discord로 보냅니다. - `ApiLatencyHigh`: p95 latency 2초 초과 5분 지속. - `PublicEndpointDown`: dev 공개 HTTPS endpoint probe 실패 2분 지속. frontend와 hub-auth를 포함합니다. @@ -375,7 +375,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Discord webhook URL은 Git에 평문으로 저장하지 않습니다. 운영 Secret은 SealedSecret으로 관리합니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. - OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지하고 dev overlay에서 먼저 활성화합니다. -- P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. +- P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. dev P1은 `dev-p1-muted` receiver로 보내 외부 전송을 막고, prod/monitoring P1은 Discord receiver로 보냅니다. - Metrics backend는 kube-prometheus-stack의 Prometheus입니다. Mimir는 현재 온프렘 단기 운영 기준선에 포함하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - Kubernetes event는 Kubernetes Event Exporter가 Warning event만 stdout JSON으로 내보내고, Alloy가 일반 Pod log와 같은 경로로 Loki에 적재합니다. diff --git a/kubernetes/monitoring/alertmanager-config.yaml b/kubernetes/monitoring/alertmanager-config.yaml index c26d85c4..4242c086 100644 --- a/kubernetes/monitoring/alertmanager-config.yaml +++ b/kubernetes/monitoring/alertmanager-config.yaml @@ -22,6 +22,15 @@ spec: groupInterval: 1m repeatInterval: 15m receiver: p0-discord + - matchers: + - name: priority + value: P1 + - name: namespace + value: code-place-dev + groupWait: 30s + groupInterval: 5m + repeatInterval: 1h + receiver: dev-p1-muted - matchers: - name: priority value: P1 @@ -40,6 +49,7 @@ spec: content: '{{ .CommonAnnotations.summary }}' message: '{{ .CommonAnnotations.description }}' username: CodePlace Alertmanager + - name: dev-p1-muted - name: p1-discord discordConfigs: - sendResolved: true From 06a540a0f205e5a3af2494514f4698b06d10b003 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 21:02:48 +0900 Subject: [PATCH 095/147] =?UTF-8?q?=ED=95=B5=EC=8B=AC=20=EC=9E=91=EC=97=85?= =?UTF-8?q?=20=EC=97=94=EB=93=9C=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=EA=B2=B0?= =?UTF-8?q?=EC=B8=A1=20=EC=95=8C=EB=A6=BC=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 6 +++--- kubernetes/monitoring/prometheus-rules.yaml | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index b1bc5973..68ac02ba 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -279,7 +279,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - `PendingSubmissionsStuck`: oldest pending submission age 300초 초과 3분 지속. - `JudgingSubmissionsStuck`: oldest judging submission age 900초 초과 5분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. -- `CeleryBeatDown`: beat Pod not ready 2분 지속. +- `CeleryBeatDown`: beat Deployment available replica 0개 또는 availability metric 누락 2분 지속. - `CeleryTaskFailures`: task failure 발생 2분 지속. - `CeleryTaskRetries`: task retry 3회 초과/10분, 5분 지속. - `CeleryTaskRuntimeHigh`: task p95 runtime 120초 초과 10분 지속. @@ -294,8 +294,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - `KubernetesPodUnschedulable`: Pending 또는 Unschedulable 상태 10분 지속. - `CodePlaceDeploymentUnavailable`: 주요 Deployment unavailable replica 5분 지속. - `CodePlaceDeploymentRolloutStuck`: Deployment observed generation 미반영 10분 지속. -- `CodePlaceServiceNoReadyEndpoints`: 주요 Service ready endpoint 0개 2분 지속. -- `VLLMServiceNoReadyEndpoints`: prod vLLM Service ready endpoint 0개 2분 지속. +- `CodePlaceServiceNoReadyEndpoints`: 주요 Service ready endpoint 0개 또는 endpoint availability metric 누락 2분 지속. +- `VLLMServiceNoReadyEndpoints`: prod vLLM Service ready endpoint 0개 또는 endpoint availability metric 누락 2분 지속. - `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. - `CodePlaceContainerMemoryHigh`: 주요 컨테이너 memory limit 사용률 85% 초과 10분 지속. - `PostgresCollectorError`: CNPG PostgreSQL metrics collection error 5분 지속. diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index a607a506..81b0495c 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -521,14 +521,14 @@ spec: summary: "Celery worker is restarting repeatedly" description: "Celery worker pods restarted more than 3 times in 15 minutes." - alert: CeleryBeatDown - expr: min by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"celery-beat-.*"}) == 0 + expr: (kube_deployment_status_replicas_available{namespace=~"code-place-(dev|prod)", deployment="celery-beat"} < 1) or (kube_deployment_spec_replicas{namespace=~"code-place-(dev|prod)", deployment="celery-beat"} unless on (namespace, deployment) kube_deployment_status_replicas_available{namespace=~"code-place-(dev|prod)", deployment="celery-beat"}) for: 2m labels: severity: warning priority: P1 annotations: summary: "Celery beat is not ready" - description: "The celery-beat pod has not been ready for 2 minutes." + description: "The celery-beat Deployment has no available replica for 2 minutes, or its availability metric is missing." - alert: CeleryTaskFailures expr: sum by (namespace, task_name) (increase(codeplace_celery_task_total{namespace=~"code-place-(dev|prod)", status="failure"}[10m])) > 0 for: 2m @@ -656,16 +656,16 @@ spec: summary: "CodePlace deployment rollout is stuck" description: "A CodePlace deployment observed generation has not caught up with metadata generation for 10 minutes. Check deployment events, ReplicaSet state, and rollout history." - alert: CodePlaceServiceNoReadyEndpoints - expr: max by (namespace, endpoint) (kube_endpoint_address_available{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server"}) < 1 + expr: (max by (namespace, endpoint) (kube_endpoint_address_available{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server"}) < 1) or (label_replace(kube_service_info{namespace=~"code-place-(dev|prod)", service=~"backend|frontend|hub-auth|judge-server"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address_available{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server"}) for: 2m labels: severity: warning priority: P1 annotations: summary: "CodePlace service has no ready endpoints" - description: "A CodePlace Service has had zero ready endpoints for 2 minutes. Check Service selector labels, Pod readiness, EndpointSlice state, and rollout events." + description: "A CodePlace Service has had zero ready endpoints for 2 minutes, or the endpoint availability metric is missing while the Service exists. Check Service selector labels, Pod readiness, EndpointSlice state, and rollout events." - alert: VLLMServiceNoReadyEndpoints - expr: max by (namespace, endpoint) (kube_endpoint_address_available{namespace="code-place-prod", endpoint="vllm"}) < 1 + expr: (max by (namespace, endpoint) (kube_endpoint_address_available{namespace="code-place-prod", endpoint="vllm"}) < 1) or (label_replace(kube_service_info{namespace="code-place-prod", service="vllm"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address_available{namespace="code-place-prod", endpoint="vllm"}) for: 2m labels: severity: warning @@ -673,7 +673,7 @@ spec: namespace: code-place-prod annotations: summary: "vLLM service has no ready endpoints" - description: "The prod vLLM Service has had zero ready endpoints for 2 minutes. AI hint routing may fail even if the Service object exists." + description: "The prod vLLM Service has had zero ready endpoints for 2 minutes, or the endpoint availability metric is missing while the Service exists. AI hint routing may fail even if the Service object exists." - alert: CodePlaceContainerCPUHigh expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 0.001) > 0.8 for: 10m From 2ca60719b584b2d60659a770c0d9c50f24046d78 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 21:07:05 +0900 Subject: [PATCH 096/147] =?UTF-8?q?GitHub=20Actions=20=EB=B3=80=EA=B2=BD?= =?UTF-8?q?=20=EC=A0=9C=EC=99=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci2develop.yml | 14 +------------- .github/workflows/ci2production-and-release.yml | 14 +------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index 227f64ec..b6fe14fd 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -136,15 +136,6 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml - - name: Update Backend Sentry Release - if: needs.ci-backend-dev.result == 'success' - uses: mikefarah/yq@master - with: - cmd: | - yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/backend-deployment-patch.yaml - yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-worker-deployment-patch.yaml - yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-beat-deployment-patch.yaml - - name: Update Frontend Image Tag if: needs.ci-frontend-dev.result == 'success' uses: mikefarah/yq@master @@ -161,10 +152,7 @@ jobs: run: | git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git add kubernetes/overlays/dev/kustomization.yaml \ - kubernetes/overlays/dev/backend-deployment-patch.yaml \ - kubernetes/overlays/dev/celery-worker-deployment-patch.yaml \ - kubernetes/overlays/dev/celery-beat-deployment-patch.yaml + git add kubernetes/overlays/dev/kustomization.yaml # 변경사항이 없으면 종료 if git diff-index --quiet HEAD; then diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index f0488ac1..28e3db59 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -165,15 +165,6 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml - - name: Update Backend Sentry Release - if: needs.ci-backend.result == 'success' - uses: mikefarah/yq@master - with: - cmd: | - yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/backend-deployment-patch.yaml - yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-worker-deployment-patch.yaml - yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-beat-deployment-patch.yaml - - name: Update Frontend Image Tag if: needs.ci-frontend.result == 'success' uses: mikefarah/yq@master @@ -190,10 +181,7 @@ jobs: run: | git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git add kubernetes/overlays/prod/kustomization.yaml \ - kubernetes/overlays/prod/backend-deployment-patch.yaml \ - kubernetes/overlays/prod/celery-worker-deployment-patch.yaml \ - kubernetes/overlays/prod/celery-beat-deployment-patch.yaml + git add kubernetes/overlays/prod/kustomization.yaml # 변경사항이 없으면 종료 if git diff-index --quiet HEAD; then From 274b7b89a3d80497569b2ecc026f747a1ea5fa83 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 21:14:08 +0900 Subject: [PATCH 097/147] =?UTF-8?q?Prometheus=20=EA=B3=A0=EA=B0=80?= =?UTF-8?q?=EC=9A=A9=EC=84=B1=20=EA=B4=80=EC=B8=A1=20=EA=B8=B0=EC=A4=80=20?= =?UTF-8?q?=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 7 ++++--- .../developer/infra/observability-runbook/_index.md | 9 ++++++--- kubernetes/monitoring/backend-service-monitor.yaml | 4 +++- .../monitoring/grafana-dashboard-monitoring-stack.yaml | 7 ++++--- .../monitoring/kube-prometheus-stack-values.yaml | 4 ++++ kubernetes/monitoring/logs/README.md | 2 +- kubernetes/monitoring/prometheus-rules.yaml | 10 ++++++++++ 7 files changed, 32 insertions(+), 11 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 68ac02ba..191a7db7 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -198,7 +198,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto 신규 monitoring 리소스는 `kubernetes/monitoring` 아래에 두고, 애플리케이션 `dev/prod` overlay에 포함하지 않습니다. -- `backend-service-monitor.yaml`: backend `/metrics` scrape, interval 15s. +- `backend-service-monitor.yaml`: `code-place-dev`와 `code-place-prod` backend `/metrics` scrape, interval 15s. - `traefik-service-monitor.yaml`: kube-system Traefik Prometheus metrics scrape, interval 15s. `Ingress5xxSpike`의 `traefik_service_requests_total` 원천입니다. - `blackbox-exporter.yaml`: public endpoint synthetic probe용 Blackbox exporter. - `public-endpoint-probes.yaml`: dev/prod frontend, hub-auth, Grafana 공개 HTTPS URL Probe, interval 30s. @@ -210,7 +210,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `grafana-dashboard-monitoring-stack.yaml`: Prometheus, Alertmanager, Grafana, Prometheus Operator readiness와 rule/notification 상태 dashboard. - `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. - `kubernetes-event-exporter.yaml`: Kubernetes Warning event를 stdout JSON으로 export해서 Alloy/Loki가 수집하도록 하는 exporter, ServiceMonitor, 최소 event read RBAC. -- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, Alertmanager 2 replicas, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. +- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, Prometheus 2 replicas, Prometheus replica external label 제거, Alertmanager 2 replicas, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `logs/loki-values.yaml`: Loki SingleBinary, Longhorn PVC, dev/prod retention, Loki gateway 2 replicas 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. - `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, Loki ingest/error/canary, Alloy write backpressure, 최근 backend error, judge/celery log 조회 dashboard. @@ -270,6 +270,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. - `PrometheusConfigReloadFailed`: Prometheus generated config reload 실패 또는 reload metric 누락 2분 지속. - `PrometheusAlertmanagerDiscoveryFailed`: Prometheus가 Alertmanager target을 0개 discovery 2분 지속. +- `PrometheusReplicaUnavailable`: Prometheus ready replica 2개 미만 5분 지속. - `AlertmanagerReplicaUnavailable`: Alertmanager ready replica 2개 미만 5분 지속. - `AlertmanagerConfigReloadFailed`: Alertmanager generated config reload 실패 또는 reload metric 누락 2분 지속. - `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. @@ -340,7 +341,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev ## 5. 운영 확인 절차 -1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 Alertmanager 2 replicas, `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. +1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 Prometheus 2 replicas, Alertmanager 2 replicas, `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. - `helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --version 86.3.1 --values kubernetes/monitoring/kube-prometheus-stack-values.yaml` - values는 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None`을 설정합니다. AlertmanagerConfig는 `monitoring` namespace에 있지만 CodePlace alert의 `namespace` label은 `code-place-dev`/`code-place-prod`이므로, 기본 `OnNamespace` matcher를 쓰면 외부 namespace alert가 Discord receiver까지 도달하지 않을 수 있습니다. 2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 SealedSecret으로 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 9c37fa99..d4348732 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -416,15 +416,16 @@ kubectl -n monitoring describe pvc -l app.kubernetes.io/name=loki - readiness만 실패하면 Loki config, retention, disk full, compactor error를 확인합니다. - Loki가 내려가도 앱 요청 처리는 계속되지만 로그 수집/조회가 끊기므로 빠르게 복구합니다. -### PrometheusUnavailable / AlertmanagerUnavailable +### PrometheusUnavailable / PrometheusReplicaUnavailable / AlertmanagerUnavailable -증상: Prometheus 또는 Alertmanager Pod가 1분 이상 ready가 아닙니다. +증상: Prometheus 또는 Alertmanager Pod가 ready가 아니거나, Prometheus ready replica가 2개 미만입니다. 확인: ```sh kubectl -n monitoring get pod,svc | grep -E 'prometheus|alertmanager' kubectl -n monitoring describe pod prometheus-kube-prometheus-stack-prometheus-0 +kubectl -n monitoring describe pod prometheus-kube-prometheus-stack-prometheus-1 kubectl -n monitoring describe pod alertmanager-kube-prometheus-stack-alertmanager-0 kubectl -n monitoring logs prometheus-kube-prometheus-stack-prometheus-0 -c prometheus --tail=200 kubectl -n monitoring logs alertmanager-kube-prometheus-stack-alertmanager-0 -c alertmanager --tail=200 @@ -438,7 +439,8 @@ prometheus_rule_evaluation_failures_total 판단: -- Prometheus가 down이면 metric 수집과 rule evaluation이 멈춥니다. Alertmanager가 살아 있어도 새 알림은 생성되지 않습니다. +- Prometheus ready replica가 1개만 남으면 metric 수집과 rule evaluation은 계속되지만 monitoring path HA를 잃은 상태입니다. 남은 replica까지 내려가기 전에 PVC attach, config reload, resource pressure, scheduling event를 확인합니다. +- Prometheus가 전부 down이면 metric 수집과 rule evaluation이 멈춥니다. Alertmanager가 살아 있어도 새 알림은 생성되지 않습니다. - Alertmanager가 down이면 firing alert가 있어도 Discord/email 전달이 실패합니다. P0 수신 지연 원인으로 먼저 봅니다. - PVC, config reload, rule syntax, resource pressure, operator reconciliation event를 함께 확인합니다. @@ -600,6 +602,7 @@ PromQL: ```promql kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) +sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) prometheus_notifications_alertmanagers_discovered sum by (integration) (increase(alertmanager_notifications_failed_total[5m])) sum by (job, namespace) (up == 0) diff --git a/kubernetes/monitoring/backend-service-monitor.yaml b/kubernetes/monitoring/backend-service-monitor.yaml index 20966309..928150fc 100644 --- a/kubernetes/monitoring/backend-service-monitor.yaml +++ b/kubernetes/monitoring/backend-service-monitor.yaml @@ -6,7 +6,9 @@ metadata: release: kube-prometheus-stack spec: namespaceSelector: - any: true + matchNames: + - code-place-dev + - code-place-prod selector: matchLabels: app: backend diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index 0ce5484f..e5cf23d9 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -21,7 +21,7 @@ data: "panels": [ { "type": "stat", - "title": "Prometheus Ready", + "title": "Prometheus Ready Replicas", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -31,14 +31,15 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} ] } } }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\"}) or vector(0)", "legendFormat": "prometheus"} + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\"}) or vector(0)", "legendFormat": "prometheus ready"} ] }, { diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index 11c3f64f..692f558d 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -1,5 +1,9 @@ prometheus: prometheusSpec: + replicas: 2 + podAntiAffinity: soft + podAntiAffinityTopologyKey: kubernetes.io/hostname + replicaExternalLabelNameClear: true evaluationInterval: 15s scrapeInterval: 15s retention: 15d diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 918e3919..e0449902 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -78,7 +78,7 @@ Keep these invariants unless the storage design changes intentionally: - Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. - In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors, Loki gateway, and probe exporters can be replicated across nodes. -- kube-prometheus-stack Prometheus, Alertmanager, and Grafana also use Longhorn PVCs so metric data, silences, and UI state survive pod rescheduling. +- kube-prometheus-stack Prometheus and Alertmanager run with 2 replicas. Prometheus clears the replica external label so HA replicas do not create replica-labeled duplicate Alertmanager notifications. Prometheus, Alertmanager, and Grafana use Longhorn PVCs so metric data, silences, and UI state survive pod rescheduling. - Alloy keeps namespace-based collection for `code-place-dev`, `code-place-prod`, and `monitoring`; do not depend on `app.kubernetes.io/name` for CodePlace app logs because the application manifests primarily use `app`. - Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. - The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 81b0495c..dbe8a733 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -437,6 +437,16 @@ spec: annotations: summary: "Prometheus has not discovered Alertmanager" description: "Prometheus has discovered zero Alertmanager targets for 2 minutes. Alert delivery may be broken even if alert rules are firing." + - alert: PrometheusReplicaUnavailable + expr: sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) < 2 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Prometheus replica count is degraded" + description: "Fewer than 2 Prometheus replicas have been ready for 5 minutes. Metric collection and alert evaluation still work, but the monitoring path has lost HA." - alert: AlertmanagerReplicaUnavailable expr: sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) < 2 for: 5m From 860be5624c6b2c7b00ea7188ffda6371d6201899 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 21:21:40 +0900 Subject: [PATCH 098/147] =?UTF-8?q?Grafana=20=EB=A9=94=ED=8A=B8=EB=A6=AD?= =?UTF-8?q?=20=EA=B4=80=EC=B8=A1=20=EA=B2=BD=EB=A1=9C=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 3 ++- .../infra/observability-runbook/_index.md | 4 +++- .../grafana-dashboard-monitoring-stack.yaml | 22 +++++++++++++++++++ .../kube-prometheus-stack-values.yaml | 3 +++ kubernetes/monitoring/prometheus-rules.yaml | 10 +++++++++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 191a7db7..07e417ea 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -210,7 +210,7 @@ prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collecto - `grafana-dashboard-monitoring-stack.yaml`: Prometheus, Alertmanager, Grafana, Prometheus Operator readiness와 rule/notification 상태 dashboard. - `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. - `kubernetes-event-exporter.yaml`: Kubernetes Warning event를 stdout JSON으로 export해서 Alloy/Loki가 수집하도록 하는 exporter, ServiceMonitor, 최소 event read RBAC. -- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, Prometheus 2 replicas, Prometheus replica external label 제거, Alertmanager 2 replicas, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. +- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, Prometheus 2 replicas, Prometheus replica external label 제거, Alertmanager 2 replicas, Grafana metrics, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `logs/loki-values.yaml`: Loki SingleBinary, Longhorn PVC, dev/prod retention, Loki gateway 2 replicas 설정. - `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. - `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, Loki ingest/error/canary, Alloy write backpressure, 최근 backend error, judge/celery log 조회 dashboard. @@ -266,6 +266,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - `GrafanaPublicEndpointDown`: Grafana 공개 HTTPS endpoint probe 실패 2분 지속. - `KubernetesEventExporterUnavailable`: Kubernetes event exporter Pod not ready 2분 지속. - `GrafanaUnavailable`: Grafana Pod not ready 2분 지속. +- `GrafanaMetricsTargetDown`: Grafana `/metrics` scrape 실패 또는 target series missing 2분 지속. - `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. - `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. - `PrometheusConfigReloadFailed`: Prometheus generated config reload 실패 또는 reload metric 누락 2분 지속. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index d4348732..b06daf74 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -584,7 +584,7 @@ kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condi - latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. - `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. -### GrafanaUnavailable / PrometheusOperatorUnavailable / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures +### GrafanaUnavailable / GrafanaMetricsTargetDown / PrometheusOperatorUnavailable / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures 확인: @@ -601,6 +601,7 @@ PromQL: ```promql kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} +up{namespace="monitoring", service="kube-prometheus-stack-grafana"} sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) prometheus_notifications_alertmanagers_discovered @@ -611,6 +612,7 @@ sum by (job, namespace) (up == 0) 판단: - Grafana만 down이면 알림은 계속 갈 수 있지만 dashboard/Explore 기반 원인 분석이 막힙니다. +- Grafana UI는 살아 있는데 metrics target만 down이면 Grafana 내부 health와 provisioning 상태를 Prometheus에서 못 봅니다. Grafana ServiceMonitor, Service port, `/metrics` 응답, Grafana 로그를 확인합니다. - Operator가 down이면 새 ServiceMonitor/PodMonitor/PrometheusRule/AlertmanagerConfig 변경이 Prometheus/Alertmanager에 반영되지 않을 수 있습니다. - rule evaluation failure가 있으면 최근 추가한 PromQL expression과 label cardinality, metric 존재 여부를 먼저 봅니다. - Alertmanager discovery가 0이면 AlertmanagerConfig가 맞아도 Prometheus가 알림을 보낼 대상이 없습니다. Prometheus generated config와 Alertmanager Service/endpoints를 확인합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index e5cf23d9..77516d89 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -153,6 +153,28 @@ data: {"expr": "sum(increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "failures"} ] }, + { + "type": "stat", + "title": "Grafana Metrics", + "gridPos": {"x": 0, "y": 36, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(up{namespace=\"monitoring\", service=\"kube-prometheus-stack-grafana\"}) or vector(0)", "legendFormat": "up"} + ] + }, { "type": "stat", "title": "Notify Fail", diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index 692f558d..c13ec923 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -46,6 +46,9 @@ alertmanager: storage: 5Gi grafana: + grafana.ini: + metrics: + enabled: true persistence: enabled: true type: pvc diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index dbe8a733..2496d401 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -397,6 +397,16 @@ spec: annotations: summary: "Grafana is not ready" description: "No Grafana pod in the monitoring namespace has been ready for 2 minutes. Dashboards and Explore may be unavailable." + - alert: GrafanaMetricsTargetDown + expr: (up{namespace="monitoring", service="kube-prometheus-stack-grafana"} == 0) or absent(up{namespace="monitoring", service="kube-prometheus-stack-grafana"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Grafana metrics target is down" + description: "Prometheus cannot scrape Grafana /metrics for 2 minutes. Grafana may still serve UI, but dashboard/datasource/provisioning health visibility is degraded." - alert: PrometheusOperatorUnavailable expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-operator-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-operator-.*"}) for: 2m From 4cf0495f23962753a32578e75246b5255f03f281 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 21:25:40 +0900 Subject: [PATCH 099/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?=EC=84=9C=EB=B9=84=EC=8A=A4=20target=20=EC=9E=A5=EC=95=A0=20?= =?UTF-8?q?=EA=B0=90=EC=A7=80=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 1 + .../infra/observability-runbook/_index.md | 4 +++- .../grafana-dashboard-monitoring-stack.yaml | 21 +++++++++++++++++++ kubernetes/monitoring/prometheus-rules.yaml | 10 +++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 07e417ea..01694716 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -267,6 +267,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - `KubernetesEventExporterUnavailable`: Kubernetes event exporter Pod not ready 2분 지속. - `GrafanaUnavailable`: Grafana Pod not ready 2분 지속. - `GrafanaMetricsTargetDown`: Grafana `/metrics` scrape 실패 또는 target series missing 2분 지속. +- `MonitoringServiceMetricsTargetDown`: monitoring namespace 주요 Service metrics target scrape 실패 또는 target series missing 2분 지속. - `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. - `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. - `PrometheusConfigReloadFailed`: Prometheus generated config reload 실패 또는 reload metric 누락 2분 지속. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index b06daf74..9d2537e1 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -584,7 +584,7 @@ kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condi - latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. - `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. -### GrafanaUnavailable / GrafanaMetricsTargetDown / PrometheusOperatorUnavailable / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures +### GrafanaUnavailable / GrafanaMetricsTargetDown / MonitoringServiceMetricsTargetDown / PrometheusOperatorUnavailable / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures 확인: @@ -602,6 +602,7 @@ PromQL: ```promql kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} up{namespace="monitoring", service="kube-prometheus-stack-grafana"} +up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*"} sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) prometheus_notifications_alertmanagers_discovered @@ -613,6 +614,7 @@ sum by (job, namespace) (up == 0) - Grafana만 down이면 알림은 계속 갈 수 있지만 dashboard/Explore 기반 원인 분석이 막힙니다. - Grafana UI는 살아 있는데 metrics target만 down이면 Grafana 내부 health와 provisioning 상태를 Prometheus에서 못 봅니다. Grafana ServiceMonitor, Service port, `/metrics` 응답, Grafana 로그를 확인합니다. +- monitoring service metrics target이 down이면 Pod readiness가 정상이어도 ServiceMonitor selector, Service port name, generated Prometheus config, network path를 확인합니다. - Operator가 down이면 새 ServiceMonitor/PodMonitor/PrometheusRule/AlertmanagerConfig 변경이 Prometheus/Alertmanager에 반영되지 않을 수 있습니다. - rule evaluation failure가 있으면 최근 추가한 PromQL expression과 label cardinality, metric 존재 여부를 먼저 봅니다. - Alertmanager discovery가 0이면 AlertmanagerConfig가 맞아도 Prometheus가 알림을 보낼 대상이 없습니다. Prometheus generated config와 Alertmanager Service/endpoints를 확인합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index 77516d89..91e69c6f 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -217,6 +217,27 @@ data: {"expr": "sum by (job, namespace) (up == 0) or vector(0)", "legendFormat": "{{namespace}} {{job}}"} ] }, + { + "type": "timeseries", + "title": "Monitoring Service Target Up", + "gridPos": {"x": 8, "y": 36, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "targets": [ + {"expr": "max by (service) (up{namespace=\"monitoring\", service=~\"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*\"})", "legendFormat": "{{service}}"} + ] + }, { "type": "timeseries", "title": "Prometheus TSDB Head Series", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 2496d401..69e7220a 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -407,6 +407,16 @@ spec: annotations: summary: "Grafana metrics target is down" description: "Prometheus cannot scrape Grafana /metrics for 2 minutes. Grafana may still serve UI, but dashboard/datasource/provisioning health visibility is degraded." + - alert: MonitoringServiceMetricsTargetDown + expr: (up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*"} == 0) or (kube_service_info{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*"} unless on (namespace, service) up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "A monitoring service metrics target is down" + description: "A monitoring Service exists but Prometheus cannot scrape its metrics target for 2 minutes, or the target series is missing. Check ServiceMonitor selection, Service port names, Pod readiness, and generated Prometheus config." - alert: PrometheusOperatorUnavailable expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-operator-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-operator-.*"}) for: 2m From 9f57e6ec15b488f6593d5c24279a4d3ca1519bda Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 21:32:08 +0900 Subject: [PATCH 100/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?target=20=EC=95=8C=EB=A6=BC=20=EB=B2=94=EC=9C=84=20=EC=A0=95?= =?UTF-8?q?=EA=B5=90=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/developer/infra/observability-runbook/_index.md | 2 +- kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml | 2 +- kubernetes/monitoring/prometheus-rules.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 9d2537e1..2d526e73 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -602,7 +602,7 @@ PromQL: ```promql kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} up{namespace="monitoring", service="kube-prometheus-stack-grafana"} -up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*"} +up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"} sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) prometheus_notifications_alertmanagers_discovered diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index 91e69c6f..e155bc2b 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -235,7 +235,7 @@ data: } }, "targets": [ - {"expr": "max by (service) (up{namespace=\"monitoring\", service=~\"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*\"})", "legendFormat": "{{service}}"} + {"expr": "max by (service) (up{namespace=\"monitoring\", service=~\"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy\"})", "legendFormat": "{{service}}"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 69e7220a..678c4b34 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -408,7 +408,7 @@ spec: summary: "Grafana metrics target is down" description: "Prometheus cannot scrape Grafana /metrics for 2 minutes. Grafana may still serve UI, but dashboard/datasource/provisioning health visibility is degraded." - alert: MonitoringServiceMetricsTargetDown - expr: (up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*"} == 0) or (kube_service_info{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*"} unless on (namespace, service) up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki.*|alloy.*"}) + expr: (up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"} == 0) or (kube_service_info{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"} unless on (namespace, service) up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"}) for: 2m labels: severity: warning From 09d3c5f81b54aee082db469ae8050a98992c7442 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 21:37:23 +0900 Subject: [PATCH 101/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=20=EC=95=8C=EB=A6=BC?= =?UTF-8?q?=20runbook=20=EA=B2=80=EC=83=89=EC=84=B1=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-runbook/_index.md | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 2d526e73..18f35b3e 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -416,7 +416,7 @@ kubectl -n monitoring describe pvc -l app.kubernetes.io/name=loki - readiness만 실패하면 Loki config, retention, disk full, compactor error를 확인합니다. - Loki가 내려가도 앱 요청 처리는 계속되지만 로그 수집/조회가 끊기므로 빠르게 복구합니다. -### PrometheusUnavailable / PrometheusReplicaUnavailable / AlertmanagerUnavailable +### PrometheusUnavailable / PrometheusReplicaUnavailable / AlertmanagerUnavailable / AlertmanagerReplicaUnavailable 증상: Prometheus 또는 Alertmanager Pod가 ready가 아니거나, Prometheus ready replica가 2개 미만입니다. @@ -433,6 +433,7 @@ kubectl -n monitoring logs alertmanager-kube-prometheus-stack-alertmanager-0 -c ```promql kube_pod_status_ready{namespace="monitoring", pod=~"prometheus-kube-prometheus-stack-prometheus-.*|alertmanager-kube-prometheus-stack-alertmanager-.*", condition="true"} +sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) prometheus_notifications_alertmanagers_discovered prometheus_rule_evaluation_failures_total ``` @@ -444,7 +445,7 @@ prometheus_rule_evaluation_failures_total - Alertmanager가 down이면 firing alert가 있어도 Discord/email 전달이 실패합니다. P0 수신 지연 원인으로 먼저 봅니다. - PVC, config reload, rule syntax, resource pressure, operator reconciliation event를 함께 확인합니다. -### LonghornManagerDown / LonghornVolumeFaulted / LonghornVolumeReadOnly +### LonghornManagerDown / LonghornManagerReplicaUnavailable / LonghornVolumeFaulted / LonghornVolumeReadOnly 증상: Longhorn manager가 전부 ready가 아니거나, volume이 faulted/read-only 상태입니다. @@ -458,6 +459,8 @@ kubectl -n longhorn-system logs -l app=longhorn-manager --tail=200 ```promql sum(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) +kube_daemonset_status_number_available{namespace="longhorn-system", daemonset="longhorn-manager"} +kube_daemonset_status_desired_number_scheduled{namespace="longhorn-system", daemonset="longhorn-manager"} longhorn_volume_robustness longhorn_volume_file_system_read_only longhorn_volume_actual_size_bytes @@ -521,7 +524,7 @@ DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"} ## P1 -### OTelCollectorUnavailable / OTelCollectorRefusedSpans / OTelCollectorExportFailures / TempoUnavailable / TempoPVCAlmostFull +### OTelCollectorUnavailable / OTelCollectorReplicaUnavailable / OTelCollectorRefusedSpans / OTelCollectorExportFailures / TempoUnavailable / TempoPVCAlmostFull 확인: @@ -538,6 +541,8 @@ PromQL: ```promql kube_pod_status_ready{namespace="monitoring", pod=~"otel-collector-.*|tempo-.*", condition="true"} +kube_deployment_status_replicas_available{namespace="monitoring", deployment="otel-collector"} +kube_deployment_spec_replicas{namespace="monitoring", deployment="otel-collector"} max(kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) / max(kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) sum(rate(otelcol_receiver_accepted_spans_total{namespace="monitoring"}[5m])) @@ -566,7 +571,7 @@ sum by (namespace) (codeplace_waiting_queue_length{namespace=""}) backend latency와 DB/Redis readiness, CPU/memory를 같이 봅니다. -### PublicEndpointDown / PublicEndpointLatencyHigh / PublicEndpointTLSExpiringSoon / BlackboxExporterUnavailable +### PublicEndpointDown / PublicEndpointLatencyHigh / PublicEndpointTLSExpiringSoon / BlackboxExporterUnavailable / BlackboxExporterReplicaUnavailable 확인: @@ -576,6 +581,8 @@ probe_duration_seconds{probe_type="public-http"} probe_http_duration_seconds{probe_type="public-http"} (probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) / 86400 kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condition="true"} +kube_deployment_status_replicas_available{namespace="monitoring", deployment="blackbox-exporter"} +kube_deployment_spec_replicas{namespace="monitoring", deployment="blackbox-exporter"} ``` 판단: @@ -584,7 +591,7 @@ kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condi - latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. - `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. -### GrafanaUnavailable / GrafanaMetricsTargetDown / MonitoringServiceMetricsTargetDown / PrometheusOperatorUnavailable / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures +### GrafanaUnavailable / GrafanaMetricsTargetDown / MonitoringServiceMetricsTargetDown / PrometheusOperatorUnavailable / PrometheusConfigReloadFailed / AlertmanagerConfigReloadFailed / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures 확인: @@ -605,6 +612,8 @@ up{namespace="monitoring", service="kube-prometheus-stack-grafana"} up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"} sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) +prometheus_config_last_reload_successful{namespace="monitoring"} +alertmanager_config_last_reload_successful{namespace="monitoring"} prometheus_notifications_alertmanagers_discovered sum by (integration) (increase(alertmanager_notifications_failed_total[5m])) sum by (job, namespace) (up == 0) @@ -876,13 +885,15 @@ topk(10, container_memory_working_set_bytes{namespace="", container!= CPU/memory가 지속적으로 높으면 replica 조정, limit 조정, 최근 배포 변경을 확인합니다. -### PostgresCollectorError / PostgresHADegraded / RedisMemoryHigh / PostgresConnectionUsageHigh / PostgresLongTransactions / PostgresLockWaits / RedisConnectedClientsHigh / RedisRejectedConnections +### PostgresCollectorError / PostgresHADegraded / PostgresReplicaUnavailable / RedisReplicaUnavailable / RedisMemoryHigh / PostgresConnectionUsageHigh / PostgresLongTransactions / PostgresLockWaits / RedisConnectedClientsHigh / RedisRejectedConnections 확인: ```promql cnpg_collector_last_collection_error{namespace="", cluster="postgres"} cnpg_collector_nodes_used{namespace="", cluster="postgres"} +sum by (namespace) (kube_pod_status_ready{namespace="", condition="true", pod=~"postgres-[0-9]+"}) +sum by (namespace) (kube_pod_status_ready{namespace="", condition="true", pod=~"redis-.*|redis-replication-.*"}) redis_memory_used_bytes{namespace=""} redis_memory_max_bytes{namespace=""} codeplace_postgres_connections{namespace=""} @@ -904,6 +915,7 @@ kubectl -n get pod -l app=redis-replication -o wide - `PostgresCollectorError`는 metrics query 실패이므로 CNPG exporter 로그와 PostgreSQL 권한/상태를 확인합니다. - `PostgresHADegraded`는 3개 instance가 서로 다른 node에 분산되지 않은 상태입니다. node 수, taint, Longhorn volume attach 상태를 확인합니다. +- `PostgresReplicaUnavailable` / `RedisReplicaUnavailable`은 3노드 분산 상태에서 남은 replica 수와 node placement를 같이 봅니다. 일부 replica만 down이면 failover 여지는 있지만 다음 장애에 취약합니다. - `PostgresConnectionUsageHigh`는 backend connection leak, worker 동시성 증가, DB max_connections 부족, slow query 누적으로 구분합니다. API latency와 동시에 오르면 DB 병목 후보가 강합니다. - `PostgresLongTransactions`는 transaction 범위가 긴 view/task, idle-in-transaction, migration/admin 작업을 확인합니다. - `PostgresLockWaits`는 쓰기 contention이나 DDL/migration lock 가능성이 큽니다. 같은 시간대 backend error와 Celery runtime 증가를 같이 봅니다. @@ -912,13 +924,15 @@ kubectl -n get pod -l app=redis-replication -o wide - `RedisRejectedConnections`는 Redis `maxclients`에 실제로 걸린 상태입니다. API/cache와 Celery queue가 동시에 영향을 받을 수 있습니다. - Redis memory가 높으면 eviction 정책, queue/backlog, cache key 증가를 확인합니다. -### PVCAlmostFull +### PVCAlmostFull / MonitoringPVCAlmostFull 확인: ```promql max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=""}) / max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=""}) +max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim!~".*loki.*|tempo-data"}) / +max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim!~".*loki.*|tempo-data"}) ``` ```sh @@ -1055,7 +1069,7 @@ kubectl -n monitoring describe pvc - 당장 여유가 없으면 Longhorn PVC size를 증설합니다. - 반복되면 retention 기간을 줄이거나 prod 로그 저장 방식을 내부 S3 호환 object storage로 분리합니다. -### AlloyDaemonSetUnavailable / LokiGatewayUnavailable / LokiIngestionStalled / LokiRequestErrors / LokiCanaryMissingEntries / AlloyLokiWriteDrops / AlloyLokiWriteRetries +### AlloyDaemonSetUnavailable / LokiGatewayUnavailable / LokiGatewayReplicaUnavailable / LokiIngestionStalled / LokiRequestErrors / LokiCanaryMissingEntries / AlloyLokiWriteDrops / AlloyLokiWriteRetries 확인: @@ -1072,6 +1086,7 @@ PromQL: ```promql sum(rate(loki_distributor_lines_received_total{namespace="monitoring"}[5m])) +sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) sum(rate(loki_request_duration_seconds_count{namespace="monitoring", status_code=~"5.."}[5m])) sum(increase(loki_canary_missing_entries_total{namespace="monitoring"}[5m])) sum(increase(loki_write_dropped_entries_total{namespace="monitoring"}[5m])) From 8784d2a151fba807f45da2f8287094607ebd1440 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 21:43:50 +0900 Subject: [PATCH 102/147] =?UTF-8?q?Trace=20=EB=8C=80=EC=8B=9C=EB=B3=B4?= =?UTF-8?q?=EB=93=9C=20=EC=A1=B0=ED=9A=8C=20=ED=8C=A8=EB=84=90=20=EB=B3=B4?= =?UTF-8?q?=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../monitoring/grafana-dashboard-traces.yaml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/kubernetes/monitoring/grafana-dashboard-traces.yaml b/kubernetes/monitoring/grafana-dashboard-traces.yaml index f333003a..3831d174 100644 --- a/kubernetes/monitoring/grafana-dashboard-traces.yaml +++ b/kubernetes/monitoring/grafana-dashboard-traces.yaml @@ -143,6 +143,34 @@ data: {"expr": "sum(rate(otelcol_receiver_refused_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "refused"}, {"expr": "sum(rate(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "failed"} ] + }, + { + "type": "table", + "title": "Recent Backend/Celery Traces", + "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, + "datasource": "Tempo", + "targets": [ + { + "refId": "A", + "queryType": "traceql", + "query": "{ resource.service.name =~ \"codeplace-(backend|celery)\" }", + "limit": 20 + } + ] + }, + { + "type": "table", + "title": "Submission/Judge Trace Search", + "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, + "datasource": "Tempo", + "targets": [ + { + "refId": "A", + "queryType": "traceql", + "query": "{ name =~ \"judge_task|submission.judge|judge_server.select|judge_server.request|submission.result_save\" }", + "limit": 20 + } + ] } ] } From 10c6d99b466fc9b00f393c525b87a0dd1c7b5155 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:00:59 +0900 Subject: [PATCH 103/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=EC=9C=A0=EC=9E=85?= =?UTF-8?q?=20=EB=8C=80=EC=8B=9C=EB=B3=B4=EB=93=9C=20=EC=84=B8=EB=B6=84?= =?UTF-8?q?=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-runbook/_index.md | 9 ++++ .../monitoring/grafana-dashboard-logs.yaml | 52 +++++++++++++++---- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 18f35b3e..5e1a4fcf 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -1093,11 +1093,20 @@ sum(increase(loki_write_dropped_entries_total{namespace="monitoring"}[5m])) sum(increase(loki_write_batch_retries_total{namespace="monitoring"}[5m])) ``` +LogQL: + +```logql +sum by (namespace) (rate({namespace=~"code-place-dev|code-place-prod|monitoring|kube-system"}[5m])) +sum by (namespace, app) (rate({namespace=~"code-place-dev|code-place-prod", app=~"backend|frontend"}[5m])) +sum by (namespace, container) (rate({namespace=~"code-place-dev|code-place-prod", container=~"celery-worker|celery-beat|judge-server|hub-auth|vllm"}[5m])) +``` + 판단: - Alloy가 특정 node에서만 비면 해당 node의 scheduling, taint, resource pressure를 봅니다. - Loki gateway가 비면 Alloy write와 Grafana query가 실패할 수 있습니다. - Loki가 ready인데 ingest rate가 0이면 Alloy discovery/relabel, node별 container log path, gateway write path를 먼저 확인합니다. +- Loki 전체 ingest rate는 정상인데 특정 namespace/app 로그만 0이면 해당 Pod stdout/stderr, container restart, Alloy relabel label, node별 Alloy target discovery를 확인합니다. - Loki 5xx가 있으면 gateway와 `loki-0` 로그를 같이 보고, PVC full/Longhorn degraded/compactor error를 확인합니다. - Loki canary missing entry는 write 성공 후 readback 또는 storage/query path가 깨진 상태일 수 있습니다. - Alloy write drop/retry가 있으면 Loki gateway 응답, network, Loki backpressure, Alloy resource limit을 확인합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index 5adcb9f1..ebad60f2 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -159,10 +159,42 @@ data: {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 500 [5m]))", "legendFormat": "5xx"} ] }, + { + "type": "timeseries", + "title": "Log Lines by Namespace", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 7}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum by (namespace) (rate({namespace=~\"code-place-dev|code-place-prod|monitoring|kube-system\"}[5m]))", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "timeseries", + "title": "Application Log Lines by App", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 7}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum by (namespace, app) (rate({namespace=~\"code-place-dev|code-place-prod\", app=~\"backend|frontend\"}[5m]))", "legendFormat": "{{namespace}} {{app}}"}, + {"expr": "sum by (namespace, container) (rate({namespace=~\"code-place-dev|code-place-prod\", container=~\"celery-worker|celery-beat|judge-server|hub-auth|vllm\"}[5m]))", "legendFormat": "{{namespace}} {{container}}"} + ] + }, + { + "type": "timeseries", + "title": "Structured Error Logs by Component", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 7}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum by (namespace, app) (rate({namespace=~\"code-place-dev|code-place-prod\", app=~\"backend|frontend\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{app}}"}, + {"expr": "sum by (namespace, container) (rate({namespace=~\"code-place-dev|code-place-prod\", container=~\"celery-worker|celery-beat|judge-server|hub-auth|vllm\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{container}}"} + ] + }, { "type": "stat", "title": "Ingress 5xx", - "gridPos": {"x": 0, "y": 4, "w": 4, "h": 4}, + "gridPos": {"x": 0, "y": 11, "w": 4, "h": 4}, "datasource": "Loki", "fieldConfig": { "defaults": { @@ -185,7 +217,7 @@ data: { "type": "logs", "title": "Recent Backend Errors", - "gridPos": {"x": 4, "y": 4, "w": 10, "h": 10}, + "gridPos": {"x": 4, "y": 11, "w": 10, "h": 10}, "datasource": "Loki", "options": { "showLabels": false, @@ -200,7 +232,7 @@ data: { "type": "logs", "title": "Auth, Judge, Celery, and vLLM Logs", - "gridPos": {"x": 14, "y": 4, "w": 10, "h": 10}, + "gridPos": {"x": 14, "y": 11, "w": 10, "h": 10}, "datasource": "Loki", "options": { "showLabels": false, @@ -215,7 +247,7 @@ data: { "type": "logs", "title": "Ingress and Frontend 4xx/5xx Logs", - "gridPos": {"x": 0, "y": 14, "w": 24, "h": 8}, + "gridPos": {"x": 0, "y": 21, "w": 24, "h": 8}, "datasource": "Loki", "options": { "showLabels": false, @@ -231,7 +263,7 @@ data: { "type": "logs", "title": "Request ID Lookup", - "gridPos": {"x": 0, "y": 22, "w": 24, "h": 9}, + "gridPos": {"x": 0, "y": 29, "w": 24, "h": 9}, "datasource": "Loki", "options": { "showLabels": false, @@ -246,7 +278,7 @@ data: { "type": "logs", "title": "Frontend Runtime Error Reports", - "gridPos": {"x": 0, "y": 31, "w": 24, "h": 8}, + "gridPos": {"x": 0, "y": 38, "w": 24, "h": 8}, "datasource": "Loki", "options": { "showLabels": false, @@ -261,7 +293,7 @@ data: { "type": "timeseries", "title": "Loki Ingestion Rate", - "gridPos": {"x": 0, "y": 39, "w": 8, "h": 7}, + "gridPos": {"x": 0, "y": 46, "w": 8, "h": 7}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "logs/sec"}}, "targets": [ @@ -272,7 +304,7 @@ data: { "type": "timeseries", "title": "Loki Request Errors", - "gridPos": {"x": 8, "y": 39, "w": 8, "h": 7}, + "gridPos": {"x": 8, "y": 46, "w": 8, "h": 7}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "reqps"}}, "targets": [ @@ -283,7 +315,7 @@ data: { "type": "timeseries", "title": "Alloy Loki Write Backpressure", - "gridPos": {"x": 16, "y": 39, "w": 8, "h": 7}, + "gridPos": {"x": 16, "y": 46, "w": 8, "h": 7}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ @@ -294,7 +326,7 @@ data: { "type": "timeseries", "title": "Loki Canary Missing Entries", - "gridPos": {"x": 0, "y": 46, "w": 8, "h": 6}, + "gridPos": {"x": 0, "y": 53, "w": 8, "h": 6}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ From 07af749ff1fce5b5dedc020edbb584594725f96b Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:03:38 +0900 Subject: [PATCH 104/147] =?UTF-8?q?=EC=95=8C=EB=A6=BC=20=EB=A9=94=EC=8B=9C?= =?UTF-8?q?=EC=A7=80=20=EC=A7=84=EB=8B=A8=20=EB=A7=81=ED=81=AC=20=EB=B3=B4?= =?UTF-8?q?=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../monitoring/alertmanager-config.yaml | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/kubernetes/monitoring/alertmanager-config.yaml b/kubernetes/monitoring/alertmanager-config.yaml index 4242c086..795a0da6 100644 --- a/kubernetes/monitoring/alertmanager-config.yaml +++ b/kubernetes/monitoring/alertmanager-config.yaml @@ -47,7 +47,16 @@ spec: key: webhook-url title: '[{{ .Status }}][P0] {{ .CommonLabels.alertname }}' content: '{{ .CommonAnnotations.summary }}' - message: '{{ .CommonAnnotations.description }}' + message: |- + {{ .CommonAnnotations.description }} + + Priority: {{ .CommonLabels.priority }} + Namespace: {{ .CommonLabels.namespace }} + Overview: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-namespace={{ .CommonLabels.namespace }} + Logs: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-namespace={{ .CommonLabels.namespace }} + Monitoring: https://monitoring.code-place-dev.site/d/codeplace-monitoring-stack/codeplace-monitoring-stack?orgId=1 + Runbook: https://github.com/pnu-code-place/code-place/blob/develop/docs/content/developer/infra/observability-runbook/_index.md + Source: {{ range .Alerts }}{{ .GeneratorURL }}{{ end }} username: CodePlace Alertmanager - name: dev-p1-muted - name: p1-discord @@ -58,5 +67,14 @@ spec: key: webhook-url title: '[{{ .Status }}][P1] {{ .CommonLabels.alertname }}' content: '{{ .CommonAnnotations.summary }}' - message: '{{ .CommonAnnotations.description }}' + message: |- + {{ .CommonAnnotations.description }} + + Priority: {{ .CommonLabels.priority }} + Namespace: {{ .CommonLabels.namespace }} + Overview: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-namespace={{ .CommonLabels.namespace }} + Logs: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-namespace={{ .CommonLabels.namespace }} + Monitoring: https://monitoring.code-place-dev.site/d/codeplace-monitoring-stack/codeplace-monitoring-stack?orgId=1 + Runbook: https://github.com/pnu-code-place/code-place/blob/develop/docs/content/developer/infra/observability-runbook/_index.md + Source: {{ range .Alerts }}{{ .GeneratorURL }}{{ end }} username: CodePlace Alertmanager From 4b3a7f1311064c7ba5536b4768a5813c94c97a75 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:06:10 +0900 Subject: [PATCH 105/147] =?UTF-8?q?Frontend=20readiness=20probe=20?= =?UTF-8?q?=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/base/frontend/deployment.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/kubernetes/base/frontend/deployment.yaml b/kubernetes/base/frontend/deployment.yaml index d60ffe24..e95b0a1f 100644 --- a/kubernetes/base/frontend/deployment.yaml +++ b/kubernetes/base/frontend/deployment.yaml @@ -42,6 +42,29 @@ spec: value: "0" - name: SERVER_NAME value: "localhost" + startupProbe: + httpGet: + path: / + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 24 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 volumeMounts: - name: vol-public mountPath: /data/public From 6f0929515fcb895799ccaa2eef5a5390f6ba552a Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:08:35 +0900 Subject: [PATCH 106/147] =?UTF-8?q?=EC=82=AC=EC=9A=A9=EC=9E=90=20=EA=B2=BD?= =?UTF-8?q?=EB=A1=9C=20PDB=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/base/backend/pdb.yaml | 9 +++++++++ kubernetes/base/frontend/pdb.yaml | 9 +++++++++ kubernetes/base/kustomization.yaml | 4 ++++ 3 files changed, 22 insertions(+) create mode 100644 kubernetes/base/backend/pdb.yaml create mode 100644 kubernetes/base/frontend/pdb.yaml diff --git a/kubernetes/base/backend/pdb.yaml b/kubernetes/base/backend/pdb.yaml new file mode 100644 index 00000000..38357806 --- /dev/null +++ b/kubernetes/base/backend/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: backend +spec: + minAvailable: 1 + selector: + matchLabels: + app: backend diff --git a/kubernetes/base/frontend/pdb.yaml b/kubernetes/base/frontend/pdb.yaml new file mode 100644 index 00000000..86e6d780 --- /dev/null +++ b/kubernetes/base/frontend/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: frontend +spec: + minAvailable: 1 + selector: + matchLabels: + app: frontend diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml index 3c1362d6..f2311043 100644 --- a/kubernetes/base/kustomization.yaml +++ b/kubernetes/base/kustomization.yaml @@ -19,6 +19,10 @@ resources: - ./judge-server/service.yaml - ./hub-auth/service.yaml + # Application Disruption Budgets + - ./backend/pdb.yaml + - ./frontend/pdb.yaml + # Application Deployments - ./backend/deployment.yaml - ./frontend/deployment.yaml From a06bb70409b16ace2c02963beb3aeabff03c32b0 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:13:23 +0900 Subject: [PATCH 107/147] =?UTF-8?q?PDB=20=EC=9C=84=ED=97=98=20=EC=83=81?= =?UTF-8?q?=ED=83=9C=20=EA=B4=80=EC=B8=A1=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../developer/infra/observability-runbook/_index.md | 5 ++++- .../monitoring/grafana-dashboard-codeplace.yaml | 12 +++++++++++- kubernetes/monitoring/prometheus-rules.yaml | 9 +++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 5e1a4fcf..76c6376a 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -821,7 +821,7 @@ Grafana Explore: - `OOMKilled`는 memory limit, 최근 트래픽, 캐시/큐 증가, JVM/Python heap 설정을 확인합니다. - `Pending`/`Unschedulable`은 node capacity, taint/toleration, nodeSelector/affinity, PVC binding, Longhorn volume attach event를 확인합니다. -### CodePlaceDeploymentUnavailable / CodePlaceDeploymentRolloutStuck +### CodePlaceDeploymentUnavailable / CodePlaceDeploymentRolloutStuck / CodePlacePDBAtRisk 확인: @@ -838,6 +838,8 @@ PromQL: kube_deployment_status_replicas_unavailable{namespace=""} kube_deployment_status_observed_generation{namespace=""} kube_deployment_metadata_generation{namespace=""} +kube_poddisruptionbudget_status_current_healthy{namespace="", poddisruptionbudget=~"backend|frontend"} +kube_poddisruptionbudget_status_desired_healthy{namespace="", poddisruptionbudget=~"backend|frontend"} kube_pod_container_status_waiting_reason{namespace=""} ``` @@ -845,6 +847,7 @@ kube_pod_container_status_waiting_reason{namespace=""} - unavailable replica가 늘면 readiness probe, image pull, crash loop, resource pressure, scheduling event를 먼저 봅니다. - observed generation이 metadata generation을 따라가지 못하면 Deployment controller reconcile, ReplicaSet 생성 실패, admission/image pull/secret 문제를 확인합니다. +- PDB current healthy가 desired healthy 이하이면 node drain/upgrade 전에 backend/frontend replica readiness를 먼저 복구합니다. - 배포 직후 API 5xx/latency와 같이 발생하면 사용자 영향이 있는 rollout 장애로 봅니다. ### CodePlaceServiceNoReadyEndpoints / VLLMServiceNoReadyEndpoints diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 829e5211..c4caeeb6 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -391,7 +391,7 @@ data: { "type": "timeseries", "title": "Service Ready Endpoints", - "gridPos": {"x": 0, "y": 48, "w": 24, "h": 6}, + "gridPos": {"x": 0, "y": 48, "w": 12, "h": 6}, "fieldConfig": { "defaults": { "unit": "short", @@ -408,6 +408,16 @@ data: {"expr": "max by (namespace, endpoint) (kube_endpoint_address_available{namespace=~\"$namespace\", endpoint=~\"backend|frontend|hub-auth|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{endpoint}} ready"} ] }, + { + "type": "timeseries", + "title": "PDB Healthy Pods", + "gridPos": {"x": 12, "y": 48, "w": 12, "h": 6}, + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "kube_poddisruptionbudget_status_current_healthy{namespace=~\"$namespace\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} current"}, + {"expr": "kube_poddisruptionbudget_status_desired_healthy{namespace=~\"$namespace\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} desired"} + ] + }, { "type": "row", "title": "Data Stores", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 678c4b34..ea4ed489 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -685,6 +685,15 @@ spec: annotations: summary: "CodePlace deployment rollout is stuck" description: "A CodePlace deployment observed generation has not caught up with metadata generation for 10 minutes. Check deployment events, ReplicaSet state, and rollout history." + - alert: CodePlacePDBAtRisk + expr: kube_poddisruptionbudget_status_current_healthy{namespace=~"code-place-(dev|prod)", poddisruptionbudget=~"backend|frontend"} <= kube_poddisruptionbudget_status_desired_healthy{namespace=~"code-place-(dev|prod)", poddisruptionbudget=~"backend|frontend"} + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace PDB has no disruption headroom" + description: "A backend or frontend PodDisruptionBudget has been at or below its desired healthy count for 10 minutes. Node drain or voluntary disruption may reduce user-path availability." - alert: CodePlaceServiceNoReadyEndpoints expr: (max by (namespace, endpoint) (kube_endpoint_address_available{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server"}) < 1) or (label_replace(kube_service_info{namespace=~"code-place-(dev|prod)", service=~"backend|frontend|hub-auth|judge-server"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address_available{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server"}) for: 2m From 6b140fa5323d9b790f357043f44fd6fe988b9c01 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:16:47 +0900 Subject: [PATCH 108/147] =?UTF-8?q?Grafana=20=EA=B4=80=EC=B8=A1=20?= =?UTF-8?q?=EB=8C=80=EC=83=81=20=EC=A0=95=EA=B7=9C=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../developer/infra/observability-runbook/_index.md | 4 ++-- .../monitoring/grafana-dashboard-monitoring-stack.yaml | 8 ++++---- kubernetes/monitoring/prometheus-rules.yaml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 76c6376a..af882b49 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -74,7 +74,7 @@ kubectl -n monitoring get alertmanager kube-prometheus-stack-alertmanager -o jso ``` ```promql -kube_pod_status_ready{namespace="monitoring", pod=~"prometheus-kube-prometheus-stack-prometheus-.*|alertmanager-kube-prometheus-stack-alertmanager-.*|kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} +kube_pod_status_ready{namespace="monitoring", pod=~"prometheus-kube-prometheus-stack-prometheus-.*|alertmanager-kube-prometheus-stack-alertmanager-.*|kube-prometheus-stack-grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} prometheus_notifications_alertmanagers_discovered increase(prometheus_rule_evaluation_failures_total[5m]) increase(alertmanager_notifications_failed_total[5m]) @@ -607,7 +607,7 @@ kubectl -n monitoring get prometheusrule,servicemonitor,podmonitor,alertmanagerc PromQL: ```promql -kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} +kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} up{namespace="monitoring", service="kube-prometheus-stack-grafana"} up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"} sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index e155bc2b..f685868b 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -84,7 +84,7 @@ data: }, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kube-prometheus-stack-grafana-.*|grafana-.*\"}) or vector(0)", "legendFormat": "grafana"} + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kube-prometheus-stack-grafana-.*\"}) or vector(0)", "legendFormat": "grafana"} ] }, { @@ -204,7 +204,7 @@ data: "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=\"monitoring\", pod=~\"prometheus-.*|alertmanager-.*|kube-prometheus-stack-grafana-.*|grafana-.*|kube-prometheus-stack-operator-.*\"}[1h]))", "legendFormat": "{{pod}}"} + {"expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=\"monitoring\", pod=~\"prometheus-.*|alertmanager-.*|kube-prometheus-stack-grafana-.*|kube-prometheus-stack-operator-.*\"}[1h]))", "legendFormat": "{{pod}}"} ] }, { @@ -289,8 +289,8 @@ data: "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*|grafana-.*\", container!=\"\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, - {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*|grafana-.*\", container!=\"\"})", "legendFormat": "{{pod}} memory bytes"} + {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*\", container!=\"\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, + {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*\", container!=\"\"})", "legendFormat": "{{pod}} memory bytes"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index ea4ed489..ad880e94 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -388,7 +388,7 @@ spec: summary: "Kubernetes event exporter is not ready" description: "Kubernetes Warning events may not be available in Loki for deployment and scheduling failure triage." - alert: GrafanaUnavailable - expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*|grafana-.*"}) + expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*"}) for: 2m labels: severity: warning @@ -396,7 +396,7 @@ spec: namespace: monitoring annotations: summary: "Grafana is not ready" - description: "No Grafana pod in the monitoring namespace has been ready for 2 minutes. Dashboards and Explore may be unavailable." + description: "No kube-prometheus-stack Grafana pod in the monitoring namespace has been ready for 2 minutes. Dashboards and Explore may be unavailable." - alert: GrafanaMetricsTargetDown expr: (up{namespace="monitoring", service="kube-prometheus-stack-grafana"} == 0) or absent(up{namespace="monitoring", service="kube-prometheus-stack-grafana"}) for: 2m From 614cb88ca2f3c8e55132d2c6933a5ab0237b5e9c Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:20:54 +0900 Subject: [PATCH 109/147] =?UTF-8?q?=EB=A1=9C=EA=B7=B8=20=EB=85=B8=EB=93=9C?= =?UTF-8?q?=EB=B3=84=20=EA=B4=80=EC=B8=A1=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../developer/infra/observability-runbook/_index.md | 3 ++- kubernetes/monitoring/grafana-dashboard-logs.yaml | 10 ++++++++++ kubernetes/monitoring/logs/alloy-values.yaml | 5 +++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index af882b49..f700c206 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -1100,13 +1100,14 @@ LogQL: ```logql sum by (namespace) (rate({namespace=~"code-place-dev|code-place-prod|monitoring|kube-system"}[5m])) +sum by (node) (rate({namespace=~"code-place-dev|code-place-prod|monitoring|kube-system"}[5m])) sum by (namespace, app) (rate({namespace=~"code-place-dev|code-place-prod", app=~"backend|frontend"}[5m])) sum by (namespace, container) (rate({namespace=~"code-place-dev|code-place-prod", container=~"celery-worker|celery-beat|judge-server|hub-auth|vllm"}[5m])) ``` 판단: -- Alloy가 특정 node에서만 비면 해당 node의 scheduling, taint, resource pressure를 봅니다. +- Alloy 또는 LogQL node별 유입이 특정 node에서만 비면 해당 node의 scheduling, taint, resource pressure, `/var/log/pods` mount 상태를 봅니다. - Loki gateway가 비면 Alloy write와 Grafana query가 실패할 수 있습니다. - Loki가 ready인데 ingest rate가 0이면 Alloy discovery/relabel, node별 container log path, gateway write path를 먼저 확인합니다. - Loki 전체 ingest rate는 정상인데 특정 namespace/app 로그만 0이면 해당 Pod stdout/stderr, container restart, Alloy relabel label, node별 Alloy target discovery를 확인합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index ebad60f2..48e62d9a 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -214,6 +214,16 @@ data: {"expr": "sum(count_over_time({namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | DownstreamStatus >= 500 [5m]))", "legendFormat": "5xx"} ] }, + { + "type": "timeseries", + "title": "Log Lines by Node", + "gridPos": {"x": 0, "y": 15, "w": 4, "h": 6}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum by (node) (rate({namespace=~\"code-place-dev|code-place-prod|monitoring|kube-system\"}[5m]))", "legendFormat": "{{node}}"} + ] + }, { "type": "logs", "title": "Recent Backend Errors", diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml index 2007bf8b..25c9631a 100644 --- a/kubernetes/monitoring/logs/alloy-values.yaml +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -49,6 +49,11 @@ alloy: target_label = "container" } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { source_labels = ["__meta_kubernetes_pod_label_app"] action = "replace" From b5f3067f6f4c84142b6d3e893d5b0a56c1d0676c Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:39:51 +0900 Subject: [PATCH 110/147] =?UTF-8?q?=EC=B1=84=EC=A0=90=20=EC=86=8C=EC=9A=94?= =?UTF-8?q?=20=EC=8B=9C=EA=B0=84=20=EA=B4=80=EC=B8=A1=20=EA=B8=B0=EC=A4=80?= =?UTF-8?q?=20=EB=B3=B4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_metrics.py | 2 +- backend/utils/tests.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 20866120..a3dd51f7 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -282,7 +282,7 @@ def _judge_duration_histogram(self): durations = [] try: submissions = Submission.objects.filter( - judge_start_time__gte=cutoff, + judge_end_time__gte=cutoff, judge_start_time__isnull=False, judge_end_time__isnull=False, ).only("judge_start_time", "judge_end_time") diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 07603dc8..c0dffd0a 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -83,6 +83,32 @@ def test_submission_oldest_age_metrics_are_collected_for_inflight_statuses(self) self.assertGreaterEqual(by_status["pending"], 419) self.assertGreaterEqual(by_status["judging"], 179) + def test_judge_duration_histogram_uses_recent_completion_time(self): + now = timezone.now() + submission = MagicMock( + judge_start_time=now - timedelta(minutes=20), + judge_end_time=now - timedelta(minutes=1), + ) + queryset = MagicMock() + queryset.only.return_value = [submission] + + with patch("utils.observability_metrics.timezone.now", return_value=now), \ + patch("utils.observability_metrics.Submission.objects.filter", return_value=queryset) as filter_mock: + samples = self._samples_by_metric([CodePlaceCollector()._judge_duration_histogram()]) + + filter_kwargs = filter_mock.call_args.kwargs + self.assertEqual(filter_kwargs["judge_end_time__gte"], now - timedelta(minutes=10)) + self.assertNotIn("judge_start_time__gte", filter_kwargs) + bucket_samples = [ + sample for sample in samples["codeplace_submission_judge_duration_seconds"] + if sample.name.endswith("_bucket") + ] + self.assertTrue(any(sample.labels["le"] == "+Inf" and sample.value == 1 for sample in bucket_samples)) + self.assertTrue(any( + sample.name.endswith("_sum") and sample.value == 1140 + for sample in samples["codeplace_submission_judge_duration_seconds"] + )) + def test_celery_task_metrics_are_collected_from_redis(self): hashes = { celery_observability.CELERY_TASK_TOTAL_KEY: { From 06191a2220d8b5e9e77542e19f04732507128694 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 22:47:47 +0900 Subject: [PATCH 111/147] =?UTF-8?q?=EC=8A=A4=EC=BC=80=EC=A4=84=20=EC=9E=91?= =?UTF-8?q?=EC=97=85=20=EB=AF=B8=EA=B4=80=EC=B8=A1=20=EC=95=8C=EB=A6=BC=20?= =?UTF-8?q?=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-runbook/_index.md | 1 + kubernetes/monitoring/prometheus-rules.yaml | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index f700c206..99963556 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -742,6 +742,7 @@ Grafana Explore: - `judge.tasks.judge_task` 실패와 `JudgeWaitingQueueBacklog`가 같이 발생하면 judge-server availability와 Redis `waiting_queue`를 우선 확인합니다. - scheduled task stale은 beat가 task를 발행하지 못했거나 worker가 성공 종료하지 못한 상태입니다. `calculate_user_score_fluctuation`은 1분 주기라 10분 stale이면 빠르게 확인하고, `calculate_user_score_basis`는 일간, `update_weekly_stats`와 `update_bonus_problem`은 주간 기준으로 봅니다. +- `codeplace_celery_task_last_success_age_seconds` series가 아예 없으면 해당 task 성공이 한 번도 관측되지 않은 상태입니다. 배포 직후라면 beat schedule 등록, worker task autodiscovery, Redis metric write, backend `/metrics` collector를 순서대로 확인합니다. - task metric은 worker가 Redis에 누적하고 backend `/metrics` collector가 노출합니다. worker 로그에는 `task_id`, `task_name`, `task_status`, `duration_ms`, `trace_id`가 포함됩니다. ### PodCrashLooping / CodePlacePodNotReady diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index ad880e94..4ecac41a 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -587,32 +587,43 @@ spec: summary: "Celery task runtime is high" description: "Celery task p95 runtime has been above 120 seconds for 10 minutes." - alert: CeleryMinuteScheduledTaskStale - expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_fluctuation"} > 600 + expr: >- + (codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_fluctuation"} > 600) + or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-dev", task_name="account.tasks.calculate_user_score_fluctuation"}) + or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-prod", task_name="account.tasks.calculate_user_score_fluctuation"}) for: 5m labels: severity: warning priority: P1 annotations: summary: "Minute scheduled Celery task is stale" - description: "calculate_user_score_fluctuation has not completed successfully for more than 10 minutes. Check celery-beat, celery-worker, Redis broker, and task logs." + description: "calculate_user_score_fluctuation has not completed successfully for more than 10 minutes, or no success metric has ever been observed. Check celery-beat, celery-worker, Redis broker, and task logs." - alert: CeleryDailyScheduledTaskStale - expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_basis"} > 93600 + expr: >- + (codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_basis"} > 93600) + or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-dev", task_name="account.tasks.calculate_user_score_basis"}) + or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-prod", task_name="account.tasks.calculate_user_score_basis"}) for: 30m labels: severity: warning priority: P1 annotations: summary: "Daily scheduled Celery task is stale" - description: "calculate_user_score_basis has not completed successfully for more than 26 hours. Check celery-beat, celery-worker, Redis broker, and task logs." + description: "calculate_user_score_basis has not completed successfully for more than 26 hours, or no success metric has ever been observed. Check celery-beat, celery-worker, Redis broker, and task logs." - alert: CeleryWeeklyScheduledTaskStale - expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name=~"problem.tasks.update_weekly_stats|problem.tasks.update_bonus_problem"} > 691200 + expr: >- + (codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name=~"problem.tasks.update_weekly_stats|problem.tasks.update_bonus_problem"} > 691200) + or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-dev", task_name="problem.tasks.update_weekly_stats"}) + or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-prod", task_name="problem.tasks.update_weekly_stats"}) + or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-dev", task_name="problem.tasks.update_bonus_problem"}) + or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-prod", task_name="problem.tasks.update_bonus_problem"}) for: 1h labels: severity: warning priority: P1 annotations: summary: "Weekly scheduled Celery task is stale" - description: "A weekly problem maintenance task has not completed successfully for more than 8 days. Check celery-beat, celery-worker, Redis broker, and task logs." + description: "A weekly problem maintenance task has not completed successfully for more than 8 days, or no success metric has ever been observed. Check celery-beat, celery-worker, Redis broker, and task logs." - alert: PodCrashLooping expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|vllm-.*"}[5m])) > 0 for: 5m From 2313d142a3e28b574a644e15f8e2646eb0a25a3b Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 23:42:31 +0900 Subject: [PATCH 112/147] =?UTF-8?q?=EC=B1=84=EC=A0=90=20=EC=99=84=EB=A3=8C?= =?UTF-8?q?=20=EC=8B=9C=EA=B0=84=20=EA=B4=80=EC=B8=A1=20=EC=9D=B8=EB=8D=B1?= =?UTF-8?q?=EC=8A=A4=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../0005_submission_judge_end_time_index.py | 30 +++++++++++++++++++ backend/submission/models.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 backend/submission/migrations/0005_submission_judge_end_time_index.py diff --git a/backend/submission/migrations/0005_submission_judge_end_time_index.py b/backend/submission/migrations/0005_submission_judge_end_time_index.py new file mode 100644 index 00000000..c7e59a91 --- /dev/null +++ b/backend/submission/migrations/0005_submission_judge_end_time_index.py @@ -0,0 +1,30 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("submission", "0004_auto_20250904_1525"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunSQL( + sql=( + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS ' + '"submission_judge_end_time_c28a7535" ON "submission" ("judge_end_time")' + ), + reverse_sql='DROP INDEX CONCURRENTLY IF EXISTS "submission_judge_end_time_c28a7535"', + ), + ], + state_operations=[ + migrations.AlterField( + model_name="submission", + name="judge_end_time", + field=models.DateTimeField(db_index=True, null=True), + ), + ], + ), + ] diff --git a/backend/submission/models.py b/backend/submission/models.py index 381510e5..8b1d739c 100644 --- a/backend/submission/models.py +++ b/backend/submission/models.py @@ -27,7 +27,7 @@ class Submission(models.Model): problem = models.ForeignKey(Problem, on_delete=models.CASCADE) create_time = models.DateTimeField(auto_now_add=True) judge_start_time = models.DateTimeField(null=True) - judge_end_time = models.DateTimeField(null=True) + judge_end_time = models.DateTimeField(null=True, db_index=True) user_id = models.IntegerField(db_index=True) username = models.TextField() code = models.TextField() From 478903fcde058cee516198ad0e206070213d04d7 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 23:53:58 +0900 Subject: [PATCH 113/147] =?UTF-8?q?=EC=A0=9C=EC=B6=9C=20=EB=8C=80=EA=B8=B0?= =?UTF-8?q?=20=EA=B4=80=EC=B8=A1=20=EC=9D=B8=EB=8D=B1=EC=8A=A4=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...006_submission_result_create_time_index.py | 32 +++++++++++++++++++ backend/submission/models.py | 3 ++ 2 files changed, 35 insertions(+) create mode 100644 backend/submission/migrations/0006_submission_result_create_time_index.py diff --git a/backend/submission/migrations/0006_submission_result_create_time_index.py b/backend/submission/migrations/0006_submission_result_create_time_index.py new file mode 100644 index 00000000..cdf17043 --- /dev/null +++ b/backend/submission/migrations/0006_submission_result_create_time_index.py @@ -0,0 +1,32 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("submission", "0005_submission_judge_end_time_index"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunSQL( + sql=( + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "sub_result_ctime_idx" ' + 'ON "submission" ("result", "create_time")' + ), + reverse_sql='DROP INDEX CONCURRENTLY IF EXISTS "sub_result_ctime_idx"', + ), + ], + state_operations=[ + migrations.AddIndex( + model_name="submission", + index=models.Index( + fields=["result", "create_time"], + name="sub_result_ctime_idx", + ), + ), + ], + ), + ] diff --git a/backend/submission/models.py b/backend/submission/models.py index 8b1d739c..0020a0ce 100644 --- a/backend/submission/models.py +++ b/backend/submission/models.py @@ -58,6 +58,9 @@ def check_user_permission(self, user, check_share=True): class Meta: db_table = "submission" ordering = ("-create_time",) + indexes = [ + models.Index(fields=["result", "create_time"], name="sub_result_ctime_idx"), + ] def __str__(self): return self.id From 61ff3945c79a08b0d6e2c5ecd4a9ff920253a7dd Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 23:55:50 +0900 Subject: [PATCH 114/147] =?UTF-8?q?Alloy=20=EB=85=B8=EB=93=9C=20=EB=A1=9C?= =?UTF-8?q?=EA=B7=B8=20=EC=88=98=EC=A7=91=20=EB=B2=94=EC=9C=84=20=EB=B3=B4?= =?UTF-8?q?=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/logs/README.md | 3 ++- kubernetes/monitoring/logs/alloy-values.yaml | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index e0449902..eced9013 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -4,7 +4,7 @@ This directory contains Helm values for the Kubernetes logs stack. - Loki: SingleBinary mode, filesystem storage, Longhorn PVC. - Loki gateway: 2 replicas for the stateless write/query entrypoint. -- Alloy: DaemonSet log collector for every pod in `code-place-dev`, `code-place-prod`, and `monitoring`, plus Traefik pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. +- Alloy: DaemonSet log collector for every pod in `code-place-dev`, `code-place-prod`, and `monitoring`, plus Traefik pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. It tolerates the standard `control-plane` and `master` NoSchedule taints so the current three-node cluster does not silently miss logs from a tainted control-plane node. - Kubernetes Event Exporter: Warning events are written to stdout as JSON and collected by Alloy from the `monitoring` namespace. - Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. - Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. @@ -78,6 +78,7 @@ Keep these invariants unless the storage design changes intentionally: - Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. - Loki uses filesystem storage on a Longhorn PVC. - In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors, Loki gateway, and probe exporters can be replicated across nodes. +- If a node uses a custom NoSchedule taint, add the matching Alloy toleration before relying on the `AlloyDaemonSetUnavailable` alert for full-node log coverage. - kube-prometheus-stack Prometheus and Alertmanager run with 2 replicas. Prometheus clears the replica external label so HA replicas do not create replica-labeled duplicate Alertmanager notifications. Prometheus, Alertmanager, and Grafana use Longhorn PVCs so metric data, silences, and UI state survive pod rescheduling. - Alloy keeps namespace-based collection for `code-place-dev`, `code-place-prod`, and `monitoring`; do not depend on `app.kubernetes.io/name` for CodePlace app logs because the application manifests primarily use `app`. - Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml index 25c9631a..561e6ab4 100644 --- a/kubernetes/monitoring/logs/alloy-values.yaml +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -3,6 +3,13 @@ controller: type: daemonset + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule alloy: mounts: From 6ca85d72401fcdd54fd2b47a2e20e57ced7e454c Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Fri, 19 Jun 2026 23:58:56 +0900 Subject: [PATCH 115/147] =?UTF-8?q?Tempo=20PVC=20=EB=A1=A4=EC=95=84?= =?UTF-8?q?=EC=9B=83=20=EC=A0=84=EB=9E=B5=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/developer/infra/observability-runbook/_index.md | 1 + kubernetes/monitoring/tempo.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 99963556..d29f4226 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -557,6 +557,7 @@ sum(rate(otelcol_exporter_send_failed_spans_total{namespace="monitoring"}[5m])) - export failure가 증가하면 Tempo readiness, `tempo.monitoring.svc.cluster.local:4317` 경로, Collector exporter queue/retry 로그를 봅니다. - Tempo가 down이면 Collector가 span export 실패를 기록합니다. - Tempo PVC가 85%를 넘으면 retention 단축, noisy trace sampling 조정, PVC 증설 중 하나를 먼저 결정합니다. +- Tempo는 단일 replica와 Longhorn RWO PVC 조합이므로 Deployment strategy가 `Recreate`여야 합니다. RollingUpdate로 바뀌면 업그레이드 중 새 Pod와 기존 Pod가 `tempo-data` PVC를 두고 경합해 rollout이 멈출 수 있습니다. - 채점 병목은 Tempo에서 `judge_task`, `submission.judge`, `judge_server.select`, `judge_server.request`, `submission.result_save` span duration을 순서대로 봅니다. 선택 span이 길면 DB row lock/available judge 부족, request span이 길면 judge-server 실행/네트워크, result save span이 길면 DB lock/write 병목을 의심합니다. - prod는 기본 `OTEL_ENABLED=0`입니다. prod trace가 없다고 해서 장애는 아니며, dev에서 Collector/Tempo를 먼저 확인한 뒤 prod 승격 여부를 결정합니다. diff --git a/kubernetes/monitoring/tempo.yaml b/kubernetes/monitoring/tempo.yaml index 050328af..a8b0c3c6 100644 --- a/kubernetes/monitoring/tempo.yaml +++ b/kubernetes/monitoring/tempo.yaml @@ -51,6 +51,8 @@ metadata: app: tempo spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: app: tempo From 21b6a789fd974b11beaefda8219d8e789c724740 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 00:47:46 +0900 Subject: [PATCH 116/147] =?UTF-8?q?=EC=A0=9C=EC=B6=9C=20=ED=85=8C=EC=9D=B4?= =?UTF-8?q?=EB=B8=94=20=EA=B4=80=EC=B8=A1=20=EC=A1=B0=ED=9A=8C=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/judge/dispatcher.py | 2 + backend/utils/observability_metrics.py | 165 ++++++++++-------- backend/utils/tests.py | 73 +++----- .../observability-improvement-plan/_index.md | 8 +- .../infra/observability-runbook/_index.md | 6 +- .../grafana-dashboard-codeplace.yaml | 16 +- kubernetes/monitoring/prometheus-rules.yaml | 18 +- 7 files changed, 138 insertions(+), 150 deletions(-) diff --git a/backend/judge/dispatcher.py b/backend/judge/dispatcher.py index e3b63fde..9b995c88 100644 --- a/backend/judge/dispatcher.py +++ b/backend/judge/dispatcher.py @@ -20,6 +20,7 @@ from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.constants import CacheKey, ProblemScore, ProblemField, Tier +from utils.observability_metrics import record_judge_duration from utils.observability_tracing import get_tracer logger = logging.getLogger(__name__) @@ -188,6 +189,7 @@ def _judge(self, span): self.submission.judge_start_time = timezone.now() resp = self._request(urljoin(server.service_url, "/judge"), data=data) self.submission.judge_end_time = timezone.now() + record_judge_duration((self.submission.judge_end_time - self.submission.judge_start_time).total_seconds()) if not resp: Submission.objects.filter(id=self.submission.id).update(result=JudgeStatus.SYSTEM_ERROR) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index a3dd51f7..c6b53929 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -2,12 +2,10 @@ from datetime import timedelta from django.db import connection -from django.db.models import Count, Min from django.utils import timezone from prometheus_client import Counter, Histogram, REGISTRY from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, HistogramMetricFamily -from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.celery_observability import ( CELERY_TASK_RUNTIME_BUCKET_KEY, @@ -23,19 +21,10 @@ logger = logging.getLogger(__name__) JUDGE_DURATION_BUCKETS = (1, 3, 5, 10, 30, 60, 120, 300, 600, float("inf")) -JUDGE_STATUS_LABELS = { - JudgeStatus.COMPILE_ERROR: "compile_error", - JudgeStatus.WRONG_ANSWER: "wrong_answer", - JudgeStatus.ACCEPTED: "accepted", - JudgeStatus.CPU_TIME_LIMIT_EXCEEDED: "cpu_time_limit_exceeded", - JudgeStatus.REAL_TIME_LIMIT_EXCEEDED: "real_time_limit_exceeded", - JudgeStatus.MEMORY_LIMIT_EXCEEDED: "memory_limit_exceeded", - JudgeStatus.RUNTIME_ERROR: "runtime_error", - JudgeStatus.SYSTEM_ERROR: "system_error", - JudgeStatus.PENDING: "pending", - JudgeStatus.JUDGING: "judging", - JudgeStatus.PARTIALLY_ACCEPTED: "partially_accepted", -} +CELERY_BROKER_QUEUE_KEY = "celery" +JUDGE_DURATION_BUCKET_KEY = "codeplace:judge_duration_seconds_bucket" +JUDGE_DURATION_COUNT_KEY = "codeplace:judge_duration_seconds_count" +JUDGE_DURATION_SUM_KEY = "codeplace:judge_duration_seconds_sum" HTTP_REQUESTS_TOTAL = Counter( "codeplace_http_requests_total", @@ -84,20 +73,14 @@ class CodePlaceCollector: def describe(self): - yield GaugeMetricFamily( - "codeplace_submission_status_count", - "Current number of submissions grouped by judge status.", - labels=["status"], - ) - yield GaugeMetricFamily( - "codeplace_submission_oldest_age_seconds", - "Age of the oldest submission in each in-flight judge status.", - labels=["status"], - ) yield GaugeMetricFamily( "codeplace_waiting_queue_length", "Number of submissions waiting because no judge-server was available.", ) + yield GaugeMetricFamily( + "codeplace_celery_broker_queue_length", + "Number of Celery tasks waiting in the Redis broker default queue.", + ) yield GaugeMetricFamily( "codeplace_judge_server_available", "Whether each judge-server is enabled and has a fresh heartbeat.", @@ -170,31 +153,22 @@ def describe(self): "Seconds since each Celery task last completed successfully.", labels=["task_name"], ) + yield GaugeMetricFamily( + "codeplace_observability_collector_success", + "Whether each CodePlace custom metrics collector completed successfully.", + labels=["collector"], + ) def collect(self): - yield from self._submission_status_count() - yield self._submission_oldest_age_seconds() + self._collector_success = {} yield self._waiting_queue_length() + yield self._celery_broker_queue_length() yield from self._judge_server_metrics() yield self._judge_duration_histogram() yield from self._celery_task_metrics() yield from self._postgres_metrics() yield from self._redis_health_metrics() - - def _submission_status_count(self): - metric = GaugeMetricFamily( - "codeplace_submission_status_count", - "Current number of submissions grouped by judge status.", - labels=["status"], - ) - try: - counts = dict(Submission.objects.values_list("result").order_by("result").annotate(count=Count("id"))) - except Exception as e: - logger.warning("Failed to collect submission status counts: %s", e) - counts = {} - for status_value, status_label in JUDGE_STATUS_LABELS.items(): - metric.add_metric([status_label], counts.get(status_value, 0)) - yield metric + yield self._collector_success_metric() def _waiting_queue_length(self): metric = GaugeMetricFamily( @@ -206,34 +180,24 @@ def _waiting_queue_length(self): except Exception as e: logger.warning("Failed to collect waiting queue length: %s", e) metric.add_metric([], 0) + self._mark_collector_success("waiting_queue", False) + else: + self._mark_collector_success("waiting_queue", True) return metric - def _submission_oldest_age_seconds(self): + def _celery_broker_queue_length(self): metric = GaugeMetricFamily( - "codeplace_submission_oldest_age_seconds", - "Age of the oldest submission in each in-flight judge status.", - labels=["status"], + "codeplace_celery_broker_queue_length", + "Number of Celery tasks waiting in the Redis broker default queue.", ) - now = timezone.now() - status_map = { - JudgeStatus.PENDING: "pending", - JudgeStatus.JUDGING: "judging", - } try: - oldest_by_status = { - row["result"]: row["oldest_create_time"] - for row in Submission.objects.filter( - result__in=status_map.keys(), - ).values("result").annotate(oldest_create_time=Min("create_time")) - } + metric.add_metric([], cache.llen(CELERY_BROKER_QUEUE_KEY) or 0) except Exception as e: - logger.warning("Failed to collect oldest submission age metrics: %s", e) - oldest_by_status = {} - - for status_value, status_label in status_map.items(): - created_at = oldest_by_status.get(status_value) - age_seconds = max((now - created_at).total_seconds(), 0) if created_at else 0 - metric.add_metric([status_label], age_seconds) + logger.warning("Failed to collect Celery broker queue length: %s", e) + metric.add_metric([], 0) + self._mark_collector_success("celery_broker_queue", False) + else: + self._mark_collector_success("celery_broker_queue", True) return metric def _judge_server_metrics(self): @@ -260,6 +224,9 @@ def _judge_server_metrics(self): except Exception as e: logger.warning("Failed to collect judge-server metrics: %s", e) servers = [] + self._mark_collector_success("judge_server", False) + else: + self._mark_collector_success("judge_server", True) for server in servers: if server.is_disabled: continue @@ -276,26 +243,24 @@ def _judge_server_metrics(self): def _judge_duration_histogram(self): metric = HistogramMetricFamily( "codeplace_submission_judge_duration_seconds", - "Judge duration for submissions completed in the last 10 minutes.", + "Judge duration recorded by celery workers.", ) - cutoff = timezone.now() - timedelta(minutes=10) - durations = [] try: - submissions = Submission.objects.filter( - judge_end_time__gte=cutoff, - judge_start_time__isnull=False, - judge_end_time__isnull=False, - ).only("judge_start_time", "judge_end_time") - for submission in submissions: - durations.append(max((submission.judge_end_time - submission.judge_start_time).total_seconds(), 0)) + duration_buckets = self._redis_hash(JUDGE_DURATION_BUCKET_KEY) + duration_sum = self._float_or_zero(cache.get(JUDGE_DURATION_SUM_KEY, 0)) except Exception as e: logger.warning("Failed to collect judge duration metrics: %s", e) + duration_buckets = {} + duration_sum = 0 + self._mark_collector_success("judge_duration", False) + else: + self._mark_collector_success("judge_duration", True) cumulative_buckets = [] for bucket in JUDGE_DURATION_BUCKETS: label = "+Inf" if bucket == float("inf") else str(bucket) - cumulative_buckets.append((label, sum(1 for duration in durations if duration <= bucket))) - metric.add_metric([], cumulative_buckets, sum(durations)) + cumulative_buckets.append((label, self._float_or_zero(duration_buckets.get(label, 0)))) + metric.add_metric([], cumulative_buckets, duration_sum) return metric def _celery_task_metrics(self): @@ -333,12 +298,15 @@ def _celery_task_metrics(self): last_seen_at = self._redis_hash(CELERY_TASK_LAST_SEEN_AT_KEY) except Exception as e: logger.warning("Failed to collect Celery task metrics: %s", e) + self._mark_collector_success("celery_task", False) yield total_metric yield runtime_metric yield last_runtime_metric yield last_seen_age_metric yield last_success_age_metric return + else: + self._mark_collector_success("celery_task", True) for field, value in task_totals.items(): task_name, status = self._split_field(field, 2) @@ -421,6 +389,9 @@ def _postgres_metrics(self): except Exception as e: logger.warning("Failed to collect PostgreSQL health metrics: %s", e) connections = max_connections = long_transactions = lock_waits = 0 + self._mark_collector_success("postgres", False) + else: + self._mark_collector_success("postgres", True) connections_metric.add_metric([], float(connections or 0)) max_connections_metric.add_metric([], float(max_connections or 0)) @@ -455,9 +426,14 @@ def _redis_health_metrics(self): max_clients = float(raw_max_clients.get("maxclients") or 0) except Exception as e: logger.warning("Failed to collect Redis maxclients metric: %s", e) + self._mark_collector_success("redis", False) except Exception as e: logger.warning("Failed to collect Redis health metrics: %s", e) connected_clients = max_clients = rejected_connections = 0 + self._mark_collector_success("redis", False) + else: + if not getattr(self, "_collector_success", {}).get("redis", False): + self._mark_collector_success("redis", True) connected_clients_metric.add_metric([], connected_clients) max_clients_metric.add_metric([], max_clients) @@ -466,6 +442,30 @@ def _redis_health_metrics(self): yield max_clients_metric yield rejected_connections_metric + def _collector_success_metric(self): + metric = GaugeMetricFamily( + "codeplace_observability_collector_success", + "Whether each CodePlace custom metrics collector completed successfully.", + labels=["collector"], + ) + collectors = ( + "waiting_queue", + "celery_broker_queue", + "judge_server", + "judge_duration", + "celery_task", + "postgres", + "redis", + ) + for collector in collectors: + metric.add_metric([collector], 1 if self._collector_success.get(collector, False) else 0) + return metric + + def _mark_collector_success(self, collector, success): + if not hasattr(self, "_collector_success"): + self._collector_success = {} + self._collector_success[collector] = bool(success) + @staticmethod def _redis_hash(key): values = cache.hgetall(key) or {} @@ -512,3 +512,16 @@ def register_codeplace_metrics(): except ValueError: logger.debug("CodePlace metrics collector is already registered") register_codeplace_metrics._registered = True + + +def record_judge_duration(duration_seconds): + try: + duration = max(float(duration_seconds), 0) + for bucket in JUDGE_DURATION_BUCKETS: + if duration <= bucket: + label = "+Inf" if bucket == float("inf") else str(bucket) + cache.hincrby(JUDGE_DURATION_BUCKET_KEY, label, 1) + cache.redis_incr(JUDGE_DURATION_COUNT_KEY, 1) + cache.incrbyfloat(JUDGE_DURATION_SUM_KEY, duration) + except Exception: + logger.exception("Failed to record judge duration metric") diff --git a/backend/utils/tests.py b/backend/utils/tests.py index c0dffd0a..581e5422 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -64,48 +64,29 @@ def test_judge_server_metrics_mark_stale_enabled_server_unavailable(self): self.assertEqual(available.value, 0) self.assertGreaterEqual(heartbeat_age.value, 599) - def test_submission_oldest_age_metrics_are_collected_for_inflight_statuses(self): - now = timezone.now() - rows = [ - {"result": 6, "oldest_create_time": now - timedelta(minutes=7)}, - {"result": 7, "oldest_create_time": now - timedelta(minutes=3)}, - ] - queryset = MagicMock() - queryset.values.return_value.annotate.return_value = rows - - with patch("utils.observability_metrics.Submission.objects.filter", return_value=queryset): - samples = self._samples_by_metric([CodePlaceCollector()._submission_oldest_age_seconds()]) - - by_status = { - sample.labels["status"]: sample.value - for sample in samples["codeplace_submission_oldest_age_seconds"] - } - self.assertGreaterEqual(by_status["pending"], 419) - self.assertGreaterEqual(by_status["judging"], 179) - - def test_judge_duration_histogram_uses_recent_completion_time(self): - now = timezone.now() - submission = MagicMock( - judge_start_time=now - timedelta(minutes=20), - judge_end_time=now - timedelta(minutes=1), - ) - queryset = MagicMock() - queryset.only.return_value = [submission] - - with patch("utils.observability_metrics.timezone.now", return_value=now), \ - patch("utils.observability_metrics.Submission.objects.filter", return_value=queryset) as filter_mock: + def test_celery_broker_queue_length_is_collected_from_redis(self): + with patch("utils.observability_metrics.cache.llen", return_value=7) as llen: + samples = self._samples_by_metric([CodePlaceCollector()._celery_broker_queue_length()]) + + llen.assert_called_once_with("celery") + self.assertEqual(samples["codeplace_celery_broker_queue_length"][0].value, 7) + + def test_judge_duration_histogram_is_collected_from_redis(self): + with patch("utils.observability_metrics.cache.hgetall", return_value={ + b"1": b"0", + b"3": b"1", + b"+Inf": b"2", + }), patch("utils.observability_metrics.cache.get", return_value="4.5"): samples = self._samples_by_metric([CodePlaceCollector()._judge_duration_histogram()]) - filter_kwargs = filter_mock.call_args.kwargs - self.assertEqual(filter_kwargs["judge_end_time__gte"], now - timedelta(minutes=10)) - self.assertNotIn("judge_start_time__gte", filter_kwargs) bucket_samples = [ sample for sample in samples["codeplace_submission_judge_duration_seconds"] if sample.name.endswith("_bucket") ] - self.assertTrue(any(sample.labels["le"] == "+Inf" and sample.value == 1 for sample in bucket_samples)) + self.assertTrue(any(sample.labels["le"] == "3" and sample.value == 1 for sample in bucket_samples)) + self.assertTrue(any(sample.labels["le"] == "+Inf" and sample.value == 2 for sample in bucket_samples)) self.assertTrue(any( - sample.name.endswith("_sum") and sample.value == 1140 + sample.name.endswith("_sum") and sample.value == 4.5 for sample in samples["codeplace_submission_judge_duration_seconds"] )) @@ -298,23 +279,16 @@ def test_task_prerun_and_postrun_restore_request_id_context(self): class CodePlaceMetricsEndpointTest(SimpleTestCase): def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): - submission_metric = GaugeMetricFamily( - "codeplace_submission_status_count", - "Current number of submissions grouped by judge status.", - labels=["status"], - ) - submission_metric.add_metric(["accepted"], 0) waiting_queue_metric = GaugeMetricFamily( "codeplace_waiting_queue_length", "Number of submissions waiting because no judge-server was available.", ) waiting_queue_metric.add_metric([], 0) - oldest_age_metric = GaugeMetricFamily( - "codeplace_submission_oldest_age_seconds", - "Age of the oldest submission in each in-flight judge status.", - labels=["status"], + broker_queue_metric = GaugeMetricFamily( + "codeplace_celery_broker_queue_length", + "Number of Celery tasks waiting in the Redis broker default queue.", ) - oldest_age_metric.add_metric(["pending"], 0) + broker_queue_metric.add_metric([], 0) judge_metric = GaugeMetricFamily( "codeplace_judge_server_available", "Whether each judge-server is enabled and has a fresh heartbeat.", @@ -327,9 +301,8 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): ) judge_duration_metric.add_metric([], [("+Inf", 0)], 0) - with patch.object(CodePlaceCollector, "_submission_status_count", return_value=[submission_metric]), \ - patch.object(CodePlaceCollector, "_submission_oldest_age_seconds", return_value=oldest_age_metric), \ - patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ + with patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ + patch.object(CodePlaceCollector, "_celery_broker_queue_length", return_value=broker_queue_metric), \ patch.object(CodePlaceCollector, "_judge_server_metrics", return_value=[judge_metric]), \ patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric), \ patch.object(CodePlaceCollector, "_celery_task_metrics", return_value=[]), \ @@ -340,8 +313,8 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): self.assertEqual(response.status_code, 200) body = response.content.decode() self.assertIn("django_http_requests_total_by_method_total", body) - self.assertIn("codeplace_submission_status_count", body) self.assertIn("codeplace_waiting_queue_length", body) + self.assertIn("codeplace_celery_broker_queue_length", body) self.assertIn("codeplace_judge_server_available", body) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 01694716..f1ded9e5 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -39,12 +39,12 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_http_requests_total{method,endpoint,status_code}` - `codeplace_http_request_duration_seconds{method,endpoint}` -- `codeplace_submission_status_count{status}` -- `codeplace_submission_oldest_age_seconds{status}` - `codeplace_submission_judge_duration_seconds` - `codeplace_submission_create_outcome_total{status,scope}` - `codeplace_judge_task_outcome_total{status,scope}` - `codeplace_waiting_queue_length` +- `codeplace_celery_broker_queue_length` +- `codeplace_observability_collector_success{collector}` - `codeplace_judge_server_available{hostname}` - `codeplace_judge_server_last_heartbeat_age_seconds{hostname}` - `codeplace_judge_server_task_number{hostname}` @@ -277,10 +277,10 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - `AlertmanagerConfigReloadFailed`: Alertmanager generated config reload 실패 또는 reload metric 누락 2분 지속. - `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. +- `CeleryBrokerQueueBacklog`: Redis broker default queue 20 초과 5분 지속. +- `CodePlaceCollectorFailed`: backend custom metrics collector 실패 5분 지속. - `SubmissionCreateSystemFailures`: 제출 생성 API에서 DB 또는 judge enqueue 실패 발생 2분 지속. - `JudgeTaskFailures`: `judge_task`에서 submission/user lookup 또는 dispatch 실패 발생 2분 지속. -- `PendingSubmissionsStuck`: oldest pending submission age 300초 초과 3분 지속. -- `JudgingSubmissionsStuck`: oldest judging submission age 900초 초과 5분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. - `CeleryBeatDown`: beat Deployment available replica 0개 또는 availability metric 누락 2분 지속. - `CeleryTaskFailures`: task failure 발생 2분 지속. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index d29f4226..2ec870f6 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -653,14 +653,14 @@ Grafana Explore: - `error_type="unhandled_rejection"`이 많으면 API response handling, auth/session 만료 처리, network rejection을 봅니다. - Sentry가 켜진 환경이면 같은 `release`, `environment`, `request_id`로 Sentry event와 Loki `frontend.error` 로그를 대조합니다. -### JudgeWaitingQueueBacklog / PendingSubmissionsStuck / JudgingSubmissionsStuck +### JudgeWaitingQueueBacklog / CeleryBrokerQueueBacklog / CodePlaceCollectorFailed 확인: ```promql codeplace_waiting_queue_length{namespace=""} -codeplace_submission_oldest_age_seconds{namespace=""} -codeplace_submission_status_count{namespace="", status=~"pending|judging"} +codeplace_celery_broker_queue_length{namespace=""} +codeplace_observability_collector_success{namespace=""} sum by (namespace) (codeplace_judge_server_available{namespace=""}) ``` diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index c4caeeb6..79a129a6 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -178,34 +178,34 @@ data: }, { "type": "bargauge", - "title": "Submission Status", + "title": "Judge Queue Length", "gridPos": {"x": 16, "y": 4, "w": 4, "h": 8}, "fieldConfig": {"defaults": {"unit": "short"}}, "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace_submission_status_count{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{status}}"} + {"expr": "codeplace_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} waiting_queue"}, + {"expr": "codeplace_celery_broker_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} celery"} ] }, { "type": "bargauge", - "title": "Oldest In-flight Submission", + "title": "Custom Collector Success", "gridPos": {"x": 20, "y": 4, "w": 4, "h": 8}, "fieldConfig": { "defaults": { - "unit": "s", + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ - {"color": "green", "value": null}, - {"color": "orange", "value": 300}, - {"color": "red", "value": 900} + {"color": "red", "value": null}, + {"color": "green", "value": 1} ] } } }, "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace_submission_oldest_age_seconds{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{status}}"} + {"expr": "codeplace_observability_collector_success{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{collector}}"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 4ecac41a..b5b3301d 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -506,24 +506,24 @@ spec: annotations: summary: "Judge waiting queue is backing up" description: "More than 5 submissions are waiting because no judge-server was available." - - alert: PendingSubmissionsStuck - expr: codeplace_submission_oldest_age_seconds{namespace=~"code-place-(dev|prod)", status="pending"} > 300 - for: 3m + - alert: CeleryBrokerQueueBacklog + expr: codeplace_celery_broker_queue_length{namespace=~"code-place-(dev|prod)"} > 20 + for: 5m labels: severity: warning priority: P1 annotations: - summary: "Pending submissions are stuck" - description: "The oldest pending submission has been pending for more than 5 minutes. Check Celery workers, Redis queue state, judge dispatch, and backend logs." - - alert: JudgingSubmissionsStuck - expr: codeplace_submission_oldest_age_seconds{namespace=~"code-place-(dev|prod)", status="judging"} > 900 + summary: "Celery broker queue is backing up" + description: "More than 20 Celery tasks have been waiting in the Redis broker default queue for 5 minutes. Check celery-worker capacity, Redis broker health, and task errors." + - alert: CodePlaceCollectorFailed + expr: codeplace_observability_collector_success{namespace=~"code-place-(dev|prod)"} == 0 for: 5m labels: severity: warning priority: P1 annotations: - summary: "Judging submissions are stuck" - description: "The oldest judging submission has been judging for more than 15 minutes. Check judge-server execution, callback/result save path, and worker logs." + summary: "A CodePlace custom metrics collector is failing" + description: "A backend /metrics collector block has failed for 5 minutes. Metric values may be stale or zero even though the scrape target is up." - alert: SubmissionCreateSystemFailures expr: sum by (namespace, scope, status) (increase(codeplace_submission_create_outcome_total{namespace=~"code-place-(dev|prod)", status=~"db_error|enqueue_error"}[5m])) > 0 for: 2m From 1526adb9091c03a9e55be165e25ee74da6c31ccb Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 00:51:22 +0900 Subject: [PATCH 117/147] =?UTF-8?q?=EB=B6=88=ED=95=84=EC=9A=94=ED=95=9C=20?= =?UTF-8?q?=EC=A0=9C=EC=B6=9C=20=EA=B4=80=EC=B8=A1=20=EC=9D=B8=EB=8D=B1?= =?UTF-8?q?=EC=8A=A4=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../0005_submission_judge_end_time_index.py | 30 ----------------- ...006_submission_result_create_time_index.py | 32 ------------------- backend/submission/models.py | 5 +-- 3 files changed, 1 insertion(+), 66 deletions(-) delete mode 100644 backend/submission/migrations/0005_submission_judge_end_time_index.py delete mode 100644 backend/submission/migrations/0006_submission_result_create_time_index.py diff --git a/backend/submission/migrations/0005_submission_judge_end_time_index.py b/backend/submission/migrations/0005_submission_judge_end_time_index.py deleted file mode 100644 index c7e59a91..00000000 --- a/backend/submission/migrations/0005_submission_judge_end_time_index.py +++ /dev/null @@ -1,30 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - atomic = False - - dependencies = [ - ("submission", "0004_auto_20250904_1525"), - ] - - operations = [ - migrations.SeparateDatabaseAndState( - database_operations=[ - migrations.RunSQL( - sql=( - 'CREATE INDEX CONCURRENTLY IF NOT EXISTS ' - '"submission_judge_end_time_c28a7535" ON "submission" ("judge_end_time")' - ), - reverse_sql='DROP INDEX CONCURRENTLY IF EXISTS "submission_judge_end_time_c28a7535"', - ), - ], - state_operations=[ - migrations.AlterField( - model_name="submission", - name="judge_end_time", - field=models.DateTimeField(db_index=True, null=True), - ), - ], - ), - ] diff --git a/backend/submission/migrations/0006_submission_result_create_time_index.py b/backend/submission/migrations/0006_submission_result_create_time_index.py deleted file mode 100644 index cdf17043..00000000 --- a/backend/submission/migrations/0006_submission_result_create_time_index.py +++ /dev/null @@ -1,32 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - atomic = False - - dependencies = [ - ("submission", "0005_submission_judge_end_time_index"), - ] - - operations = [ - migrations.SeparateDatabaseAndState( - database_operations=[ - migrations.RunSQL( - sql=( - 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "sub_result_ctime_idx" ' - 'ON "submission" ("result", "create_time")' - ), - reverse_sql='DROP INDEX CONCURRENTLY IF EXISTS "sub_result_ctime_idx"', - ), - ], - state_operations=[ - migrations.AddIndex( - model_name="submission", - index=models.Index( - fields=["result", "create_time"], - name="sub_result_ctime_idx", - ), - ), - ], - ), - ] diff --git a/backend/submission/models.py b/backend/submission/models.py index 0020a0ce..381510e5 100644 --- a/backend/submission/models.py +++ b/backend/submission/models.py @@ -27,7 +27,7 @@ class Submission(models.Model): problem = models.ForeignKey(Problem, on_delete=models.CASCADE) create_time = models.DateTimeField(auto_now_add=True) judge_start_time = models.DateTimeField(null=True) - judge_end_time = models.DateTimeField(null=True, db_index=True) + judge_end_time = models.DateTimeField(null=True) user_id = models.IntegerField(db_index=True) username = models.TextField() code = models.TextField() @@ -58,9 +58,6 @@ def check_user_permission(self, user, check_share=True): class Meta: db_table = "submission" ordering = ("-create_time",) - indexes = [ - models.Index(fields=["result", "create_time"], name="sub_result_ctime_idx"), - ] def __str__(self): return self.id From f3930f5cca539f9cdb781bdffb68622a75f07d51 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 00:55:50 +0900 Subject: [PATCH 118/147] =?UTF-8?q?=EB=B0=B1=EC=97=94=EB=93=9C=20=EB=A9=94?= =?UTF-8?q?=ED=8A=B8=EB=A6=AD=EC=9D=98=20DB=20=EC=A7=81=EC=A0=91=20?= =?UTF-8?q?=EC=A1=B0=ED=9A=8C=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_metrics.py | 71 ------------------- backend/utils/tests.py | 28 -------- .../observability-improvement-plan/_index.md | 11 +-- .../infra/observability-runbook/_index.md | 10 +-- .../grafana-dashboard-codeplace.yaml | 10 ++- kubernetes/monitoring/prometheus-rules.yaml | 27 ------- 6 files changed, 8 insertions(+), 149 deletions(-) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index c6b53929..6976fa9c 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -1,7 +1,6 @@ import logging from datetime import timedelta -from django.db import connection from django.utils import timezone from prometheus_client import Counter, Histogram, REGISTRY from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, HistogramMetricFamily @@ -115,22 +114,6 @@ def describe(self): "Latest observed Celery task runtime grouped by task name and status.", labels=["task_name", "status"], ) - yield GaugeMetricFamily( - "codeplace_postgres_connections", - "Current PostgreSQL connections for the active database.", - ) - yield GaugeMetricFamily( - "codeplace_postgres_max_connections", - "Configured PostgreSQL max_connections.", - ) - yield GaugeMetricFamily( - "codeplace_postgres_long_transactions", - "Current PostgreSQL transactions older than five minutes.", - ) - yield GaugeMetricFamily( - "codeplace_postgres_lock_waits", - "Current PostgreSQL locks waiting to be granted.", - ) yield GaugeMetricFamily( "codeplace_redis_connected_clients", "Current Redis connected clients from INFO clients.", @@ -166,7 +149,6 @@ def collect(self): yield from self._judge_server_metrics() yield self._judge_duration_histogram() yield from self._celery_task_metrics() - yield from self._postgres_metrics() yield from self._redis_health_metrics() yield self._collector_success_metric() @@ -350,58 +332,6 @@ def _celery_task_metrics(self): yield last_seen_age_metric yield last_success_age_metric - def _postgres_metrics(self): - connections_metric = GaugeMetricFamily( - "codeplace_postgres_connections", - "Current PostgreSQL connections for the active database.", - ) - max_connections_metric = GaugeMetricFamily( - "codeplace_postgres_max_connections", - "Configured PostgreSQL max_connections.", - ) - long_transactions_metric = GaugeMetricFamily( - "codeplace_postgres_long_transactions", - "Current PostgreSQL transactions older than five minutes.", - ) - lock_waits_metric = GaugeMetricFamily( - "codeplace_postgres_lock_waits", - "Current PostgreSQL locks waiting to be granted.", - ) - try: - with connection.cursor() as cursor: - cursor.execute("SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()") - connections = cursor.fetchone()[0] - cursor.execute("SELECT setting::float FROM pg_settings WHERE name = 'max_connections'") - max_connections = cursor.fetchone()[0] - cursor.execute( - """ - SELECT count(*) - FROM pg_stat_activity - WHERE datname = current_database() - AND xact_start IS NOT NULL - AND now() - xact_start > interval '5 minutes' - AND state <> 'idle' - """ - ) - long_transactions = cursor.fetchone()[0] - cursor.execute("SELECT count(*) FROM pg_locks WHERE NOT granted") - lock_waits = cursor.fetchone()[0] - except Exception as e: - logger.warning("Failed to collect PostgreSQL health metrics: %s", e) - connections = max_connections = long_transactions = lock_waits = 0 - self._mark_collector_success("postgres", False) - else: - self._mark_collector_success("postgres", True) - - connections_metric.add_metric([], float(connections or 0)) - max_connections_metric.add_metric([], float(max_connections or 0)) - long_transactions_metric.add_metric([], float(long_transactions or 0)) - lock_waits_metric.add_metric([], float(lock_waits or 0)) - yield connections_metric - yield max_connections_metric - yield long_transactions_metric - yield lock_waits_metric - def _redis_health_metrics(self): connected_clients_metric = GaugeMetricFamily( "codeplace_redis_connected_clients", @@ -454,7 +384,6 @@ def _collector_success_metric(self): "judge_server", "judge_duration", "celery_task", - "postgres", "redis", ) for collector in collectors: diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 581e5422..cb3f7fde 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -174,33 +174,6 @@ def test_celery_task_metrics_tolerate_malformed_redis_values(self): self.assertEqual(last_runtime_sample.value, 0) self.assertEqual(samples["codeplace_celery_task_last_seen_age_seconds"], []) - def test_postgres_health_metrics_are_collected(self): - class FakeCursor: - def __init__(self): - self.values = iter([(12,), (100,), (2,), (1,)]) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - return False - - def execute(self, _query): - return None - - def fetchone(self): - return next(self.values) - - fake_connection = MagicMock() - fake_connection.cursor.return_value = FakeCursor() - with patch("utils.observability_metrics.connection", fake_connection): - samples = self._samples_by_metric(list(CodePlaceCollector()._postgres_metrics())) - - self.assertEqual(samples["codeplace_postgres_connections"][0].value, 12) - self.assertEqual(samples["codeplace_postgres_max_connections"][0].value, 100) - self.assertEqual(samples["codeplace_postgres_long_transactions"][0].value, 2) - self.assertEqual(samples["codeplace_postgres_lock_waits"][0].value, 1) - def test_redis_health_metrics_are_collected(self): redis_client = MagicMock() redis_client.info.return_value = { @@ -306,7 +279,6 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): patch.object(CodePlaceCollector, "_judge_server_metrics", return_value=[judge_metric]), \ patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric), \ patch.object(CodePlaceCollector, "_celery_task_metrics", return_value=[]), \ - patch.object(CodePlaceCollector, "_postgres_metrics", return_value=[]), \ patch.object(CodePlaceCollector, "_redis_health_metrics", return_value=[]): response = self.client.get("/metrics") diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index f1ded9e5..37f8d507 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -56,10 +56,6 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_ai_hint_api_outcome_total{status,scope}` - `codeplace_ai_hint_requests_total{status}` - `codeplace_ai_hint_duration_seconds{status}` -- `codeplace_postgres_connections` -- `codeplace_postgres_max_connections` -- `codeplace_postgres_long_transactions` -- `codeplace_postgres_lock_waits` - `codeplace_redis_connected_clients` - `codeplace_redis_max_clients` - `codeplace_redis_rejected_connections_total` @@ -304,9 +300,6 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - `PostgresCollectorError`: CNPG PostgreSQL metrics collection error 5분 지속. - `PostgresHADegraded`: PostgreSQL instance가 3개 node에 분산되지 않음 10분 지속. - `RedisMemoryHigh`: Redis memory 사용률 85% 초과 10분 지속. -- `PostgresConnectionUsageHigh`: backend 기준 PostgreSQL connection/max_connections 80% 초과 10분 지속. -- `PostgresLongTransactions`: 5분 초과 transaction이 10분 지속. -- `PostgresLockWaits`: lock wait가 5분 지속. - `RedisConnectedClientsHigh`: backend 기준 Redis connected_clients/maxclients 80% 초과 10분 지속. - `RedisRejectedConnections`: Redis rejected connection 증가 2분 지속. - `PVCAlmostFull`: PVC 사용률 85% 초과 10분 지속. @@ -358,7 +351,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - `kubectl apply -k kubernetes/monitoring` 6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`, `codeplace-grafana-http`가 healthy인지 확인합니다. 7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev/Grafana availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission status, submission/judge outcome, oldest in-flight submission age, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, Service ready endpoint, PostgreSQL/Redis readiness/connection/lock/client panel을 확인합니다. +8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission/judge outcome, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, Service ready endpoint, PostgreSQL CNPG collector/HA 상태, Redis readiness/memory/client panel을 확인합니다. 9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. 10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. 11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. @@ -382,7 +375,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - Metrics backend는 kube-prometheus-stack의 Prometheus입니다. Mimir는 현재 온프렘 단기 운영 기준선에 포함하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - Kubernetes event는 Kubernetes Event Exporter가 Warning event만 stdout JSON으로 내보내고, Alloy가 일반 Pod log와 같은 경로로 Loki에 적재합니다. -- PostgreSQL은 CNPG instance exporter와 PodMonitor로 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor로 `redis_*` 지표를 수집합니다. +- PostgreSQL은 backend `/metrics`에서 직접 조회하지 않고 CNPG instance exporter와 PodMonitor의 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor의 `redis_*` 지표를 기본으로 보고, backend custom collector는 queue length와 client/rejection 같은 O(1) Redis 상태만 수집합니다. - Sentry backend SDK는 기본 PII 자동 전송을 비활성화하고 전송 직전 `authorization`, `cookie`, `password`, `token`, `secret`, 제출 source code 계열 필드를 redaction합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. - Frontend는 axios 요청마다 `X-Request-ID`를 전파하고 마지막 request ID를 `window.__CODEPLACE_LAST_REQUEST_ID__`와 Sentry request context 및 `/api/client_error` report에 저장합니다. - Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. DSN이 없으면 Sentry는 비활성화되지만 `/api/client_error` 기반 metric/log는 계속 동작합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 2ec870f6..971bfbc0 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -890,7 +890,7 @@ topk(10, container_memory_working_set_bytes{namespace="", container!= CPU/memory가 지속적으로 높으면 replica 조정, limit 조정, 최근 배포 변경을 확인합니다. -### PostgresCollectorError / PostgresHADegraded / PostgresReplicaUnavailable / RedisReplicaUnavailable / RedisMemoryHigh / PostgresConnectionUsageHigh / PostgresLongTransactions / PostgresLockWaits / RedisConnectedClientsHigh / RedisRejectedConnections +### PostgresCollectorError / PostgresHADegraded / PostgresReplicaUnavailable / RedisReplicaUnavailable / RedisMemoryHigh / RedisConnectedClientsHigh / RedisRejectedConnections 확인: @@ -901,10 +901,6 @@ sum by (namespace) (kube_pod_status_ready{namespace="", condition="tr sum by (namespace) (kube_pod_status_ready{namespace="", condition="true", pod=~"redis-.*|redis-replication-.*"}) redis_memory_used_bytes{namespace=""} redis_memory_max_bytes{namespace=""} -codeplace_postgres_connections{namespace=""} -codeplace_postgres_max_connections{namespace=""} -codeplace_postgres_long_transactions{namespace=""} -codeplace_postgres_lock_waits{namespace=""} codeplace_redis_connected_clients{namespace=""} codeplace_redis_max_clients{namespace=""} increase(codeplace_redis_rejected_connections_total{namespace=""}[5m]) @@ -921,9 +917,7 @@ kubectl -n get pod -l app=redis-replication -o wide - `PostgresCollectorError`는 metrics query 실패이므로 CNPG exporter 로그와 PostgreSQL 권한/상태를 확인합니다. - `PostgresHADegraded`는 3개 instance가 서로 다른 node에 분산되지 않은 상태입니다. node 수, taint, Longhorn volume attach 상태를 확인합니다. - `PostgresReplicaUnavailable` / `RedisReplicaUnavailable`은 3노드 분산 상태에서 남은 replica 수와 node placement를 같이 봅니다. 일부 replica만 down이면 failover 여지는 있지만 다음 장애에 취약합니다. -- `PostgresConnectionUsageHigh`는 backend connection leak, worker 동시성 증가, DB max_connections 부족, slow query 누적으로 구분합니다. API latency와 동시에 오르면 DB 병목 후보가 강합니다. -- `PostgresLongTransactions`는 transaction 범위가 긴 view/task, idle-in-transaction, migration/admin 작업을 확인합니다. -- `PostgresLockWaits`는 쓰기 contention이나 DDL/migration lock 가능성이 큽니다. 같은 시간대 backend error와 Celery runtime 증가를 같이 봅니다. +- PostgreSQL connection/long transaction/lock wait는 backend `/metrics`에서 직접 DB 조회하지 않습니다. 필요하면 CNPG exporter에 동일 의미의 metric이 있는지 운영 클러스터에서 먼저 확인한 뒤, 확정된 exporter metric만 dashboard/rule에 추가합니다. - `RedisMemoryHigh`는 Redis `maxmemory`가 0보다 클 때만 동작합니다. `maxmemory=0`이면 Redis 자체 제한이 없으므로 container memory alert를 기준으로 보고, 제한을 둘지 별도로 결정합니다. - `RedisConnectedClientsHigh`는 connection pool 설정, worker replica/concurrency, client leak를 확인합니다. - `RedisRejectedConnections`는 Redis `maxclients`에 실제로 걸린 상태입니다. API/cache와 Celery queue가 동시에 영향을 받을 수 있습니다. diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 79a129a6..783ffd08 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -550,22 +550,20 @@ data: }, { "type": "timeseries", - "title": "PostgreSQL Connections", + "title": "PostgreSQL Collector Errors", "gridPos": {"x": 0, "y": 65, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "codeplace_postgres_connections{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} connections"}, - {"expr": "codeplace_postgres_max_connections{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} max"} + {"expr": "max by (namespace, pod) (cnpg_collector_last_collection_error{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} {{pod}}"} ] }, { "type": "timeseries", - "title": "PostgreSQL Long Tx and Lock Waits", + "title": "PostgreSQL Nodes Used", "gridPos": {"x": 8, "y": 65, "w": 8, "h": 6}, "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "codeplace_postgres_long_transactions{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} long tx"}, - {"expr": "codeplace_postgres_lock_waits{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} lock waits"} + {"expr": "min by (namespace) (cnpg_collector_nodes_used{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} nodes"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index b5b3301d..7a75e5d2 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -787,33 +787,6 @@ spec: annotations: summary: "Redis memory usage is high" description: "Redis exporter reports memory usage above 85% of configured maxmemory for 10 minutes." - - alert: PostgresConnectionUsageHigh - expr: codeplace_postgres_connections{namespace=~"code-place-(dev|prod)"} / clamp_min(codeplace_postgres_max_connections{namespace=~"code-place-(dev|prod)"}, 1) > 0.8 - for: 10m - labels: - severity: warning - priority: P1 - annotations: - summary: "PostgreSQL connection usage is high" - description: "Backend reports PostgreSQL connection usage above 80% of max_connections for 10 minutes." - - alert: PostgresLongTransactions - expr: codeplace_postgres_long_transactions{namespace=~"code-place-(dev|prod)"} > 0 - for: 10m - labels: - severity: warning - priority: P1 - annotations: - summary: "PostgreSQL long transactions detected" - description: "Backend reports at least one PostgreSQL transaction older than 5 minutes for 10 minutes." - - alert: PostgresLockWaits - expr: codeplace_postgres_lock_waits{namespace=~"code-place-(dev|prod)"} > 0 - for: 5m - labels: - severity: warning - priority: P1 - annotations: - summary: "PostgreSQL lock waits detected" - description: "Backend reports PostgreSQL locks waiting to be granted for 5 minutes." - alert: RedisConnectedClientsHigh expr: (codeplace_redis_connected_clients{namespace=~"code-place-(dev|prod)"} / clamp_min(codeplace_redis_max_clients{namespace=~"code-place-(dev|prod)"}, 1) > 0.8) and codeplace_redis_max_clients{namespace=~"code-place-(dev|prod)"} > 0 for: 10m From ae5c36e295c0a928d71ab98eafafba07a9dec869 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 01:02:49 +0900 Subject: [PATCH 119/147] =?UTF-8?q?Loki=20=EA=B2=8C=EC=9D=B4=ED=8A=B8?= =?UTF-8?q?=EC=9B=A8=EC=9D=B4=20=EB=A9=94=ED=8A=B8=EB=A6=AD=20=EB=8C=80?= =?UTF-8?q?=EC=83=81=20=EC=A0=95=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/developer/infra/observability-runbook/_index.md | 2 +- kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml | 2 +- kubernetes/monitoring/prometheus-rules.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 971bfbc0..813de142 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -610,7 +610,7 @@ PromQL: ```promql kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} up{namespace="monitoring", service="kube-prometheus-stack-grafana"} -up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"} +up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy"} sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) prometheus_config_last_reload_successful{namespace="monitoring"} diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index f685868b..7ced1fec 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -235,7 +235,7 @@ data: } }, "targets": [ - {"expr": "max by (service) (up{namespace=\"monitoring\", service=~\"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy\"})", "legendFormat": "{{service}}"} + {"expr": "max by (service) (up{namespace=\"monitoring\", service=~\"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy\"})", "legendFormat": "{{service}}"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 7a75e5d2..0b6b0aea 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -408,7 +408,7 @@ spec: summary: "Grafana metrics target is down" description: "Prometheus cannot scrape Grafana /metrics for 2 minutes. Grafana may still serve UI, but dashboard/datasource/provisioning health visibility is degraded." - alert: MonitoringServiceMetricsTargetDown - expr: (up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"} == 0) or (kube_service_info{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"} unless on (namespace, service) up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-gateway|loki-canary|alloy"}) + expr: (up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy"} == 0) or (kube_service_info{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy"} unless on (namespace, service) up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy"}) for: 2m labels: severity: warning From 21685cda8099301edad780b6cdef488c02b231d4 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 01:19:00 +0900 Subject: [PATCH 120/147] =?UTF-8?q?=EC=95=8C=EB=A6=BC=20=EB=9D=BC=EC=9A=B0?= =?UTF-8?q?=ED=8C=85=20=EA=B8=B0=EB=B3=B8=20=EC=88=98=EC=8B=A0=EC=9E=90=20?= =?UTF-8?q?=EB=AC=B4=ED=95=B4=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../developer/infra/observability-improvement-plan/_index.md | 2 +- docs/content/developer/infra/observability-runbook/_index.md | 1 + kubernetes/monitoring/alertmanager-config-email.example.yaml | 3 ++- kubernetes/monitoring/alertmanager-config.yaml | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 37f8d507..2fff3d10 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -371,7 +371,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - Discord webhook URL은 Git에 평문으로 저장하지 않습니다. 운영 Secret은 SealedSecret으로 관리합니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. - OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지하고 dev overlay에서 먼저 활성화합니다. -- P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. dev P1은 `dev-p1-muted` receiver로 보내 외부 전송을 막고, prod/monitoring P1은 Discord receiver로 보냅니다. +- P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. root fallback은 `unmatched-muted`로 두어 CodePlace priority label이 없는 기본/외부 alert가 Discord로 새지 않게 합니다. dev P1은 `dev-p1-muted` receiver로 보내 외부 전송을 막고, prod/monitoring P1은 Discord receiver로 보냅니다. - Metrics backend는 kube-prometheus-stack의 Prometheus입니다. Mimir는 현재 온프렘 단기 운영 기준선에 포함하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - Kubernetes event는 Kubernetes Event Exporter가 Warning event만 stdout JSON으로 내보내고, Alloy가 일반 Pod log와 같은 경로로 Loki에 적재합니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 813de142..2cd088c4 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -49,6 +49,7 @@ Status -> Configuration -> p0-discord / p1-discord ``` Discord webhook은 SealedSecret으로 관리합니다. AlertmanagerConfig는 최종적으로 `monitoring` namespace의 `alertmanager-contact-points` Secret key `webhook-url`을 참조하므로, 알림이 오지 않으면 SealedSecret controller와 복호화된 Secret을 함께 확인합니다. +CodePlace AlertmanagerConfig의 root fallback receiver는 `unmatched-muted`입니다. `priority=P0` 또는 `priority=P1` label이 없는 기본/외부 alert는 이 route에서 Discord로 보내지 않습니다. ```sh kubectl get crd sealedsecrets.bitnami.com diff --git a/kubernetes/monitoring/alertmanager-config-email.example.yaml b/kubernetes/monitoring/alertmanager-config-email.example.yaml index 4c448539..f15a2658 100644 --- a/kubernetes/monitoring/alertmanager-config-email.example.yaml +++ b/kubernetes/monitoring/alertmanager-config-email.example.yaml @@ -23,7 +23,7 @@ spec: groupWait: 30s groupInterval: 5m repeatInterval: 1h - receiver: p1-email + receiver: unmatched-muted routes: - matchers: - name: priority @@ -40,6 +40,7 @@ spec: repeatInterval: 1h receiver: p1-email receivers: + - name: unmatched-muted - name: p0-email emailConfigs: - sendResolved: true diff --git a/kubernetes/monitoring/alertmanager-config.yaml b/kubernetes/monitoring/alertmanager-config.yaml index 795a0da6..99de5764 100644 --- a/kubernetes/monitoring/alertmanager-config.yaml +++ b/kubernetes/monitoring/alertmanager-config.yaml @@ -13,7 +13,7 @@ spec: groupWait: 30s groupInterval: 5m repeatInterval: 1h - receiver: p1-discord + receiver: unmatched-muted routes: - matchers: - name: priority @@ -39,6 +39,7 @@ spec: repeatInterval: 1h receiver: p1-discord receivers: + - name: unmatched-muted - name: p0-discord discordConfigs: - sendResolved: true From 72a2c015d4e310d3783363995aa16a2fd6533ec9 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 01:39:12 +0900 Subject: [PATCH 121/147] =?UTF-8?q?judge=20=EC=84=9C=EB=B2=84=20=EA=B4=80?= =?UTF-8?q?=EC=B8=A1=20DB=20=EC=A1=B0=ED=9A=8C=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/conf/views.py | 12 ++- backend/judge/dispatcher.py | 4 + backend/utils/constants.py | 1 + backend/utils/judge_server_observability.py | 101 ++++++++++++++++++ backend/utils/observability_metrics.py | 25 ++--- backend/utils/tests.py | 54 +++++----- .../observability-improvement-plan/_index.md | 2 +- kubernetes/monitoring/prometheus-rules.yaml | 1 + 8 files changed, 156 insertions(+), 44 deletions(-) create mode 100644 backend/utils/judge_server_observability.py diff --git a/backend/conf/views.py b/backend/conf/views.py index 5593c8d2..8bd42c09 100644 --- a/backend/conf/views.py +++ b/backend/conf/views.py @@ -22,6 +22,7 @@ from problem.models import Problem from submission.models import Submission from utils.api import APIView, CSRFExemptAPIView, validate_serializer +from utils.judge_server_observability import remove_judge_server_snapshot, sync_judge_server_snapshot from utils.shortcuts import send_email, get_env from utils.xss_filter import XSSHtml from .models import JudgeServer @@ -133,13 +134,18 @@ def delete(self, request): hostname = request.GET.get("hostname") if hostname: JudgeServer.objects.filter(hostname=hostname).delete() + remove_judge_server_snapshot(hostname) return self.success() @validate_serializer(EditJudgeServerSerializer) @super_admin_required def put(self, request): is_disabled = request.data.get("is_disabled", False) - JudgeServer.objects.filter(id=request.data["id"]).update(is_disabled=is_disabled) + server = JudgeServer.objects.filter(id=request.data["id"]).first() + if server: + server.is_disabled = is_disabled + server.save(update_fields=["is_disabled"]) + sync_judge_server_snapshot(server) if not is_disabled: process_pending_task() return self.success() @@ -165,8 +171,9 @@ def post(self, request): server.last_heartbeat = timezone.now() server.save( update_fields=["judger_version", "cpu_core", "memory_usage", "service_url", "ip", "last_heartbeat"]) + sync_judge_server_snapshot(server) except JudgeServer.DoesNotExist: - JudgeServer.objects.create( + server = JudgeServer.objects.create( hostname=data["hostname"], judger_version=data["judger_version"], cpu_core=data["cpu_core"], @@ -176,6 +183,7 @@ def post(self, request): service_url=data["service_url"], last_heartbeat=timezone.now(), ) + sync_judge_server_snapshot(server) # 新server上线 处理队列中的,防止没有新的提交而导致一直waiting process_pending_task() diff --git a/backend/judge/dispatcher.py b/backend/judge/dispatcher.py index 9b995c88..58273040 100644 --- a/backend/judge/dispatcher.py +++ b/backend/judge/dispatcher.py @@ -20,6 +20,7 @@ from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.constants import CacheKey, ProblemScore, ProblemField, Tier +from utils.judge_server_observability import increment_judge_server_task_snapshot, sync_judge_server_snapshot from utils.observability_metrics import record_judge_duration from utils.observability_tracing import get_tracer @@ -52,8 +53,10 @@ def __enter__(self) -> [JudgeServer, None]: span.set_attribute("codeplace.judge_server.candidates", len(servers)) for server in servers: if server.task_number <= server.cpu_core * 2: + selected_task_number = server.task_number + 1 server.task_number = F("task_number") + 1 server.save(update_fields=["task_number"]) + sync_judge_server_snapshot(server, task_number=selected_task_number) self.server = server span.set_attribute("codeplace.judge_server.selected", True) span.set_attribute("codeplace.judge_server.hostname", server.hostname) @@ -64,6 +67,7 @@ def __enter__(self) -> [JudgeServer, None]: def __exit__(self, exc_type, exc_val, exc_tb): if self.server: JudgeServer.objects.filter(id=self.server.id).update(task_number=F("task_number") - 1) + increment_judge_server_task_snapshot(self.server.hostname, -1) class DispatcherBase(object): diff --git a/backend/utils/constants.py b/backend/utils/constants.py index ad2836a1..7cad629e 100644 --- a/backend/utils/constants.py +++ b/backend/utils/constants.py @@ -24,6 +24,7 @@ class ContestRuleType(Choices): class CacheKey: waiting_queue = "waiting_queue" + judge_server_observability = "codeplace:judge_server_observability" contest_rank_cache = "contest_rank_cache" website_config = "website_config" diff --git a/backend/utils/judge_server_observability.py b/backend/utils/judge_server_observability.py new file mode 100644 index 00000000..335f12d8 --- /dev/null +++ b/backend/utils/judge_server_observability.py @@ -0,0 +1,101 @@ +import json +import logging + +from django.utils import timezone + +from utils.cache import cache +from utils.constants import CacheKey + +logger = logging.getLogger(__name__) + + +def sync_judge_server_snapshot(server, is_disabled=None, task_number=None): + hostname = getattr(server, "hostname", None) + if not hostname: + return + + disabled = getattr(server, "is_disabled", False) if is_disabled is None else is_disabled + if disabled: + remove_judge_server_snapshot(hostname) + return + + last_heartbeat = getattr(server, "last_heartbeat", None) + if last_heartbeat is None: + last_heartbeat_timestamp = 0 + else: + last_heartbeat_timestamp = last_heartbeat.timestamp() + + payload = { + "last_heartbeat_timestamp": last_heartbeat_timestamp, + "task_number": int(task_number if task_number is not None else getattr(server, "task_number", 0) or 0), + } + try: + cache.hset(CacheKey.judge_server_observability, hostname, json.dumps(payload)) + except Exception: + logger.exception("Failed to sync judge-server observability snapshot") + + +def remove_judge_server_snapshot(hostname): + if not hostname: + return + try: + cache.hdel(CacheKey.judge_server_observability, hostname) + except Exception: + logger.exception("Failed to remove judge-server observability snapshot") + + +def increment_judge_server_task_snapshot(hostname, delta): + if not hostname: + return + try: + raw = cache.hget(CacheKey.judge_server_observability, hostname) + if raw is None: + return + payload = _loads_snapshot(raw) + payload["task_number"] = max(int(payload.get("task_number", 0)) + int(delta), 0) + cache.hset(CacheKey.judge_server_observability, hostname, json.dumps(payload)) + except Exception: + logger.exception("Failed to increment judge-server task snapshot") + + +def load_judge_server_snapshots(): + snapshots = [] + try: + raw_snapshots = cache.hgetall(CacheKey.judge_server_observability) or {} + except Exception: + logger.exception("Failed to load judge-server observability snapshots") + return None + + now_timestamp = timezone.now().timestamp() + for raw_hostname, raw_payload in raw_snapshots.items(): + hostname = _decode(raw_hostname) + payload = _loads_snapshot(raw_payload) + last_heartbeat_timestamp = _float_or_zero(payload.get("last_heartbeat_timestamp")) + heartbeat_age = max(now_timestamp - last_heartbeat_timestamp, 0) if last_heartbeat_timestamp else float("inf") + snapshots.append({ + "hostname": hostname, + "heartbeat_age": heartbeat_age, + "available": 1 if heartbeat_age <= 6 else 0, + "task_number": max(int(_float_or_zero(payload.get("task_number"))), 0), + }) + return snapshots + + +def _loads_snapshot(raw): + try: + return json.loads(_decode(raw)) + except (TypeError, ValueError): + return {} + + +def _decode(value): + if isinstance(value, bytes): + return value.decode() + return str(value) + + +def _float_or_zero(value): + try: + return float(value) + except (TypeError, ValueError): + return 0 diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 6976fa9c..359685c5 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -16,6 +16,7 @@ CELERY_TASK_TOTAL_KEY, ) from utils.constants import CacheKey +from utils.judge_server_observability import load_judge_server_snapshots logger = logging.getLogger(__name__) @@ -183,8 +184,6 @@ def _celery_broker_queue_length(self): return metric def _judge_server_metrics(self): - from conf.models import JudgeServer - available_metric = GaugeMetricFamily( "codeplace_judge_server_available", "Whether each judge-server is enabled and has a fresh heartbeat.", @@ -200,24 +199,16 @@ def _judge_server_metrics(self): "Current task_number recorded for each judge-server.", labels=["hostname"], ) - now = timezone.now() - try: - servers = list(JudgeServer.objects.all()) - except Exception as e: - logger.warning("Failed to collect judge-server metrics: %s", e) - servers = [] + snapshots = load_judge_server_snapshots() + if snapshots is None: self._mark_collector_success("judge_server", False) else: self._mark_collector_success("judge_server", True) - for server in servers: - if server.is_disabled: - continue - hostname = server.hostname or "unknown" - heartbeat_age = max((now - server.last_heartbeat).total_seconds(), 0) - available = 1 if server.status == "normal" else 0 - available_metric.add_metric([hostname], available) - heartbeat_metric.add_metric([hostname], heartbeat_age) - task_metric.add_metric([hostname], server.task_number) + for snapshot in snapshots or []: + hostname = snapshot["hostname"] or "unknown" + available_metric.add_metric([hostname], snapshot["available"]) + heartbeat_metric.add_metric([hostname], snapshot["heartbeat_age"]) + task_metric.add_metric([hostname], snapshot["task_number"]) yield available_metric yield heartbeat_metric yield task_metric diff --git a/backend/utils/tests.py b/backend/utils/tests.py index cb3f7fde..1968baf9 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -11,6 +11,8 @@ from utils import celery_observability from utils.json_logging import CodePlaceJsonFormatter from utils.observability_context import get_request_id, reset_request_id, set_request_id +from utils.constants import CacheKey +from utils.judge_server_observability import increment_judge_server_task_snapshot from utils.observability_metrics import CodePlaceCollector from utils.observability_tracing import get_tracer from utils.testcase_cache import TestCaseCacheManager @@ -21,24 +23,16 @@ class CodePlaceCollectorTest(SimpleTestCase): def _samples_by_metric(self, metrics): return {metric.name: metric.samples for metric in metrics} - def test_judge_server_metrics_skip_disabled_servers(self): + def test_judge_server_metrics_emit_snapshot_servers(self): now = timezone.now() - enabled = MagicMock( - hostname="enabled-judge", - is_disabled=False, - last_heartbeat=now, - status="normal", - task_number=2, - ) - disabled = MagicMock( - hostname="disabled-judge", - is_disabled=True, - last_heartbeat=now - timedelta(days=30), - status="abnormal", - task_number=99, - ) + snapshots = { + b"enabled-judge": json.dumps({ + "last_heartbeat_timestamp": now.timestamp(), + "task_number": 2, + }).encode(), + } - with patch("conf.models.JudgeServer.objects.all", return_value=[enabled, disabled]): + with patch("utils.judge_server_observability.cache.hgetall", return_value=snapshots): samples = self._samples_by_metric(list(CodePlaceCollector()._judge_server_metrics())) for metric_samples in samples.values(): @@ -47,15 +41,14 @@ def test_judge_server_metrics_skip_disabled_servers(self): self.assertNotIn("disabled-judge", hostnames) def test_judge_server_metrics_mark_stale_enabled_server_unavailable(self): - stale = MagicMock( - hostname="stale-judge", - is_disabled=False, - last_heartbeat=timezone.now() - timedelta(minutes=10), - status="abnormal", - task_number=1, - ) + snapshots = { + b"stale-judge": json.dumps({ + "last_heartbeat_timestamp": (timezone.now() - timedelta(minutes=10)).timestamp(), + "task_number": 1, + }).encode(), + } - with patch("conf.models.JudgeServer.objects.all", return_value=[stale]): + with patch("utils.judge_server_observability.cache.hgetall", return_value=snapshots): samples = self._samples_by_metric(list(CodePlaceCollector()._judge_server_metrics())) available = samples["codeplace_judge_server_available"][0] @@ -64,6 +57,19 @@ def test_judge_server_metrics_mark_stale_enabled_server_unavailable(self): self.assertEqual(available.value, 0) self.assertGreaterEqual(heartbeat_age.value, 599) + def test_judge_server_metrics_are_loaded_from_redis_snapshot(self): + with patch("utils.judge_server_observability.cache.hgetall", return_value={}) as hgetall: + list(CodePlaceCollector()._judge_server_metrics()) + + hgetall.assert_called_once_with(CacheKey.judge_server_observability) + + def test_judge_server_task_increment_does_not_create_snapshot_without_heartbeat(self): + with patch("utils.judge_server_observability.cache.hget", return_value=None), \ + patch("utils.judge_server_observability.cache.hset") as hset: + increment_judge_server_task_snapshot("judge-1", 1) + + hset.assert_not_called() + def test_celery_broker_queue_length_is_collected_from_redis(self): with patch("utils.observability_metrics.cache.llen", return_value=7) as llen: samples = self._samples_by_metric([CodePlaceCollector()._celery_broker_queue_length()]) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 2fff3d10..1aced3aa 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -375,7 +375,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - Metrics backend는 kube-prometheus-stack의 Prometheus입니다. Mimir는 현재 온프렘 단기 운영 기준선에 포함하지 않습니다. - 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. - Kubernetes event는 Kubernetes Event Exporter가 Warning event만 stdout JSON으로 내보내고, Alloy가 일반 Pod log와 같은 경로로 Loki에 적재합니다. -- PostgreSQL은 backend `/metrics`에서 직접 조회하지 않고 CNPG instance exporter와 PodMonitor의 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor의 `redis_*` 지표를 기본으로 보고, backend custom collector는 queue length와 client/rejection 같은 O(1) Redis 상태만 수집합니다. +- PostgreSQL은 backend `/metrics`에서 직접 조회하지 않고 CNPG instance exporter와 PodMonitor의 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor의 `redis_*` 지표를 기본으로 보고, backend custom collector는 queue length, judge-server heartbeat snapshot, client/rejection 같은 O(1) Redis 상태만 수집합니다. - Sentry backend SDK는 기본 PII 자동 전송을 비활성화하고 전송 직전 `authorization`, `cookie`, `password`, `token`, `secret`, 제출 source code 계열 필드를 redaction합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. - Frontend는 axios 요청마다 `X-Request-ID`를 전파하고 마지막 request ID를 `window.__CODEPLACE_LAST_REQUEST_ID__`와 Sentry request context 및 `/api/client_error` report에 저장합니다. - Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. DSN이 없으면 Sentry는 비활성화되지만 `/api/client_error` 기반 metric/log는 계속 동작합니다. diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 0b6b0aea..a268d960 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -122,6 +122,7 @@ spec: description: "No enabled judge-server has a fresh heartbeat." - alert: JudgeHeartbeatCritical expr: max by (namespace) (codeplace_judge_server_last_heartbeat_age_seconds{namespace=~"code-place-(dev|prod)"}) > 180 + for: 1m labels: severity: critical priority: P0 From 381583020eb5187fbeac38fb8e8b1bf4c0add15f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 01:42:30 +0900 Subject: [PATCH 122/147] =?UTF-8?q?Redis=20=EA=B4=80=EC=B8=A1=20=EC=88=98?= =?UTF-8?q?=EC=A7=91=20=EC=84=B1=EA=B3=B5=20=ED=8C=90=EC=A0=95=20=EB=B3=B4?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_metrics.py | 26 +++++++++++-------- backend/utils/tests.py | 35 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 359685c5..7db14e2e 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -121,7 +121,7 @@ def describe(self): ) yield GaugeMetricFamily( "codeplace_redis_max_clients", - "Configured Redis maxclients from CONFIG GET maxclients.", + "Configured Redis maxclients from INFO clients or CONFIG GET maxclients.", ) yield CounterMetricFamily( "codeplace_redis_rejected_connections_total", @@ -330,7 +330,7 @@ def _redis_health_metrics(self): ) max_clients_metric = GaugeMetricFamily( "codeplace_redis_max_clients", - "Configured Redis maxclients from CONFIG GET maxclients.", + "Configured Redis maxclients from INFO clients or CONFIG GET maxclients.", ) rejected_connections_metric = CounterMetricFamily( "codeplace_redis_rejected_connections_total", @@ -341,20 +341,24 @@ def _redis_health_metrics(self): info = client.info() connected_clients = float(info.get("connected_clients") or 0) rejected_connections = float(info.get("rejected_connections") or 0) - max_clients = 0 - try: - raw_max_clients = client.config_get("maxclients") - max_clients = float(raw_max_clients.get("maxclients") or 0) - except Exception as e: - logger.warning("Failed to collect Redis maxclients metric: %s", e) - self._mark_collector_success("redis", False) + max_clients = self._float_or_zero(info.get("maxclients")) + if max_clients == 0: + try: + raw_max_clients = client.config_get("maxclients") + max_clients = self._float_or_zero(raw_max_clients.get("maxclients")) + except Exception as e: + logger.warning("Failed to collect Redis maxclients metric: %s", e) + redis_success = False + else: + redis_success = True + else: + redis_success = True except Exception as e: logger.warning("Failed to collect Redis health metrics: %s", e) connected_clients = max_clients = rejected_connections = 0 self._mark_collector_success("redis", False) else: - if not getattr(self, "_collector_success", {}).get("redis", False): - self._mark_collector_success("redis", True) + self._mark_collector_success("redis", redis_success) connected_clients_metric.add_metric([], connected_clients) max_clients_metric.add_metric([], max_clients) diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 1968baf9..da596599 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -197,6 +197,41 @@ def test_redis_health_metrics_are_collected(self): self.assertEqual(rejected_sample.name, "codeplace_redis_rejected_connections_total") self.assertEqual(rejected_sample.value, 3) + def test_redis_health_metrics_prefer_info_maxclients(self): + redis_client = MagicMock() + redis_client.info.return_value = { + "connected_clients": 25, + "rejected_connections": 3, + "maxclients": 2000, + } + + with patch("utils.observability_metrics.cache.client.get_client", return_value=redis_client): + samples = self._samples_by_metric(list(CodePlaceCollector()._redis_health_metrics())) + + redis_client.config_get.assert_not_called() + self.assertEqual(samples["codeplace_redis_max_clients"][0].value, 2000) + + def test_redis_collector_failure_is_reported_when_maxclients_is_unavailable(self): + redis_client = MagicMock() + redis_client.info.return_value = { + "connected_clients": 25, + "rejected_connections": 3, + } + redis_client.config_get.side_effect = RuntimeError("CONFIG disabled") + collector = CodePlaceCollector() + collector._collector_success = {} + + with patch("utils.observability_metrics.cache.client.get_client", return_value=redis_client), \ + patch("utils.observability_metrics.logger"): + list(collector._redis_health_metrics()) + samples = self._samples_by_metric([collector._collector_success_metric()]) + + redis_success = [ + sample for sample in samples["codeplace_observability_collector_success"] + if sample.labels["collector"] == "redis" + ][0] + self.assertEqual(redis_success.value, 0) + class CeleryObservabilityTest(SimpleTestCase): From 9a15919b9c28ac5dd9694b72ca44d73463f751ea Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 01:45:35 +0900 Subject: [PATCH 123/147] =?UTF-8?q?Overview=20=EB=8C=80=EC=8B=9C=EB=B3=B4?= =?UTF-8?q?=EB=93=9C=20Prometheus=20=EB=AA=85=EC=8B=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grafana-dashboard-codeplace.yaml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 783ffd08..a9b497f1 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -41,6 +41,7 @@ data: "type": "stat", "title": "Backend Up", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -62,6 +63,7 @@ data: "type": "stat", "title": "API / Ingress Request Rate", "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "reqps"}}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ @@ -73,6 +75,7 @@ data: "type": "stat", "title": "API / Ingress 5xx Ratio", "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "percentunit", @@ -96,6 +99,7 @@ data: "type": "stat", "title": "p95 Latency", "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "s", @@ -118,6 +122,7 @@ data: "type": "stat", "title": "Judge Waiting Queue", "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -140,6 +145,7 @@ data: "type": "stat", "title": "Judge Available", "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -161,6 +167,7 @@ data: "type": "timeseries", "title": "Backend Requests by Status", "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "reqps"}}, "targets": [ {"expr": "sum by (namespace, status_code) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{status_code}}"} @@ -170,6 +177,7 @@ data: "type": "timeseries", "title": "Backend Latency", "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "s"}}, "targets": [ {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p95"}, @@ -180,6 +188,7 @@ data: "type": "bargauge", "title": "Judge Queue Length", "gridPos": {"x": 16, "y": 4, "w": 4, "h": 8}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ @@ -191,6 +200,7 @@ data: "type": "bargauge", "title": "Custom Collector Success", "gridPos": {"x": 20, "y": 4, "w": 4, "h": 8}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -219,6 +229,7 @@ data: "type": "timeseries", "title": "Celery Task Throughput", "gridPos": {"x": 0, "y": 13, "w": 8, "h": 7}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "ops"}}, "targets": [ {"expr": "sum by (namespace, task_name, status) (rate(codeplace_celery_task_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{task_name}} {{status}}"} @@ -228,6 +239,7 @@ data: "type": "timeseries", "title": "Celery Task Runtime / Freshness", "gridPos": {"x": 8, "y": 13, "w": 8, "h": 7}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "s"}}, "targets": [ {"expr": "histogram_quantile(0.95, sum by (namespace, task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} {{task_name}} p95"}, @@ -238,6 +250,7 @@ data: "type": "bargauge", "title": "Celery Last Runtime", "gridPos": {"x": 16, "y": 13, "w": 8, "h": 7}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "s"}}, "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ @@ -248,6 +261,7 @@ data: "type": "timeseries", "title": "Judge Duration p95", "gridPos": {"x": 0, "y": 20, "w": 8, "h": 8}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "s"}}, "targets": [ {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_submission_judge_duration_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} judge p95"} @@ -257,6 +271,7 @@ data: "type": "timeseries", "title": "Judge Heartbeat Max Age", "gridPos": {"x": 8, "y": 20, "w": 8, "h": 8}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "s", @@ -278,6 +293,7 @@ data: "type": "timeseries", "title": "Judge Server Load", "gridPos": {"x": 16, "y": 20, "w": 8, "h": 8}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "topk(10, codeplace_judge_server_task_number{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{hostname}} tasks"} @@ -287,6 +303,7 @@ data: "type": "timeseries", "title": "Submission / Judge Outcomes", "gridPos": {"x": 0, "y": 28, "w": 24, "h": 8}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "ops"}}, "targets": [ {"expr": "sum by (namespace, scope, status) (rate(codeplace_submission_create_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} create {{scope}} {{status}}/s"}, @@ -304,6 +321,7 @@ data: "type": "stat", "title": "Pod Ready", "gridPos": {"x": 0, "y": 37, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -325,6 +343,7 @@ data: "type": "timeseries", "title": "Pod Restarts", "gridPos": {"x": 6, "y": 37, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -334,6 +353,7 @@ data: "type": "timeseries", "title": "Container CPU Usage", "gridPos": {"x": 12, "y": 37, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "cores"}}, "targets": [ {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} @@ -343,6 +363,7 @@ data: "type": "timeseries", "title": "Container Memory Usage", "gridPos": {"x": 18, "y": 37, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} @@ -352,6 +373,7 @@ data: "type": "timeseries", "title": "PVC Usage", "gridPos": {"x": 0, "y": 42, "w": 8, "h": 6}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "percentunit", @@ -373,6 +395,7 @@ data: "type": "timeseries", "title": "Node Pressure", "gridPos": {"x": 8, "y": 42, "w": 8, "h": 6}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "kube_node_status_condition{condition=~\"MemoryPressure|DiskPressure|PIDPressure\", status=\"true\"}", "legendFormat": "{{node}} {{condition}}"} @@ -382,6 +405,7 @@ data: "type": "timeseries", "title": "Deployment Unavailable Replicas", "gridPos": {"x": 16, "y": 42, "w": 8, "h": 6}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "max by (namespace, deployment) (kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} unavailable"}, @@ -392,6 +416,7 @@ data: "type": "timeseries", "title": "Service Ready Endpoints", "gridPos": {"x": 0, "y": 48, "w": 12, "h": 6}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -412,6 +437,7 @@ data: "type": "timeseries", "title": "PDB Healthy Pods", "gridPos": {"x": 12, "y": 48, "w": 12, "h": 6}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "kube_poddisruptionbudget_status_current_healthy{namespace=~\"$namespace\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} current"}, @@ -429,6 +455,7 @@ data: "type": "stat", "title": "PostgreSQL Ready Replicas", "gridPos": {"x": 0, "y": 55, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -451,6 +478,7 @@ data: "type": "stat", "title": "Redis/Sentinel Ready Replicas", "gridPos": {"x": 6, "y": 55, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -473,6 +501,7 @@ data: "type": "stat", "title": "PostgreSQL Exporter Up", "gridPos": {"x": 12, "y": 55, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -494,6 +523,7 @@ data: "type": "stat", "title": "Redis Exporter Up", "gridPos": {"x": 18, "y": 55, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", @@ -515,6 +545,7 @@ data: "type": "timeseries", "title": "PostgreSQL Restarts", "gridPos": {"x": 0, "y": 60, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -524,6 +555,7 @@ data: "type": "timeseries", "title": "Redis Restarts", "gridPos": {"x": 6, "y": 60, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} @@ -533,6 +565,7 @@ data: "type": "timeseries", "title": "PostgreSQL WAL Bytes", "gridPos": {"x": 12, "y": 60, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "Bps"}}, "targets": [ {"expr": "sum by (namespace) (rate(cnpg_collector_wal_bytes{namespace=~\"$namespace\", cluster=\"postgres\"}[5m]))", "legendFormat": "{{namespace}}"} @@ -542,6 +575,7 @@ data: "type": "timeseries", "title": "Redis Memory Usage", "gridPos": {"x": 18, "y": 60, "w": 6, "h": 5}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "bytes"}}, "targets": [ {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} used"}, @@ -552,6 +586,7 @@ data: "type": "timeseries", "title": "PostgreSQL Collector Errors", "gridPos": {"x": 0, "y": 65, "w": 8, "h": 6}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "max by (namespace, pod) (cnpg_collector_last_collection_error{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} {{pod}}"} @@ -561,6 +596,7 @@ data: "type": "timeseries", "title": "PostgreSQL Nodes Used", "gridPos": {"x": 8, "y": 65, "w": 8, "h": 6}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "min by (namespace) (cnpg_collector_nodes_used{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} nodes"} @@ -570,6 +606,7 @@ data: "type": "timeseries", "title": "Redis Clients and Rejections", "gridPos": {"x": 16, "y": 65, "w": 8, "h": 6}, + "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "codeplace_redis_connected_clients{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} clients"}, @@ -581,6 +618,7 @@ data: "type": "timeseries", "title": "Frontend Runtime Errors", "gridPos": {"x": 0, "y": 71, "w": 12, "h": 6}, + "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "ops", From 853d9bb73f3f448d08c0aff1ab8b23f413ca179f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:03:26 +0900 Subject: [PATCH 124/147] =?UTF-8?q?judge=20=EC=84=9C=EB=B2=84=20=EA=B4=80?= =?UTF-8?q?=EC=B8=A1=20=EC=8A=A4=EB=83=85=EC=83=B7=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/judge_server_observability.py | 5 ++++ backend/utils/tests.py | 23 ++++++++++++++++++- .../infra/observability-runbook/_index.md | 4 +++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/utils/judge_server_observability.py b/backend/utils/judge_server_observability.py index 335f12d8..70101fe5 100644 --- a/backend/utils/judge_server_observability.py +++ b/backend/utils/judge_server_observability.py @@ -8,6 +8,8 @@ logger = logging.getLogger(__name__) +JUDGE_SERVER_SNAPSHOT_RETENTION_SECONDS = 24 * 60 * 60 + def sync_judge_server_snapshot(server, is_disabled=None, task_number=None): hostname = getattr(server, "hostname", None) @@ -72,6 +74,9 @@ def load_judge_server_snapshots(): payload = _loads_snapshot(raw_payload) last_heartbeat_timestamp = _float_or_zero(payload.get("last_heartbeat_timestamp")) heartbeat_age = max(now_timestamp - last_heartbeat_timestamp, 0) if last_heartbeat_timestamp else float("inf") + if heartbeat_age > JUDGE_SERVER_SNAPSHOT_RETENTION_SECONDS: + remove_judge_server_snapshot(hostname) + continue snapshots.append({ "hostname": hostname, "heartbeat_age": heartbeat_age, diff --git a/backend/utils/tests.py b/backend/utils/tests.py index da596599..28626ddf 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -12,7 +12,10 @@ from utils.json_logging import CodePlaceJsonFormatter from utils.observability_context import get_request_id, reset_request_id, set_request_id from utils.constants import CacheKey -from utils.judge_server_observability import increment_judge_server_task_snapshot +from utils.judge_server_observability import ( + JUDGE_SERVER_SNAPSHOT_RETENTION_SECONDS, + increment_judge_server_task_snapshot, +) from utils.observability_metrics import CodePlaceCollector from utils.observability_tracing import get_tracer from utils.testcase_cache import TestCaseCacheManager @@ -63,6 +66,24 @@ def test_judge_server_metrics_are_loaded_from_redis_snapshot(self): hgetall.assert_called_once_with(CacheKey.judge_server_observability) + def test_judge_server_metrics_prune_retired_snapshot(self): + retired_heartbeat = timezone.now() - timedelta(seconds=JUDGE_SERVER_SNAPSHOT_RETENTION_SECONDS + 1) + snapshots = { + b"retired-judge": json.dumps({ + "last_heartbeat_timestamp": retired_heartbeat.timestamp(), + "task_number": 0, + }).encode(), + } + + with patch("utils.judge_server_observability.cache.hgetall", return_value=snapshots), \ + patch("utils.judge_server_observability.cache.hdel") as hdel: + samples = self._samples_by_metric(list(CodePlaceCollector()._judge_server_metrics())) + + for metric_samples in samples.values(): + hostnames = {sample.labels["hostname"] for sample in metric_samples} + self.assertNotIn("retired-judge", hostnames) + hdel.assert_called_once_with(CacheKey.judge_server_observability, "retired-judge") + def test_judge_server_task_increment_does_not_create_snapshot_without_heartbeat(self): with patch("utils.judge_server_observability.cache.hget", return_value=None), \ patch("utils.judge_server_observability.cache.hset") as hset: diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index 2cd088c4..c0f7d5cc 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -318,6 +318,7 @@ kubectl -n logs deploy/celery-worker --tail=200 - judge Pod가 not ready이면 platform/deploy 문제입니다. - Pod는 ready인데 heartbeat가 stale이면 judge-server와 backend 통신 또는 token 문제입니다. - waiting queue가 같이 증가하면 사용자 제출 영향이 있습니다. +- backend `/metrics`는 judge DB row를 직접 조회하지 않고 Redis heartbeat snapshot만 노출합니다. 의도적으로 retire한 judge는 disable/delete API로 snapshot을 제거합니다. ### JudgeHeartbeatCritical @@ -338,7 +339,8 @@ kubectl -n describe pod -l app=judge-server 판단: - 특정 judge만 stale이면 해당 Pod/Node 문제입니다. -- 전체 judge가 stale이면 backend DB row, heartbeat endpoint, token 설정을 확인합니다. +- 전체 judge가 stale이면 judge-server Pod, heartbeat endpoint, token 설정을 확인합니다. +- 24시간 이상 heartbeat가 없는 Redis snapshot은 retired 상태로 보고 collector가 제거합니다. 그 이후에도 전체 judge가 없으면 `JudgeAllUnavailable`이 계속 장애를 잡습니다. ### PostgresUnavailable From 6e37405a6d8ab05905f4256b5bff350f23ea91d5 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:03:45 +0900 Subject: [PATCH 125/147] =?UTF-8?q?Alloy=20=EB=A1=9C=EA=B7=B8=20=EC=88=98?= =?UTF-8?q?=EC=A7=91=20=EC=84=A4=EC=A0=95=20=EB=AC=B8=EB=B2=95=20=EC=A0=95?= =?UTF-8?q?=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/logs/alloy-values.yaml | 52 ++++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml index 561e6ab4..8903943a 100644 --- a/kubernetes/monitoring/logs/alloy-values.yaml +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -21,12 +21,12 @@ alloy: fieldPath: spec.nodeName configMap: create: true - content: |- + content: | discovery.kubernetes "pod" { role = "pod" selectors { - role = "pod" + role = "pod" field = "spec.nodeName=" + sys.env("NODE_NAME") } } @@ -36,65 +36,65 @@ alloy: rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app", "__meta_kubernetes_pod_label_app_kubernetes_io_name"] - separator = "/" - regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik" - action = "keep" + separator = "/" + regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik" + action = "keep" } rule { source_labels = ["__meta_kubernetes_namespace"] - target_label = "namespace" + target_label = "namespace" } rule { source_labels = ["__meta_kubernetes_pod_name"] - target_label = "pod" + target_label = "pod" } rule { source_labels = ["__meta_kubernetes_pod_container_name"] - target_label = "container" + target_label = "container" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] - target_label = "node" + target_label = "node" } rule { source_labels = ["__meta_kubernetes_pod_label_app"] - action = "replace" - target_label = "app" + action = "replace" + target_label = "app" } rule { source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] - regex = "(.+)" - action = "replace" - target_label = "app_kubernetes_io_name" + regex = "(.+)" + action = "replace" + target_label = "app_kubernetes_io_name" } rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - action = "replace" - separator = "/" - target_label = "job" + action = "replace" + separator = "/" + target_label = "job" } rule { source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "__path__" - separator = "/" - replacement = "/var/log/pods/*$1/*.log" + action = "replace" + target_label = "__path__" + separator = "/" + replacement = "/var/log/pods/*$1/*.log" } rule { source_labels = ["__meta_kubernetes_pod_container_id"] - action = "replace" - target_label = "container_runtime" - regex = `^(\S+):\/\/.+$` - replacement = "$1" + action = "replace" + target_label = "container_runtime" + regex = `^(\S+):\/\/.+$` + replacement = "$1" } } @@ -106,7 +106,7 @@ alloy: loki.process "pod_logs" { stage.static_labels { values = { - cluster = "codeplace" + cluster = "codeplace", } } From a90d8e4764b8ac90f36b6e489036aad214995fd1 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:10:25 +0900 Subject: [PATCH 126/147] =?UTF-8?q?Alertmanager=20=EC=9B=B9=ED=9B=85=20?= =?UTF-8?q?=EC=8B=9C=ED=81=AC=EB=A6=BF=20=EA=B0=90=EC=8B=9C=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 1 + .../developer/infra/observability-runbook/_index.md | 5 ++++- kubernetes/monitoring/prometheus-rules.yaml | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 1aced3aa..f00da4d5 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -248,6 +248,7 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `GPUXIDError`: NVIDIA GPU XID error 1분 지속. - `PrometheusUnavailable`: Prometheus Pod not ready 1분 지속. - `AlertmanagerUnavailable`: Alertmanager Pod not ready 1분 지속. +- `AlertmanagerWebhookSecretMissing`: `alertmanager-contact-points` Secret missing 1분 지속. Discord 알림 자체가 막힐 수 있으므로 Alertmanager/Grafana UI에서 즉시 식별하기 위한 self-alert입니다. ### P1 diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index c0f7d5cc..b8dee4a3 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -595,12 +595,13 @@ kube_deployment_spec_replicas{namespace="monitoring", deployment="blackbox-expor - latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. - `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. -### GrafanaUnavailable / GrafanaMetricsTargetDown / MonitoringServiceMetricsTargetDown / PrometheusOperatorUnavailable / PrometheusConfigReloadFailed / AlertmanagerConfigReloadFailed / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures +### GrafanaUnavailable / GrafanaMetricsTargetDown / MonitoringServiceMetricsTargetDown / PrometheusOperatorUnavailable / PrometheusConfigReloadFailed / AlertmanagerConfigReloadFailed / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures / AlertmanagerWebhookSecretMissing 확인: ```sh kubectl -n monitoring get pod | grep -E 'grafana|operator|prometheus|alertmanager' +kubectl -n monitoring get secret alertmanager-contact-points kubectl -n monitoring describe pod -l app.kubernetes.io/name=grafana kubectl -n monitoring describe pod -l app.kubernetes.io/name=prometheus-operator kubectl -n monitoring logs deploy/kube-prometheus-stack-operator --tail=200 @@ -618,6 +619,7 @@ sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) prometheus_config_last_reload_successful{namespace="monitoring"} alertmanager_config_last_reload_successful{namespace="monitoring"} +kube_secret_info{namespace="monitoring", secret="alertmanager-contact-points"} prometheus_notifications_alertmanagers_discovered sum by (integration) (increase(alertmanager_notifications_failed_total[5m])) sum by (job, namespace) (up == 0) @@ -631,6 +633,7 @@ sum by (job, namespace) (up == 0) - Operator가 down이면 새 ServiceMonitor/PodMonitor/PrometheusRule/AlertmanagerConfig 변경이 Prometheus/Alertmanager에 반영되지 않을 수 있습니다. - rule evaluation failure가 있으면 최근 추가한 PromQL expression과 label cardinality, metric 존재 여부를 먼저 봅니다. - Alertmanager discovery가 0이면 AlertmanagerConfig가 맞아도 Prometheus가 알림을 보낼 대상이 없습니다. Prometheus generated config와 Alertmanager Service/endpoints를 확인합니다. +- webhook Secret이 없으면 Discord 알림은 동작하지 않습니다. 이 경우 SealedSecret controller가 `alertmanager-contact-points` Secret을 복호화했는지, Secret key가 `webhook-url`인지 확인합니다. - notification failure가 있으면 webhook Secret 값, Alertmanager generated config, Discord endpoint 응답, cluster outbound network, rate limit을 확인합니다. 로컬 `curl` 성공만으로 Alertmanager Pod의 egress 성공을 보장하지 않습니다. ### FrontendRuntimeErrorsSpike diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index a268d960..66b64481 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -299,6 +299,16 @@ spec: annotations: summary: "Alertmanager is not ready" description: "The kube-prometheus-stack Alertmanager pod has not been ready for 1 minute. Fast alert delivery may be unavailable." + - alert: AlertmanagerWebhookSecretMissing + expr: absent(kube_secret_info{namespace="monitoring", secret="alertmanager-contact-points"}) + for: 1m + labels: + severity: critical + priority: P0 + namespace: monitoring + annotations: + summary: "Alertmanager Discord webhook secret is missing" + description: "The monitoring namespace does not expose the alertmanager-contact-points Secret through kube-state-metrics. Discord notifications cannot work until the SealedSecret is decrypted into this Secret." - name: codeplace.p1 interval: 30s rules: From 5cb9a6d13ae048e392d5c9b9142d0e37f865e804 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:15:47 +0900 Subject: [PATCH 127/147] =?UTF-8?q?=EC=84=9C=EB=B9=84=EC=8A=A4=20endpoint?= =?UTF-8?q?=20=EA=B4=80=EC=B8=A1=20=EB=A9=94=ED=8A=B8=EB=A6=AD=20=EC=A0=95?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/developer/infra/observability-runbook/_index.md | 4 ++-- kubernetes/monitoring/grafana-dashboard-codeplace.yaml | 2 +- kubernetes/monitoring/prometheus-rules.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md index b8dee4a3..4f34c56e 100644 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ b/docs/content/developer/infra/observability-runbook/_index.md @@ -872,8 +872,8 @@ kubectl -n get event --sort-by=.lastTimestamp | tail -50 PromQL: ```promql -kube_endpoint_address_available{namespace=""} -kube_endpoint_address_not_ready{namespace=""} +sum by (namespace, endpoint) (kube_endpoint_address{namespace="", ready="true"}) +sum by (namespace, endpoint) (kube_endpoint_address{namespace="", ready="false"}) kube_pod_status_ready{namespace="", condition="true"} kube_deployment_status_replicas_unavailable{namespace=""} ``` diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index a9b497f1..280b1079 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -430,7 +430,7 @@ data: } }, "targets": [ - {"expr": "max by (namespace, endpoint) (kube_endpoint_address_available{namespace=~\"$namespace\", endpoint=~\"backend|frontend|hub-auth|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{endpoint}} ready"} + {"expr": "sum by (namespace, endpoint) (kube_endpoint_address{namespace=~\"$namespace\", endpoint=~\"backend|frontend|hub-auth|judge-server|vllm\", ready=\"true\"})", "legendFormat": "{{namespace}} {{endpoint}} ready endpoints"} ] }, { diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 66b64481..b58929fd 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -717,7 +717,7 @@ spec: summary: "CodePlace PDB has no disruption headroom" description: "A backend or frontend PodDisruptionBudget has been at or below its desired healthy count for 10 minutes. Node drain or voluntary disruption may reduce user-path availability." - alert: CodePlaceServiceNoReadyEndpoints - expr: (max by (namespace, endpoint) (kube_endpoint_address_available{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server"}) < 1) or (label_replace(kube_service_info{namespace=~"code-place-(dev|prod)", service=~"backend|frontend|hub-auth|judge-server"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address_available{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server"}) + expr: (sum by (namespace, endpoint) (kube_endpoint_address{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server", ready="true"}) < 1) or (label_replace(kube_service_info{namespace=~"code-place-(dev|prod)", service=~"backend|frontend|hub-auth|judge-server"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server", ready="true"}) for: 2m labels: severity: warning @@ -726,7 +726,7 @@ spec: summary: "CodePlace service has no ready endpoints" description: "A CodePlace Service has had zero ready endpoints for 2 minutes, or the endpoint availability metric is missing while the Service exists. Check Service selector labels, Pod readiness, EndpointSlice state, and rollout events." - alert: VLLMServiceNoReadyEndpoints - expr: (max by (namespace, endpoint) (kube_endpoint_address_available{namespace="code-place-prod", endpoint="vllm"}) < 1) or (label_replace(kube_service_info{namespace="code-place-prod", service="vllm"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address_available{namespace="code-place-prod", endpoint="vllm"}) + expr: (sum by (namespace, endpoint) (kube_endpoint_address{namespace="code-place-prod", endpoint="vllm", ready="true"}) < 1) or (label_replace(kube_service_info{namespace="code-place-prod", service="vllm"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address{namespace="code-place-prod", endpoint="vllm", ready="true"}) for: 2m labels: severity: warning From f79e9be8acc668ccf6f833e37259ca3df6c8f704 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:18:37 +0900 Subject: [PATCH 128/147] =?UTF-8?q?=EB=AA=A8=EB=8B=88=ED=84=B0=EB=A7=81=20?= =?UTF-8?q?Helm=20=EC=84=A4=EC=B9=98=20=EC=A0=88=EC=B0=A8=20=EB=B3=B4?= =?UTF-8?q?=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infra/observability-improvement-plan/_index.md | 6 +++--- kubernetes/monitoring/logs/README.md | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index f00da4d5..edacc28d 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -338,13 +338,13 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev ## 5. 운영 확인 절차 1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 Prometheus 2 replicas, Alertmanager 2 replicas, `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. - - `helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --version 86.3.1 --values kubernetes/monitoring/kube-prometheus-stack-values.yaml` + - `helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --create-namespace --version 86.3.1 --values kubernetes/monitoring/kube-prometheus-stack-values.yaml` - values는 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None`을 설정합니다. AlertmanagerConfig는 `monitoring` namespace에 있지만 CodePlace alert의 `namespace` label은 `code-place-dev`/`code-place-prod`이므로, 기본 `OnNamespace` matcher를 쓰면 외부 namespace alert가 Discord receiver까지 도달하지 않을 수 있습니다. 2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 SealedSecret으로 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. Email fallback을 켤 경우 SMTP 정보를 별도로 정한 뒤 `alertmanager-email` Secret과 email AlertmanagerConfig를 추가 적용합니다. 3. Loki와 Alloy를 설치합니다. - - `helm upgrade --install loki grafana/loki --namespace monitoring --version 6.55.0 --values kubernetes/monitoring/logs/loki-values.yaml` - - `helm upgrade --install alloy grafana/alloy --namespace monitoring --version 1.10.0 --values kubernetes/monitoring/logs/alloy-values.yaml` + - `helm upgrade --install loki grafana/loki --namespace monitoring --create-namespace --version 6.55.0 --values kubernetes/monitoring/logs/loki-values.yaml` + - `helm upgrade --install alloy grafana/alloy --namespace monitoring --create-namespace --version 1.10.0 --values kubernetes/monitoring/logs/alloy-values.yaml` 4. 애플리케이션은 환경별 overlay로 적용합니다. - dev: `kubectl apply -k kubernetes/overlays/dev` - prod: `kubectl apply -k kubernetes/overlays/prod` diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index eced9013..60c42d41 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -29,16 +29,19 @@ helm repo update helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ --namespace monitoring \ + --create-namespace \ --version 86.3.1 \ --values kubernetes/monitoring/kube-prometheus-stack-values.yaml helm upgrade --install loki grafana/loki \ --namespace monitoring \ + --create-namespace \ --version 6.55.0 \ --values kubernetes/monitoring/logs/loki-values.yaml helm upgrade --install alloy grafana/alloy \ --namespace monitoring \ + --create-namespace \ --version 1.10.0 \ --values kubernetes/monitoring/logs/alloy-values.yaml ``` From fd57d9919c4414ae2194047ff17194ec12aa6496 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:23:32 +0900 Subject: [PATCH 129/147] =?UTF-8?q?OTel=20Collector=20=EB=A9=94=ED=8A=B8?= =?UTF-8?q?=EB=A6=AD=20=EC=97=94=EB=93=9C=ED=8F=AC=EC=9D=B8=ED=8A=B8=20?= =?UTF-8?q?=EB=AA=85=EC=8B=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/otel-collector.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kubernetes/monitoring/otel-collector.yaml b/kubernetes/monitoring/otel-collector.yaml index b7afdbb1..116b95d5 100644 --- a/kubernetes/monitoring/otel-collector.yaml +++ b/kubernetes/monitoring/otel-collector.yaml @@ -81,6 +81,14 @@ data: endpoint: 0.0.0.0:13133 service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 extensions: - health_check pipelines: From e7db2e9cda580ffd8e511d8a25b96443225aa249 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:34:52 +0900 Subject: [PATCH 130/147] =?UTF-8?q?OpenTelemetry=20=EC=B4=88=EA=B8=B0?= =?UTF-8?q?=ED=99=94=20=EC=A4=91=EB=B3=B5=20=EB=B0=A9=EC=A7=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_tracing.py | 11 +++++++++++ backend/utils/tests.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/backend/utils/observability_tracing.py b/backend/utils/observability_tracing.py index eab2c9fb..d5ae480e 100644 --- a/backend/utils/observability_tracing.py +++ b/backend/utils/observability_tracing.py @@ -4,6 +4,7 @@ from utils.shortcuts import get_env logger = logging.getLogger(__name__) +_OTEL_CONFIGURED = False class _NoOpSpan: @@ -45,8 +46,11 @@ def get_current_trace_context(): def configure_opentelemetry(service_name): + global _OTEL_CONFIGURED if get_env("OTEL_ENABLED", "0").lower() not in ("1", "true", "yes", "on"): return + if _OTEL_CONFIGURED: + return try: from opentelemetry import trace @@ -76,6 +80,12 @@ def configure_opentelemetry(service_name): }) provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(sampler_ratio))) endpoint = get_env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector.monitoring.svc.cluster.local:4317") + current_provider = trace.get_tracer_provider() + if type(current_provider).__name__ != "ProxyTracerProvider": + logger.info("OpenTelemetry tracer provider is already configured; skipping CodePlace setup") + _OTEL_CONFIGURED = True + return + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True))) trace.set_tracer_provider(provider) @@ -84,3 +94,4 @@ def configure_opentelemetry(service_name): Psycopg2Instrumentor().instrument() RedisInstrumentor().instrument() CeleryInstrumentor().instrument() + _OTEL_CONFIGURED = True diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 28626ddf..b381791f 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -17,6 +17,7 @@ increment_judge_server_task_snapshot, ) from utils.observability_metrics import CodePlaceCollector +from utils import observability_tracing from utils.observability_tracing import get_tracer from utils.testcase_cache import TestCaseCacheManager @@ -407,6 +408,9 @@ def test_client_error_report_bounds_metric_labels(self): class ObservabilityTracingTest(SimpleTestCase): + def tearDown(self): + observability_tracing._OTEL_CONFIGURED = False + def test_get_tracer_returns_noop_tracer_when_opentelemetry_is_unavailable(self): real_import = __import__ @@ -434,6 +438,19 @@ def fake_import(name, *args, **kwargs): with patch("builtins.__import__", side_effect=fake_import): self.assertEqual(get_current_trace_context(), {}) + def test_configure_opentelemetry_is_idempotent(self): + observability_tracing._OTEL_CONFIGURED = True + real_import = __import__ + + def fail_import(name, *args, **kwargs): + if name == "opentelemetry": + raise AssertionError("OpenTelemetry should not be imported again") + return real_import(name, *args, **kwargs) + + with patch("utils.observability_tracing.get_env", return_value="1"), \ + patch("builtins.__import__", side_effect=fail_import): + observability_tracing.configure_opentelemetry("codeplace-test") + class CodePlaceJsonFormatterTest(SimpleTestCase): From 2daad2eb1fc6d47d9081a8ba7f26a2e70266a548 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:38:37 +0900 Subject: [PATCH 131/147] =?UTF-8?q?=EB=85=B8=EB=93=9C=20=EC=95=95=EB=A0=A5?= =?UTF-8?q?=20=EC=95=8C=EB=A6=BC=20=EB=9D=BC=EB=B2=A8=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/prometheus-rules.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index b58929fd..6143df2e 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -1138,6 +1138,7 @@ spec: labels: severity: warning priority: P1 + namespace: monitoring annotations: summary: "Kubernetes node is under pressure" description: "A Kubernetes node has reported memory, disk, or PID pressure for 5 minutes." From bed6630d51a898fb162009bd13f37890b1066dd2 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:40:48 +0900 Subject: [PATCH 132/147] =?UTF-8?q?=EA=B3=B5=EA=B0=9C=20=EC=97=94=EB=93=9C?= =?UTF-8?q?=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=EC=95=8C=EB=A6=BC=20=EB=B2=94?= =?UTF-8?q?=EC=9C=84=20=EC=A0=9C=ED=95=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/monitoring/prometheus-rules.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 6143df2e..f7f00dec 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -332,7 +332,7 @@ spec: summary: "Public endpoint is down in dev" description: "A dev synthetic HTTPS probe has failed for 2 minutes." - alert: PublicEndpointLatencyHigh - expr: probe_duration_seconds{probe_type="public-http"} > 2 + expr: probe_duration_seconds{probe_type="public-http", namespace=~"code-place-(dev|prod)|monitoring"} > 2 for: 5m labels: severity: warning @@ -341,7 +341,7 @@ spec: summary: "Public frontend endpoint latency is high" description: "A synthetic HTTPS probe has taken more than 2 seconds for 5 minutes." - alert: PublicEndpointTLSExpiringSoon - expr: (probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) < 1209600 + expr: (probe_ssl_earliest_cert_expiry{probe_type="public-http", namespace=~"code-place-(dev|prod)|monitoring"} - time()) < 1209600 for: 1h labels: severity: warning From e4c5257e3eb25098b2782cf882f180739a7bd97c Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:47:37 +0900 Subject: [PATCH 133/147] =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EB=AC=B8?= =?UTF-8?q?=EC=84=9C=EC=99=80=20=EB=AF=B8=EC=82=AC=EC=9A=A9=20=EB=A9=94?= =?UTF-8?q?=ED=8A=B8=EB=A6=AD=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_metrics.py | 13 - backend/utils/tests.py | 8 +- .../observability-improvement-plan/_index.md | 326 +---- .../infra/observability-runbook/_index.md | 1181 ----------------- .../monitoring/alertmanager-config.yaml | 2 - 5 files changed, 44 insertions(+), 1486 deletions(-) delete mode 100644 docs/content/developer/infra/observability-runbook/_index.md diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 7db14e2e..08695947 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -127,11 +127,6 @@ def describe(self): "codeplace_redis_rejected_connections_total", "Total Redis rejected connections from INFO stats.", ) - yield GaugeMetricFamily( - "codeplace_celery_task_last_seen_age_seconds", - "Seconds since each Celery task status was last observed.", - labels=["task_name", "status"], - ) yield GaugeMetricFamily( "codeplace_celery_task_last_success_age_seconds", "Seconds since each Celery task last completed successfully.", @@ -252,11 +247,6 @@ def _celery_task_metrics(self): "Latest observed Celery task runtime grouped by task name and status.", labels=["task_name", "status"], ) - last_seen_age_metric = GaugeMetricFamily( - "codeplace_celery_task_last_seen_age_seconds", - "Seconds since each Celery task status was last observed.", - labels=["task_name", "status"], - ) last_success_age_metric = GaugeMetricFamily( "codeplace_celery_task_last_success_age_seconds", "Seconds since each Celery task last completed successfully.", @@ -275,7 +265,6 @@ def _celery_task_metrics(self): yield total_metric yield runtime_metric yield last_runtime_metric - yield last_seen_age_metric yield last_success_age_metric return else: @@ -313,14 +302,12 @@ def _celery_task_metrics(self): logger.warning("Skipping invalid Celery task last_seen_at metric: %s=%s", field, value) continue age_seconds = max(now_timestamp - seen_timestamp, 0) - last_seen_age_metric.add_metric([task_name, status], age_seconds) if status == "success": last_success_age_metric.add_metric([task_name], age_seconds) yield total_metric yield runtime_metric yield last_runtime_metric - yield last_seen_age_metric yield last_success_age_metric def _redis_health_metrics(self): diff --git a/backend/utils/tests.py b/backend/utils/tests.py index b381791f..df6f7011 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -158,12 +158,6 @@ def test_celery_task_metrics_are_collected_from_redis(self): bucket_samples = [sample for sample in runtime_samples if sample.name.endswith("_bucket")] self.assertTrue(any(sample.labels["le"] == "+Inf" and sample.value == 3 for sample in bucket_samples)) self.assertTrue(any(sample.name.endswith("_sum") and sample.value == 4.5 for sample in runtime_samples)) - last_seen_samples = samples["codeplace_celery_task_last_seen_age_seconds"] - self.assertTrue(any( - sample.labels == {"task_name": "judge.tasks.judge_task", "status": "success"} - and sample.value >= 29 - for sample in last_seen_samples - )) last_success_samples = samples["codeplace_celery_task_last_success_age_seconds"] self.assertEqual(last_success_samples[0].labels, {"task_name": "judge.tasks.judge_task"}) self.assertGreaterEqual(last_success_samples[0].value, 29) @@ -200,7 +194,7 @@ def test_celery_task_metrics_tolerate_malformed_redis_values(self): self.assertEqual(total_sample.value, 0) last_runtime_sample = samples["codeplace_celery_task_last_runtime_seconds"][0] self.assertEqual(last_runtime_sample.value, 0) - self.assertEqual(samples["codeplace_celery_task_last_seen_age_seconds"], []) + self.assertEqual(samples["codeplace_celery_task_last_success_age_seconds"], []) def test_redis_health_metrics_are_collected(self): redis_client = MagicMock() diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index edacc28d..d05d15c2 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -13,13 +13,12 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 관측 대상은 운영자가 장애 발생 후 1분 안에 알림을 받고, 5분 안에 원인 후보를 좁힐 수 있도록 다음 경로로 나눕니다. -- **User Path:** synthetic public HTTPS probe, Traefik/Ingress, frontend nginx, hub-auth OAuth helper, backend Django API. +- **User Path:** public HTTPS probe, Traefik/Ingress, frontend nginx, backend Django API. - **Submission/Judge Path:** submission 생성, Celery `judge_task`, Redis `waiting_queue`, judge-server heartbeat, judge-server `/judge` 호출, 결과 저장. -- **AI Hint Path:** backend AI hint API, vLLM OpenAI-compatible server, NVIDIA GPU/DCGM, vLLM HF cache PVC. - **Async Jobs:** celery-worker, celery-beat, scheduled tasks. - **Data Stores:** PostgreSQL/CNPG, Redis Sentinel. -- **Platform/Storage:** backend/frontend/celery/judge Pod, node, PVC, Longhorn volume/node/disk. -- **Client/Error:** frontend runtime error, backend exception, request ID 기반 로그, Sentry. +- **Platform:** backend/frontend/celery/judge Pod, node, PVC, monitoring stack. +- **Client/Error:** frontend runtime error, backend exception, request ID 기반 로그. ## 2. 구현 내용 @@ -27,14 +26,6 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니다. 이 엔드포인트는 ServiceMonitor가 cluster 내부에서 scrape하며 외부 Ingress에는 연결하지 않습니다. -공개 사용자 경로는 Blackbox exporter 기반 synthetic probe로 별도 확인합니다. 이 probe는 frontend, hub-auth, Grafana 공개 URL을 30초 간격으로 호출해 DNS, Traefik TLS termination, frontend nginx, OAuth helper, monitoring UI 기본 라우팅을 함께 검증합니다. - -- dev frontend: `https://k3s.code-place-dev.site/` -- prod frontend: `https://code.pusan.ac.kr/` -- dev hub-auth: `https://hub-auth-dev.code-place-dev.site/` -- prod hub-auth: `https://hub-auth.code-place-dev.site/` -- Grafana: `https://monitoring.code-place-dev.site/` - 추가된 CodePlace custom metrics는 다음과 같습니다. - `codeplace_http_requests_total{method,endpoint,status_code}` @@ -44,66 +35,18 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_judge_task_outcome_total{status,scope}` - `codeplace_waiting_queue_length` - `codeplace_celery_broker_queue_length` -- `codeplace_observability_collector_success{collector}` +- `codeplace_celery_task_total{task_name,status}` +- `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_judge_server_available{hostname}` - `codeplace_judge_server_last_heartbeat_age_seconds{hostname}` - `codeplace_judge_server_task_number{hostname}` -- `codeplace_celery_task_total{task_name,status}` -- `codeplace_celery_task_runtime_seconds{task_name}` -- `codeplace_celery_task_last_runtime_seconds{task_name,status}` -- `codeplace_celery_task_last_seen_age_seconds{task_name,status}` -- `codeplace_celery_task_last_success_age_seconds{task_name}` -- `codeplace_ai_hint_api_outcome_total{status,scope}` -- `codeplace_ai_hint_requests_total{status}` -- `codeplace_ai_hint_duration_seconds{status}` -- `codeplace_redis_connected_clients` -- `codeplace_redis_max_clients` -- `codeplace_redis_rejected_connections_total` - `codeplace_frontend_error_total{surface,error_type}` -운영 대시보드와 빠른 판단에는 다음 recording rule을 사용합니다. - -- `codeplace:api_request_rate5m` -- `codeplace:api_5xx_rate5m` -- `codeplace:api_5xx_ratio5m` -- `codeplace:api_p95_latency_seconds5m` -- `codeplace:public_endpoint_availability5m` -- `codeplace:judge_available_total` -- `codeplace:judge_waiting_queue_length` - -이 recording rule은 원본 metric을 대체하지 않습니다. 알림과 대시보드에서 반복되는 PromQL을 줄이고, user path와 judge path의 최근 상태를 한 화면에서 빠르게 읽기 위한 요약 계층입니다. - -prod AI 힌트용 vLLM은 OpenAI-compatible server의 `/metrics`를 ServiceMonitor로 scrape합니다. vLLM production metrics 기준으로 다음 항목을 봅니다. - -- `vllm:num_requests_running` -- `vllm:num_requests_waiting` -- `vllm:kv_cache_usage_perc` -- `vllm:e2e_request_latency_seconds` -- `vllm:request_queue_time_seconds` -- `vllm:time_to_first_token_seconds` -- `vllm:prompt_tokens` -- `vllm:generation_tokens` - -vLLM이 사용하는 GPU node에는 NVIDIA DCGM exporter를 DaemonSet으로 배포합니다. 현재는 vLLM node label인 `workload.code-place.ai/vllm=true`가 붙은 node만 대상으로 하며, Prometheus는 DCGM exporter의 `:9400/metrics`를 scrape합니다. - -- `DCGM_FI_DEV_GPU_UTIL` -- `DCGM_FI_DEV_FB_USED` -- `DCGM_FI_DEV_FB_FREE` -- `DCGM_FI_DEV_GPU_TEMP` -- `DCGM_FI_DEV_XID_ERRORS` -- `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` -- `DCGM_FI_PROF_DRAM_ACTIVE` - `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. ### Logs -backend는 `X-Request-ID`를 수용하고, 없으면 request ID를 생성한 뒤 응답 header로 반환합니다. Frontend axios 요청은 `X-Request-ID`를 생성해 backend로 전달하고, backend가 반환한 request ID를 axios response/error와 Sentry request context에 보존합니다. API 요청에서 Celery task를 enqueue할 때 request ID를 Celery header로 전파하고, worker 실행 중 같은 request ID를 JSON log context에 복원합니다. - -frontend runtime error는 두 경로로 수집합니다. - -- Sentry가 build-time `SENTRY_DSN`으로 활성화된 경우 Sentry event로 전송합니다. -- Sentry 설정이 빠져도 Vue error, `window.error`, `unhandledrejection`은 `/api/client_error`로 sanitized report를 보내고, backend가 `frontend.error` JSON log와 `codeplace_frontend_error_total{surface,error_type}` metric을 남깁니다. +backend는 `X-Request-ID`를 수용하고, 없으면 request ID를 생성한 뒤 응답 header로 반환합니다. Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설정합니다. JSON 로그 필드는 다음을 기준으로 합니다. @@ -120,108 +63,45 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 - `remote_addr` - `task_id` - `submission_id` -- `task_name` -- `task_status` -- `trace_id` -- `span_id` -- `frontend_surface` -- `frontend_error_type` -- `frontend_message` -- `frontend_route` -- `frontend_release` -민감 정보는 로그에 싣지 않습니다. token, password, cookie, OAuth secret, authorization header, 제출 source code는 저장 대상이 아닙니다. OpenTelemetry가 활성화된 요청/작업 로그에는 현재 span의 `trace_id`, `span_id`를 추가해 Tempo trace와 Loki JSON log를 연결합니다. +민감 정보는 로그에 싣지 않습니다. token, password, cookie, OAuth secret, authorization header, 제출 source code는 저장 대상이 아닙니다. -Traefik은 JSON access log를 출력하고, `X-Request-ID`, `X-Forwarded-For` header만 keep합니다. Alloy는 `kube-system` 전체가 아니라 Traefik Pod 로그만 추가 수집합니다. +frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. -frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. access log는 Loki에서 파싱 가능한 JSON 형식이며, `request_id`, `method`, `path`, `status_code`, `duration_seconds`, `remote_addr`, `user_agent`, `x_forwarded_for`를 포함합니다. nginx는 들어온 `X-Request-ID`를 유지하고, 없으면 nginx `$request_id`를 생성해 응답 header와 backend proxy header로 전달합니다. +Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 조건에서는 Loki SingleBinary + Longhorn PVC를 단기 보관 기준선으로 둡니다. -Kubernetes Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 운영 조건에서는 Loki를 SingleBinary mode로 배포하고 Longhorn PVC 기반 filesystem storage를 사용합니다. - -- Loki release: `loki`, namespace: `monitoring`. -- Alloy release: `alloy`, namespace: `monitoring`. -- Loki storage: `filesystem` on Longhorn PVC. -- Loki retention: `code-place-dev` 3일, `code-place-prod` 7일. -- 수집 대상: `code-place-dev`, `code-place-prod`, `monitoring`, `kube-system`의 Traefik Pod. -- Grafana datasource: 기존 kube-prometheus-stack Grafana에 `Loki` datasource를 추가합니다. - -이 구성은 클라우드 없는 단기 retention 기준선입니다. Longhorn PVC는 hostPath/emptyDir보다 낫습니다. Kubernetes에서 lifecycle을 관리할 수 있고, PVC 사용률과 Longhorn volume health를 운영자가 볼 수 있기 때문입니다. 다만 durable log archive는 아니므로 장애 분석용 단기 저장소로만 취급합니다. - -현재 유지해야 하는 로그 저장소 불변 조건은 다음과 같습니다. - -- Loki deployment mode는 `SingleBinary`입니다. -- Loki storage type은 `filesystem`입니다. -- Loki PVC storageClass는 `longhorn`, size는 `50Gi`입니다. -- Loki retention은 dev 72h, prod 168h입니다. -- Grafana Loki chart의 내장 MinIO subchart는 사용하지 않습니다. - -prod 로그량이 커지면 먼저 noisy log를 줄이거나 retention을 줄입니다. 그래도 부족하면 Longhorn replica 상태와 node 여유 disk를 확인한 뒤 PVC를 증설합니다. 로그 저장소 장애 도메인을 더 분리해야 하면, Loki는 내부에서 별도 운영하는 S3 호환 object storage, 예를 들어 독립 MinIO cluster, 로 이전합니다. - -Longhorn 자체도 kube-prometheus-stack에서 scrape합니다. `longhorn-system` namespace의 `longhorn-manager` Service를 ServiceMonitor로 연결하고, Grafana `CodePlace Storage` dashboard에서 다음 항목을 봅니다. - -- Longhorn manager ready count. -- faulted/degraded/read-only volume count. -- volume robustness와 actual/capacity size. -- Longhorn node/disk 사용률. -- Longhorn manager CPU/memory. +- dev log retention: 3일. +- prod log retention: 7일. +- 수집 대상: `code-place-dev`, `code-place-prod`, `monitoring`, Traefik Pod. ### Tracing -OpenTelemetry는 앱 초기화 코드에 내장되어 있고, base/prod 기본값은 `OTEL_ENABLED=0`입니다. dev overlay에서만 `OTEL_ENABLED=1`로 켜서 collector/Tempo/sampling을 먼저 확인합니다. +OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OTEL_ENABLED=0`입니다. dev/staging에서 먼저 활성화하고 collector/Tempo/sampling 확인 후 prod 활성화를 결정합니다. 자동 계측 대상은 Django, requests, psycopg2, Redis, Celery입니다. 제출/채점 경로에는 manual span을 추가합니다. - `judge_task` - `submission.judge` -- `judge_server.select` - `judge_server.request` -- `submission.result_save` 기본 sampling 값은 `OTEL_TRACES_SAMPLER_ARG=0.05`입니다. -현재 Kubernetes manifest에는 backend/celery의 `OTEL_EXPORTER_OTLP_ENDPOINT`가 `http://otel-collector.monitoring.svc.cluster.local:4317`로 잡혀 있습니다. monitoring namespace에는 OTel Collector와 Tempo 단일 binary를 배포합니다. - -- OTel Collector release: raw Kubernetes Deployment, namespace `monitoring`. -- Tempo deployment mode: single binary, namespace `monitoring`. -- Tempo storage: filesystem on Longhorn PVC. -- Tempo PVC: `tempo-data`, 20Gi. -- Tempo retention: 72h. -- Grafana datasource: 기존 kube-prometheus-stack Grafana에 `Tempo` datasource를 추가합니다. - -prod tracing은 dev에서 trace ingest, query, traces-to-logs 동작과 Collector/Tempo 리소스 사용량을 확인한 뒤 별도 overlay에서 `OTEL_ENABLED=1`로 승격합니다. - ## 3. Kubernetes Monitoring 신규 monitoring 리소스는 `kubernetes/monitoring` 아래에 두고, 애플리케이션 `dev/prod` overlay에 포함하지 않습니다. -- `backend-service-monitor.yaml`: `code-place-dev`와 `code-place-prod` backend `/metrics` scrape, interval 15s. -- `traefik-service-monitor.yaml`: kube-system Traefik Prometheus metrics scrape, interval 15s. `Ingress5xxSpike`의 `traefik_service_requests_total` 원천입니다. -- `blackbox-exporter.yaml`: public endpoint synthetic probe용 Blackbox exporter. -- `public-endpoint-probes.yaml`: dev/prod frontend, hub-auth, Grafana 공개 HTTPS URL Probe, interval 30s. -- `prometheus-rules.yaml`: P0/P1 fast alert rules와 운영 요약 recording rules. +- `backend-service-monitor.yaml`: backend `/metrics` scrape, interval 15s. +- `blackbox-exporter.yaml`, `public-endpoint-probes.yaml`: frontend/hub-auth/Grafana 공개 HTTPS probe. +- `datastore-pod-monitors.yaml`: CNPG/Redis exporter scrape. +- `logs/loki-values.yaml`, `logs/alloy-values.yaml`: Loki/Alloy Helm values. +- `prometheus-rules.yaml`: P0/P1 fast alert rules와 일부 recording rules. - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. -- `grafana-dashboard-ai-inference.yaml`: prod vLLM readiness, queue/cache/latency/token throughput, HF cache PVC dashboard. -- `grafana-dashboard-kubernetes-events.yaml`: image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable 상태와 Kubernetes Warning event log dashboard. -- `grafana-dashboard-monitoring-stack.yaml`: Prometheus, Alertmanager, Grafana, Prometheus Operator readiness와 rule/notification 상태 dashboard. -- `grafana-dashboard-public-endpoints.yaml`: 공개 URL availability, HTTP status, latency, TLS expiry dashboard. -- `kubernetes-event-exporter.yaml`: Kubernetes Warning event를 stdout JSON으로 export해서 Alloy/Loki가 수집하도록 하는 exporter, ServiceMonitor, 최소 event read RBAC. -- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, Prometheus 2 replicas, Prometheus replica external label 제거, Alertmanager 2 replicas, Grafana metrics, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. -- `logs/loki-values.yaml`: Loki SingleBinary, Longhorn PVC, dev/prod retention, Loki gateway 2 replicas 설정. -- `logs/alloy-values.yaml`: Alloy DaemonSet 기반 Kubernetes Pod log collection 설정. -- `grafana-dashboard-logs.yaml`: Loki/Alloy health, Loki PVC, Loki ingest/error/canary, Alloy write backpressure, 최근 backend error, judge/celery log 조회 dashboard. -- `otel-collector.yaml`: backend/celery OTLP trace를 받아 Tempo로 전송하는 OpenTelemetry Collector. -- `tempo.yaml`: Tempo single binary, Longhorn PVC 기반 trace 저장소. -- `grafana-dashboard-traces.yaml`: Collector/Tempo readiness, span ingest/export, Tempo PVC usage dashboard. -- `longhorn-service-monitor.yaml`: Longhorn manager `/metrics` scrape, interval 30s. -- `grafana-dashboard-storage.yaml`: Longhorn volume/node/disk 상태와 storage capacity dashboard. -- `vllm-service-monitor.yaml`: prod vLLM `/metrics` scrape, interval 30s. -- `dcgm-exporter.yaml`: vLLM GPU node용 NVIDIA DCGM exporter DaemonSet/Service/ServiceMonitor. +- `grafana-dashboard-logs.yaml`: Loki/Alloy 상태와 주요 앱 로그 조회 dashboard. +- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. -`alertmanager-contact-points` Secret은 repo에 평문으로 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자는 SealedSecrets로 `alertmanager-contact-points` Secret을 생성합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. - -Email fallback은 SMTP host/from/to/auth가 필요하므로 기본 kustomization에는 포함하지 않습니다. SMTP 값이 정해지면 `kubernetes/monitoring/alertmanager-config-email.example.yaml`을 실제 값으로 복사하거나 패치해서 적용합니다. Prometheus Operator의 `AlertmanagerConfig`는 receiver에 `emailConfigs`를 지원하며, SMTP password는 같은 namespace의 Secret key로 참조합니다. +`alertmanager-contact-points` Secret은 repo에 평문으로 저장하지 않습니다. 운영자는 SealedSecret으로 `monitoring` namespace에 생성합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. ## 4. 빠른 알림 정책 @@ -229,161 +109,49 @@ Email fallback은 SMTP host/from/to/auth가 필요하므로 기본 kustomization P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합니다. -- `BackendTargetDown`: backend scrape 실패 또는 target series missing 1분 지속. +- `BackendTargetDown`: backend scrape 실패 1분 지속. - `Api5xxSpike`: backend 5xx 비율 5% 초과 2분 지속. - `Ingress5xxSpike`: Traefik 5xx 비율 5% 초과 2분 지속. -- `PublicEndpointDown`: prod 공개 HTTPS endpoint probe 실패 1분 지속. frontend와 hub-auth를 포함합니다. -- `PublicEndpointTLSCritical`: prod 공개 HTTPS 인증서 만료 3일 이내. - `JudgeAllUnavailable`: 사용 가능한 judge-server 0개 1분 지속. - `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. -- `PostgresUnavailable`: PostgreSQL Pod not ready 또는 readiness series missing 1분 지속. -- `RedisUnavailable`: Redis Pod not ready 또는 readiness series missing 1분 지속. -- `PostgresExporterDown`: CNPG PostgreSQL exporter metric 수집 실패 2분 지속. -- `RedisExporterDown`: Redis exporter metric 수집 실패 2분 지속. -- `LokiUnavailable`: Loki single-binary Pod not ready 1분 지속. -- `LonghornManagerDown`: Longhorn manager 전부 not ready 2분 지속. -- `LonghornVolumeFaulted`: Longhorn volume faulted 1분 지속. -- `LonghornVolumeReadOnly`: Longhorn volume filesystem read-only 1분 지속. -- `VLLMUnavailable`: prod vLLM `/metrics` scrape 실패 2분 지속. -- `GPUXIDError`: NVIDIA GPU XID error 1분 지속. -- `PrometheusUnavailable`: Prometheus Pod not ready 1분 지속. -- `AlertmanagerUnavailable`: Alertmanager Pod not ready 1분 지속. -- `AlertmanagerWebhookSecretMissing`: `alertmanager-contact-points` Secret missing 1분 지속. Discord 알림 자체가 막힐 수 있으므로 Alertmanager/Grafana UI에서 즉시 식별하기 위한 self-alert입니다. +- `PostgresUnavailable`: PostgreSQL Pod not ready 또는 exporter target down 1분 지속. +- `RedisUnavailable`: Redis Pod not ready 또는 exporter target down 1분 지속. +- `LokiUnavailable`: Loki Pod not ready 1분 지속. ### P1 -P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev` namespace의 P1은 외부 Discord로 보내지 않고 Alertmanager/Grafana UI에만 남깁니다. `code-place-prod`, `monitoring`, `longhorn-system` 등 운영 영향이 있는 P1은 Discord로 보냅니다. +P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `ApiLatencyHigh`: p95 latency 2초 초과 5분 지속. -- `PublicEndpointDown`: dev 공개 HTTPS endpoint probe 실패 2분 지속. frontend와 hub-auth를 포함합니다. -- `PublicEndpointLatencyHigh`: 공개 HTTPS probe duration 2초 초과 5분 지속. -- `PublicEndpointTLSExpiringSoon`: 공개 HTTPS 인증서 만료 14일 이내. -- `FrontendRuntimeErrorsSpike`: browser runtime error report 10개 초과/5분, 5분 지속. -- `BlackboxExporterUnavailable`: synthetic probe exporter Pod not ready 2분 지속. -- `GrafanaPublicEndpointDown`: Grafana 공개 HTTPS endpoint probe 실패 2분 지속. -- `KubernetesEventExporterUnavailable`: Kubernetes event exporter Pod not ready 2분 지속. -- `GrafanaUnavailable`: Grafana Pod not ready 2분 지속. -- `GrafanaMetricsTargetDown`: Grafana `/metrics` scrape 실패 또는 target series missing 2분 지속. -- `MonitoringServiceMetricsTargetDown`: monitoring namespace 주요 Service metrics target scrape 실패 또는 target series missing 2분 지속. -- `PrometheusOperatorUnavailable`: Prometheus Operator Pod not ready 2분 지속. -- `PrometheusRuleEvaluationFailures`: Prometheus rule evaluation failure 5분 지속. -- `PrometheusConfigReloadFailed`: Prometheus generated config reload 실패 또는 reload metric 누락 2분 지속. -- `PrometheusAlertmanagerDiscoveryFailed`: Prometheus가 Alertmanager target을 0개 discovery 2분 지속. -- `PrometheusReplicaUnavailable`: Prometheus ready replica 2개 미만 5분 지속. -- `AlertmanagerReplicaUnavailable`: Alertmanager ready replica 2개 미만 5분 지속. -- `AlertmanagerConfigReloadFailed`: Alertmanager generated config reload 실패 또는 reload metric 누락 2분 지속. -- `AlertmanagerNotificationFailures`: Alertmanager notification delivery failure 2분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. -- `CeleryBrokerQueueBacklog`: Redis broker default queue 20 초과 5분 지속. -- `CodePlaceCollectorFailed`: backend custom metrics collector 실패 5분 지속. -- `SubmissionCreateSystemFailures`: 제출 생성 API에서 DB 또는 judge enqueue 실패 발생 2분 지속. -- `JudgeTaskFailures`: `judge_task`에서 submission/user lookup 또는 dispatch 실패 발생 2분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. -- `CeleryBeatDown`: beat Deployment available replica 0개 또는 availability metric 누락 2분 지속. -- `CeleryTaskFailures`: task failure 발생 2분 지속. -- `CeleryTaskRetries`: task retry 3회 초과/10분, 5분 지속. -- `CeleryTaskRuntimeHigh`: task p95 runtime 120초 초과 10분 지속. -- `CeleryMinuteScheduledTaskStale`: 1분 주기 점수 변동 작업 성공 age 10분 초과. -- `CeleryDailyScheduledTaskStale`: 일간 점수 기준 작업 성공 age 26시간 초과. -- `CeleryWeeklyScheduledTaskStale`: 주간 문제 통계/보너스 문제 작업 성공 age 8일 초과. +- `CeleryBeatDown`: beat Pod not ready 2분 지속. - `PodCrashLooping`: 주요 Pod restart 증가 5분 지속. -- `CodePlacePodNotReady`: 주요 앱/DB/Redis Pod not ready 5분 지속. -- `KubernetesPodImagePullBackOff`: image pull 실패 2분 지속. -- `KubernetesPodCrashLoopBackOff`: CrashLoopBackOff 5분 지속. -- `KubernetesPodOOMKilled`: OOMKilled 종료가 최근 10분 내 발생하고 2분 지속. -- `KubernetesPodUnschedulable`: Pending 또는 Unschedulable 상태 10분 지속. -- `CodePlaceDeploymentUnavailable`: 주요 Deployment unavailable replica 5분 지속. -- `CodePlaceDeploymentRolloutStuck`: Deployment observed generation 미반영 10분 지속. -- `CodePlaceServiceNoReadyEndpoints`: 주요 Service ready endpoint 0개 또는 endpoint availability metric 누락 2분 지속. -- `VLLMServiceNoReadyEndpoints`: prod vLLM Service ready endpoint 0개 또는 endpoint availability metric 누락 2분 지속. -- `CodePlaceContainerCPUHigh`: 주요 컨테이너 CPU limit 사용률 80% 초과 10분 지속. -- `CodePlaceContainerMemoryHigh`: 주요 컨테이너 memory limit 사용률 85% 초과 10분 지속. -- `PostgresCollectorError`: CNPG PostgreSQL metrics collection error 5분 지속. -- `PostgresHADegraded`: PostgreSQL instance가 3개 node에 분산되지 않음 10분 지속. -- `RedisMemoryHigh`: Redis memory 사용률 85% 초과 10분 지속. -- `RedisConnectedClientsHigh`: backend 기준 Redis connected_clients/maxclients 80% 초과 10분 지속. -- `RedisRejectedConnections`: Redis rejected connection 증가 2분 지속. - `PVCAlmostFull`: PVC 사용률 85% 초과 10분 지속. -- `LokiPVCAlmostFull`: Loki Longhorn PVC 사용률 85% 초과 10분 지속. -- `AlloyDaemonSetUnavailable`: Alloy log collector가 모든 node에서 available하지 않음 5분 지속. -- `LokiGatewayUnavailable`: Loki gateway Pod not ready 2분 지속. -- `LokiGatewayReplicaUnavailable`: Loki gateway ready replica 2개 미만 5분 지속. -- `LokiIngestionStalled`: Loki log line 수신량 0이 10분 지속. -- `LokiRequestErrors`: Loki 5xx response 발생 5분 지속. -- `LokiCanaryMissingEntries`: Loki canary write/readback 누락 발생 5분 지속. -- `AlloyLokiWriteDrops`: Alloy가 Loki write 전 log entry drop 2분 지속. -- `AlloyLokiWriteRetries`: Alloy Loki write batch retry 5분 지속. -- `OTelCollectorUnavailable`: OpenTelemetry Collector Pod not ready 2분 지속. -- `OTelCollectorRefusedSpans`: Collector span 수신 거부 발생 2분 지속. -- `OTelCollectorExportFailures`: Collector span export 실패 발생 2분 지속. -- `TempoUnavailable`: Tempo Pod not ready 2분 지속. -- `TempoPVCAlmostFull`: Tempo PVC 사용률 85% 초과 10분. -- `LonghornVolumeDegraded`: Longhorn volume degraded 5분 지속. -- `LonghornNodeNotReady`: Longhorn node not ready 5분 지속. -- `LonghornDiskNotReady`: Longhorn disk not ready 5분 지속. -- `LonghornNodeStorageHigh`: Longhorn node storage 사용/예약률 85% 초과 10분. -- `LonghornDiskUsageHigh`: Longhorn disk 사용/예약률 85% 초과 10분. -- `VLLMWaitingQueueHigh`: prod vLLM waiting request 5 초과 5분 지속. -- `VLLMKVCacheHigh`: prod vLLM KV cache 사용률 90% 초과 10분 지속. -- `VLLMRequestLatencyHigh`: prod vLLM p95 e2e latency 60초 초과 10분 지속. -- `AIHintBackendErrors`: backend AI hint stream request/parse/internal error 발생 2분 지속. -- `AIHintBackendLatencyHigh`: backend AI hint successful stream p95 duration 120초 초과 10분 지속. -- `AIHintAPIFailures`: AI hint 사용자-facing API에서 LLM error, unexpected error, empty response 발생 2분 지속. -- `DCGMExporterUnavailable`: DCGM exporter DaemonSet unavailable 2분 지속. -- `GPUUtilizationHigh`: GPU utilization 95% 초과 15분 지속. -- `GPUMemoryHigh`: GPU framebuffer memory usage 90% 초과 10분 지속. -- `GPUTemperatureHigh`: GPU temperature 85C 초과 10분 지속. -- `CodePlaceNodePressure`: node memory/disk/PID pressure 5분 지속. +- `CodePlaceCollectorFailed`: backend custom metric collector 실패 5분 지속. +- `FrontendRuntimeErrorsSpike`: frontend runtime error report 증가 5분 지속. ## 5. 운영 확인 절차 -1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 Prometheus 2 replicas, Alertmanager 2 replicas, `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. - - `helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --create-namespace --version 86.3.1 --values kubernetes/monitoring/kube-prometheus-stack-values.yaml` - - values는 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None`을 설정합니다. AlertmanagerConfig는 `monitoring` namespace에 있지만 CodePlace alert의 `namespace` label은 `code-place-dev`/`code-place-prod`이므로, 기본 `OnNamespace` matcher를 쓰면 외부 namespace alert가 Discord receiver까지 도달하지 않을 수 있습니다. -2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 SealedSecret으로 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. - Email fallback을 켤 경우 SMTP 정보를 별도로 정한 뒤 `alertmanager-email` Secret과 email AlertmanagerConfig를 추가 적용합니다. -3. Loki와 Alloy를 설치합니다. - - `helm upgrade --install loki grafana/loki --namespace monitoring --create-namespace --version 6.55.0 --values kubernetes/monitoring/logs/loki-values.yaml` - - `helm upgrade --install alloy grafana/alloy --namespace monitoring --create-namespace --version 1.10.0 --values kubernetes/monitoring/logs/alloy-values.yaml` -4. 애플리케이션은 환경별 overlay로 적용합니다. +1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. +2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. +3. 애플리케이션은 환경별 overlay로 적용합니다. - dev: `kubectl apply -k kubernetes/overlays/dev` - prod: `kubectl apply -k kubernetes/overlays/prod` -5. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. +4. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` -6. Prometheus target에서 `backend`, `longhorn`, `vllm`, `blackbox-exporter`, `kubernetes-event-exporter`, `codeplace-public-http-dev`, `codeplace-public-http-prod`, `codeplace-grafana-http`가 healthy인지 확인합니다. -7. Grafana의 `CodePlace Public Endpoints` dashboard에서 prod/dev/Grafana availability, HTTP status, latency, TLS expiry panel이 비어 있지 않은지 확인합니다. -8. Grafana의 `CodePlace Overview` dashboard에서 recording rule 기반 request rate, 5xx ratio, p95 latency, judge availability, waiting queue와 frontend runtime error, submission/judge outcome, judge heartbeat, Celery task throughput/runtime, Pod readiness/restart, CPU/memory, PVC, Deployment unavailable/rollout, Service ready endpoint, PostgreSQL CNPG collector/HA 상태, Redis readiness/memory/client panel을 확인합니다. -9. Grafana의 `CodePlace Logs` dashboard에서 Loki ready, Alloy node coverage, Loki PVC usage, ingress/frontend 4xx/5xx, 최근 backend error, frontend runtime error, judge/celery log panel을 확인합니다. -10. Grafana의 `CodePlace Logs` dashboard에서 `request_id` 변수에 실제 응답 header 또는 JSON log의 request ID를 넣고 해당 요청 로그가 좁혀지는지 확인합니다. -11. Grafana의 `CodePlace Kubernetes Events` dashboard에서 event exporter ready, image pull, CrashLoopBackOff, OOMKilled, Pending/Unschedulable, Kubernetes Warning event panel을 확인합니다. -12. Grafana의 `CodePlace Monitoring Stack` dashboard에서 Prometheus, Alertmanager, Grafana, operator readiness, target scrape failure, rule evaluation failure, notification failure panel을 확인합니다. -13. Grafana의 `CodePlace Traces` dashboard에서 Tempo ready, OTel Collector ready, trace receive/export rate, Tempo PVC usage를 확인합니다. -14. Grafana의 `CodePlace Storage` dashboard에서 Longhorn manager, volume robustness, node/disk usage panel이 비어 있지 않은지 확인합니다. -15. Grafana의 `CodePlace AI Inference` dashboard에서 vLLM scrape, pod ready, waiting/running requests, KV cache, p95 latency, token throughput, HF cache PVC, AI hint API outcome, backend AI hint stream status/duration, GPU utilization/memory/temperature/XID panel이 비어 있지 않은지 확인합니다. -16. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -17. Grafana Explore에서 `Loki` datasource를 선택하고 `{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"}` 쿼리가 Kubernetes Warning event를 반환하는지 확인합니다. -18. Grafana Explore에서 `Tempo` datasource를 선택하고 `service.name=codeplace-backend` 또는 `service.name=codeplace-celery` trace가 조회되는지 확인합니다. -19. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +5. Prometheus target에서 `backend`, `postgres`, `redis`, `blackbox-exporter`, `loki`, `alloy` target이 healthy인지 확인합니다. +6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission/judge outcome, waiting queue, judge heartbeat panel을 확인합니다. +7. Grafana의 `CodePlace Logs` dashboard 또는 Explore에서 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +8. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. - `/metrics`는 cluster 내부 scrape 전용이며 외부 Ingress에 노출하지 않습니다. -- 공개 URL synthetic probe는 monitoring namespace 내부에서 외부 도메인으로 나가는 HTTPS 요청입니다. DNS, outbound network, TLS, Traefik/frontend/hub-auth 경로를 함께 검증하지만 브라우저 JS runtime error를 대체하지는 않습니다. -- Discord webhook URL은 Git에 평문으로 저장하지 않습니다. 운영 Secret은 SealedSecret으로 관리합니다. +- Discord webhook URL은 Git에 저장하지 않습니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. -- OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지하고 dev overlay에서 먼저 활성화합니다. -- P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. root fallback은 `unmatched-muted`로 두어 CodePlace priority label이 없는 기본/외부 alert가 Discord로 새지 않게 합니다. dev P1은 `dev-p1-muted` receiver로 보내 외부 전송을 막고, prod/monitoring P1은 Discord receiver로 보냅니다. -- Metrics backend는 kube-prometheus-stack의 Prometheus입니다. Mimir는 현재 온프렘 단기 운영 기준선에 포함하지 않습니다. -- 로그 수집은 Grafana Alloy와 Loki로 관리하며 Promtail은 신규 도입하지 않습니다. -- Kubernetes event는 Kubernetes Event Exporter가 Warning event만 stdout JSON으로 내보내고, Alloy가 일반 Pod log와 같은 경로로 Loki에 적재합니다. -- PostgreSQL은 backend `/metrics`에서 직접 조회하지 않고 CNPG instance exporter와 PodMonitor의 `cnpg_collector_*` 지표를 수집합니다. Redis는 Opstree Redis exporter sidecar와 PodMonitor의 `redis_*` 지표를 기본으로 보고, backend custom collector는 queue length, judge-server heartbeat snapshot, client/rejection 같은 O(1) Redis 상태만 수집합니다. -- Sentry backend SDK는 기본 PII 자동 전송을 비활성화하고 전송 직전 `authorization`, `cookie`, `password`, `token`, `secret`, 제출 source code 계열 필드를 redaction합니다. 사용자 영향 분석은 Sentry event와 request_id 기반 backend JSON log를 함께 사용합니다. -- Frontend는 axios 요청마다 `X-Request-ID`를 전파하고 마지막 request ID를 `window.__CODEPLACE_LAST_REQUEST_ID__`와 Sentry request context 및 `/api/client_error` report에 저장합니다. -- Frontend Sentry 설정은 빌드 시점 값입니다. `APP_VERSION`, `SENTRY_ENVIRONMENT`, `SENTRY_DSN`, `USE_SENTRY`는 frontend image build-arg로 주입하며, Kubernetes Deployment의 런타임 env만 변경해도 이미 빌드된 JS bundle에는 반영되지 않습니다. DSN이 없으면 Sentry는 비활성화되지만 `/api/client_error` 기반 metric/log는 계속 동작합니다. -- Backend/Celery Sentry release는 `SENTRY_RELEASE` env가 있으면 사용합니다. 이미지 태그와 동일한 commit SHA를 넣어 Grafana/Loki/Sentry 간 배포 기준을 맞추는 것을 권장합니다. -- Tempo tracing은 monitoring namespace에 배포하지만, prod app trace export는 dev 확인 후 별도 승격합니다. -- vLLM은 prod overlay에만 포함되어 있으므로 vLLM ServiceMonitor와 AI Inference dashboard도 `code-place-prod`를 기준으로 조회합니다. dev에 vLLM을 올리면 별도 dev ServiceMonitor/Probe 또는 namespace selector 확장이 필요합니다. -- DCGM exporter는 `workload.code-place.ai/vllm=true` node label을 기준으로 배포합니다. GPU node가 늘어나거나 vLLM 외 GPU workload를 운영하면 selector와 dashboard scope를 확장해야 합니다. +- OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지합니다. Collector/Tempo 리소스는 포함하지만 dev에서 먼저 확인한 뒤 prod 활성화를 결정합니다. +- P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. ## 6. 검증 @@ -394,26 +162,18 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. `code-place-dev - Metrics endpoint: Django test client 기준 `/metrics` 200 및 `codeplace_*` metric 포함 확인 - Kubernetes app render: `kubectl kustomize kubernetes/overlays/dev`, `kubectl kustomize kubernetes/overlays/prod` - Kubernetes monitoring render: `kubectl kustomize kubernetes/monitoring` -- Logs values YAML parse: `kubernetes/monitoring/logs/loki-values.yaml`, `kubernetes/monitoring/logs/alloy-values.yaml` - Grafana dashboard JSON parse: `grafana-dashboard-codeplace.yaml`, `grafana-dashboard-logs.yaml` -- Prometheus rule syntax: `promtool check rules`에 `PrometheusRule.spec.groups`를 추출해서 확인 -- Helm render: kube-prometheus-stack, Loki, Alloy chart를 pinned version과 values로 렌더링 -- Live cluster check: CRD, monitoring namespace resource, dashboard ConfigMap, webhook Secret, kube-prometheus-stack Pod readiness, Loki/Alloy, app namespace backend service/port/readiness를 `kubectl`로 확인 + +`promtool`이 있는 환경에서는 `PrometheusRule.spec.groups`를 추출해서 `promtool check rules`로 확인합니다. ### Discord Webhook Secret -Discord webhook URL은 repo에 평문으로 커밋하지 않습니다. SealedSecrets controller가 설치된 클러스터에서 운영자가 다음 흐름으로 SealedSecret을 생성합니다. +Discord webhook URL은 repo에 커밋하지 않습니다. SealedSecrets controller가 설치된 클러스터에서 생성합니다. ```sh -mkdir -p kubernetes/monitoring/secrets - kubectl -n monitoring create secret generic alertmanager-contact-points \ --from-literal=webhook-url="$ALERT_WEBHOOK_URL" \ --dry-run=client -o yaml \ | kubeseal --controller-namespace kube-system --format yaml \ > kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml - -kubectl apply -f kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml ``` - -SealedSecret 파일은 cluster 공개키로 암호화된 값만 포함합니다. controller namespace가 다르면 `--controller-namespace`를 실제 설치 namespace로 바꿉니다. diff --git a/docs/content/developer/infra/observability-runbook/_index.md b/docs/content/developer/infra/observability-runbook/_index.md deleted file mode 100644 index 4f34c56e..00000000 --- a/docs/content/developer/infra/observability-runbook/_index.md +++ /dev/null @@ -1,1181 +0,0 @@ ---- -date: 2026-06-19T00:00:00+09:00 -draft: false -title: "Observability Runbook" -weight: 6 ---- - -{{< callout >}} -이 문서는 Kubernetes 운영 환경에서 CodePlace 알림을 받았을 때 원인 후보를 빠르게 좁히기 위한 1차 runbook입니다. Docker Swarm monitoring은 레거시로 유지합니다. -{{< /callout >}} - -## 공통 확인 - -알림을 받으면 먼저 scope를 확인합니다. - -```sh -kubectl get ns -kubectl -n monitoring get prometheusrule,alertmanagerconfig,servicemonitor -kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|loki|alloy|otel|tempo|blackbox|event-exporter' -kubectl -n longhorn-system get pod,svc -``` - -Grafana에서는 `CodePlace Overview` dashboard에서 `namespace`를 알림의 namespace로 맞춥니다. -공개 URL 장애는 `CodePlace Public Endpoints` dashboard에서 먼저 봅니다. -로그 수집이나 로그 기반 확인이 필요하면 `CodePlace Logs` dashboard를 함께 봅니다. -Pod 배포/스케줄링/이미지 pull 실패는 `CodePlace Kubernetes Events` dashboard에서 Kubernetes Warning event와 kube-state-metrics 상태를 같이 봅니다. -Prometheus, Alertmanager, Grafana 자체 이상은 `CodePlace Monitoring Stack` dashboard에서 봅니다. -PVC/volume 문제가 의심되면 `CodePlace Storage` dashboard에서 Longhorn 상태를 먼저 확인합니다. - -Prometheus target 확인: - -```text -Status -> Targets -> backend -Status -> Targets -> postgres -Status -> Targets -> redis -Status -> Targets -> longhorn -Status -> Targets -> vllm -Status -> Targets -> dcgm-exporter -Status -> Targets -> kubernetes-event-exporter -Status -> Targets -> codeplace-public-http-dev -Status -> Targets -> codeplace-public-http-prod -``` - -Alertmanager 확인: - -```text -Alerts -> alertname / namespace / priority -Status -> Configuration -> p0-discord / p1-discord -``` - -Discord webhook은 SealedSecret으로 관리합니다. AlertmanagerConfig는 최종적으로 `monitoring` namespace의 `alertmanager-contact-points` Secret key `webhook-url`을 참조하므로, 알림이 오지 않으면 SealedSecret controller와 복호화된 Secret을 함께 확인합니다. -CodePlace AlertmanagerConfig의 root fallback receiver는 `unmatched-muted`입니다. `priority=P0` 또는 `priority=P1` label이 없는 기본/외부 alert는 이 route에서 Discord로 보내지 않습니다. - -```sh -kubectl get crd sealedsecrets.bitnami.com -kubectl get pod -A | grep -i sealed -kubectl -n monitoring get sealedsecret,secret | grep alertmanager-contact-points -kubectl -n monitoring get secret alertmanager-contact-points -o jsonpath='{.data.webhook-url}' | wc -c -kubectl -n monitoring get alertmanagerconfig codeplace-alert-routing -o yaml -kubectl -n monitoring logs alertmanager-kube-prometheus-stack-alertmanager-0 -c alertmanager --tail=100 -``` - -`kube-prometheus-stack-values.yaml`의 `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type`은 `None`이어야 합니다. 기본 `OnNamespace` 전략이면 `monitoring` namespace의 AlertmanagerConfig가 `namespace=code-place-dev` 또는 `namespace=code-place-prod` alert를 라우팅하지 못할 수 있습니다. - -Email fallback을 켠 환경에서는 `Status -> Configuration`에서 `p0-email` / `p1-email` receiver도 확인합니다. - -Monitoring stack 확인: - -```sh -kubectl -n monitoring get pod | grep -E 'prometheus|alertmanager|grafana|operator' -kubectl -n monitoring get configmap kube-prometheus-stack-grafana-datasource -o yaml -kubectl -n monitoring get prometheusrule codeplace-fast-alerts -o yaml -kubectl -n monitoring get alertmanagerconfig codeplace-alert-routing -o yaml -kubectl -n monitoring get alertmanager kube-prometheus-stack-alertmanager -o jsonpath='{.spec.alertmanagerConfigMatcherStrategy.type}' -``` - -```promql -kube_pod_status_ready{namespace="monitoring", pod=~"prometheus-kube-prometheus-stack-prometheus-.*|alertmanager-kube-prometheus-stack-alertmanager-.*|kube-prometheus-stack-grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} -prometheus_notifications_alertmanagers_discovered -increase(prometheus_rule_evaluation_failures_total[5m]) -increase(alertmanager_notifications_failed_total[5m]) -sum by (job, namespace) (up == 0) -``` - -Loki 로그 확인: - -```logql -{namespace=""} -{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} -{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} |~ "FailedScheduling|ImagePullBackOff|ErrImagePull|BackOff|OOMKilling|Unhealthy|FailedMount|FailedAttachVolume" -{namespace="kube-system", app_kubernetes_io_name="traefik"} | json -{namespace="", app="frontend"} | json -{namespace="", app="backend"} | json -{namespace="", app="backend"} | json | logger="frontend.error" -{namespace="", container=~"backend|hub-auth|celery-worker|judge-server|vllm"} -``` - -Loki/Alloy 상태 확인: - -```sh -kubectl -n monitoring get pod,pvc | grep -E 'loki|alloy' -kubectl -n monitoring logs deploy/loki-gateway --tail=100 -kubectl -n monitoring logs daemonset/alloy --tail=100 -``` - -Trace 상태 확인: - -```sh -kubectl -n monitoring get pod,svc,pvc | grep -E 'otel-collector|tempo' -kubectl -n monitoring logs deploy/otel-collector --tail=100 -kubectl -n monitoring logs deploy/tempo --tail=100 -kubectl -n code-place-dev get deploy backend celery-worker celery-beat -o yaml | grep -A1 OTEL_ENABLED -``` - -Grafana에서는 `CodePlace Traces` dashboard와 `Tempo` datasource Explore를 확인합니다. dev에서 먼저 `service.name=codeplace-backend`와 `service.name=codeplace-celery` trace가 조회되어야 합니다. -Tempo에서 trace를 선택한 뒤 같은 `trace_id`를 Loki에서 조회합니다. - -```logql -{namespace="", app="backend"} | json | trace_id="" -{namespace="", container=~"celery-worker|celery-beat"} | json | trace_id="" -``` - -공개 URL synthetic probe 확인: - -```sh -kubectl -n monitoring get pod,svc,probe | grep -E 'blackbox|codeplace-public' -kubectl -n monitoring logs deploy/blackbox-exporter --tail=100 -``` - -```promql -probe_success{probe_type="public-http"} -probe_http_status_code{probe_type="public-http"} -probe_duration_seconds{probe_type="public-http"} -(probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) / 86400 -``` - -probe 대상은 frontend와 hub-auth를 모두 포함합니다. `service` label로 어느 endpoint인지 구분합니다. - -Longhorn 상태 확인: - -```sh -kubectl -n longhorn-system get pod,svc -kubectl -n monitoring get servicemonitor longhorn -o yaml -``` - -```promql -longhorn_volume_robustness -longhorn_volume_file_system_read_only -(longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1) -(longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1) -``` - -vLLM 상태 확인: - -```sh -kubectl -n code-place-prod get pod,svc,pvc | grep vllm -kubectl -n monitoring get servicemonitor vllm -o yaml -``` - -```promql -up{job="vllm", namespace="code-place-prod"} -{__name__="vllm:num_requests_running", namespace="code-place-prod"} -{__name__="vllm:num_requests_waiting", namespace="code-place-prod"} -{__name__="vllm:kv_cache_usage_perc", namespace="code-place-prod"} -histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) -``` - -GPU 상태 확인: - -```sh -kubectl -n monitoring get ds,svc,servicemonitor dcgm-exporter -kubectl -n monitoring logs daemonset/dcgm-exporter --tail=100 -kubectl -n code-place-prod get pod -l app=vllm -o wide -``` - -```promql -DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"} -DCGM_FI_DEV_FB_USED{namespace="monitoring"} / clamp_min(DCGM_FI_DEV_FB_USED{namespace="monitoring"} + DCGM_FI_DEV_FB_FREE{namespace="monitoring"}, 1) -DCGM_FI_DEV_GPU_TEMP{namespace="monitoring"} -DCGM_FI_DEV_XID_ERRORS{namespace="monitoring"} -``` - -## P0 - -### BackendTargetDown - -증상: Prometheus가 backend `/metrics`를 1분 이상 scrape하지 못합니다. - -확인: - -```sh -kubectl -n get pod,svc,endpoints backend -kubectl -n describe pod -l app=backend -kubectl -n logs deploy/backend --tail=100 -kubectl -n monitoring get servicemonitor backend -o yaml -``` - -판단: - -- backend Pod가 없거나 not ready이면 배포/리소스 문제입니다. -- Service endpoint가 비어 있으면 label selector 또는 readiness 문제입니다. -- backend는 떠 있는데 `/metrics`만 실패하면 app 설정 또는 ServiceMonitor port/path 문제입니다. - -### Api5xxSpike - -증상: backend 5xx 비율이 2분 동안 5%를 넘습니다. - -확인: - -```promql -sum by (namespace, status_code) (rate(codeplace_http_requests_total{namespace=""}[5m])) -histogram_quantile(0.95, sum by (le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=""}[5m]))) -``` - -```sh -kubectl -n logs deploy/backend --tail=200 -kubectl -n get pod -l app=backend -kubectl -n describe pod -l app=backend -``` - -Grafana Explore: - -```logql -{namespace="", app="frontend"} | json | status_code >= 500 -{namespace="", app="backend"} | json | status_code >= 500 -{namespace="", app="backend"} |= "" -``` - -판단: - -- 5xx와 latency가 같이 오르면 backend/DB/Redis 병목 가능성이 큽니다. -- 5xx만 증가하면 최근 배포, exception, 외부 dependency를 우선 확인합니다. -- Sentry event가 있으면 `environment`, `release`, `request_id` tag를 기준으로 같은 시간대 Loki 로그와 대조합니다. -- Sentry event가 없더라도 `codeplace_frontend_error_total`과 `frontend.error` backend JSON log를 확인합니다. -- 사용자 브라우저에서 재현 중이면 `window.__CODEPLACE_LAST_REQUEST_ID__` 값을 확인해 같은 request ID로 Loki/backend JSON log를 조회합니다. -- Frontend Sentry 설정은 image build 시점에 결정됩니다. `SENTRY_DSN`, `USE_SENTRY`, `SENTRY_ENVIRONMENT`, `APP_VERSION` 변경 후에는 frontend image rebuild가 필요합니다. DSN이 빠져 있으면 Sentry는 꺼지지만 `/api/client_error` metric/log 수집은 계속 동작해야 합니다. - -### Ingress5xxSpike - -증상: Traefik service 기준 5xx 비율이 2분 동안 5%를 넘습니다. - -확인: - -```promql -codeplace:ingress_5xx_rate2m{namespace=""} -codeplace:ingress_request_rate2m{namespace=""} -sum by (exported_service, code) (rate(traefik_service_requests_total{exported_service=~"-.*"}[5m])) -sum by (service, code) (rate(traefik_service_requests_total{service=~"-.*"}[5m])) -``` - -```sh -kubectl -n get ingress,svc,endpoints -kubectl -n kube-system logs deploy/traefik --tail=200 -kubectl -n kube-system get helmchartconfig traefik -o yaml -``` - -Grafana Explore: - -```logql -{namespace="kube-system", app_kubernetes_io_name="traefik"} | json | DownstreamStatus >= 500 -{namespace="kube-system", app_kubernetes_io_name="traefik"} |= "" -``` - -판단: - -- backend 5xx도 같이 증가하면 app/backend 문제입니다. -- ingress 5xx만 증가하면 Traefik route, Service endpoint, upstream connection 문제를 의심합니다. - -### PublicEndpointDown / GrafanaPublicEndpointDown / PublicEndpointTLSCritical - -증상: 공개 HTTPS endpoint synthetic probe가 실패하거나 인증서 만료가 3일 이내입니다. - -확인: - -```promql -probe_success{probe_type="public-http"} -probe_http_status_code{probe_type="public-http"} -probe_http_duration_seconds{probe_type="public-http"} -(probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) / 86400 -``` - -```sh -kubectl -n monitoring get pod,svc,probe | grep -E 'blackbox|codeplace-public|hub-auth|grafana' -kubectl -n monitoring logs deploy/blackbox-exporter --tail=200 -kubectl -n get ingress,svc,endpoints frontend hub-auth -kubectl -n kube-system logs deploy/traefik --tail=200 -``` - -판단: - -- `probe_success=0`이고 ingress/backend 지표는 정상이면 DNS, TLS, Traefik public route, frontend nginx 기본 경로 문제를 먼저 봅니다. -- `service="hub-auth"` probe가 실패하면 OAuth helper Pod, Service, Ingress, GitHub OAuth secret 설정을 우선 확인합니다. -- `service="grafana"` probe가 실패하면 Grafana Pod readiness와 별개로 Grafana ingress, Traefik route, TLS resolver, public DNS를 확인합니다. -- `probe_http_status_code`가 0이면 DNS/TCP/TLS 연결 실패 가능성이 큽니다. -- 5xx이면 Traefik/frontend/backend 로그와 같은 시간대를 대조합니다. -- TLS 만료 알림은 Traefik ACME resolver와 인증서 저장소를 확인합니다. - -### JudgeAllUnavailable - -증상: 사용 가능한 judge-server가 0개입니다. - -확인: - -```promql -codeplace_judge_server_available{namespace=""} -codeplace_judge_server_last_heartbeat_age_seconds{namespace=""} -codeplace_waiting_queue_length{namespace=""} -``` - -```sh -kubectl -n get pod -l app=judge-server -kubectl -n logs deploy/judge-server --tail=200 -kubectl -n logs deploy/celery-worker --tail=200 -``` - -판단: - -- judge Pod가 not ready이면 platform/deploy 문제입니다. -- Pod는 ready인데 heartbeat가 stale이면 judge-server와 backend 통신 또는 token 문제입니다. -- waiting queue가 같이 증가하면 사용자 제출 영향이 있습니다. -- backend `/metrics`는 judge DB row를 직접 조회하지 않고 Redis heartbeat snapshot만 노출합니다. 의도적으로 retire한 judge는 disable/delete API로 snapshot을 제거합니다. - -### JudgeHeartbeatCritical - -증상: enabled judge-server heartbeat age가 180초를 넘습니다. - -확인: - -```promql -max by (namespace) (codeplace_judge_server_last_heartbeat_age_seconds{namespace=""}) -topk(10, codeplace_judge_server_task_number{namespace=""}) -``` - -```sh -kubectl -n get pod -l app=judge-server -o wide -kubectl -n describe pod -l app=judge-server -``` - -판단: - -- 특정 judge만 stale이면 해당 Pod/Node 문제입니다. -- 전체 judge가 stale이면 judge-server Pod, heartbeat endpoint, token 설정을 확인합니다. -- 24시간 이상 heartbeat가 없는 Redis snapshot은 retired 상태로 보고 collector가 제거합니다. 그 이후에도 전체 judge가 없으면 `JudgeAllUnavailable`이 계속 장애를 잡습니다. - -### PostgresUnavailable - -증상: PostgreSQL Pod가 1분 이상 ready가 아닙니다. - -확인: - -```sh -kubectl -n get pod -l cnpg.io/cluster=postgres -kubectl -n describe cluster postgres -kubectl -n describe pod -l cnpg.io/cluster=postgres -``` - -판단: - -- Pod scheduling 실패는 node/storage 문제입니다. -- Pod restart나 readiness 실패는 CNPG 상태와 PostgreSQL 로그를 확인합니다. - -### RedisUnavailable - -증상: Redis 또는 Sentinel Pod가 1분 이상 ready가 아닙니다. - -확인: - -```sh -kubectl -n get pod | grep redis -kubectl -n describe redissentinel redis -kubectl -n describe redisreplication redis-replication -kubectl -n logs -l app=redis-sentinel --tail=100 -``` - -판단: - -- Redis replication Pod 문제면 app cache/queue 영향이 큽니다. -- Sentinel 문제면 failover와 master discovery 영향이 큽니다. - -### PostgresExporterDown / RedisExporterDown - -증상: DB/Redis Pod 자체가 ready여도 exporter metric scrape가 실패합니다. - -확인: - -```sh -kubectl -n monitoring get podmonitor postgres redis -o yaml -kubectl -n get pod -l cnpg.io/cluster=postgres -kubectl -n get pod | grep redis -``` - -```promql -cnpg_collector_up{namespace="", cluster="postgres"} -redis_up{namespace=""} -``` - -판단: - -- PostgreSQL readiness는 정상인데 `cnpg_collector_up`이 0이면 CNPG instance exporter 또는 metrics query 문제입니다. -- Redis Pod는 정상인데 `redis_up`이 0이면 exporter sidecar, Redis auth, Redis process 응답 문제입니다. -- PodMonitor target 자체가 없으면 Prometheus Operator selector 또는 Pod label/port 설정을 확인합니다. - -### LokiUnavailable - -증상: Loki single-binary Pod가 1분 이상 ready가 아닙니다. - -확인: - -```sh -kubectl -n monitoring get pod,pvc | grep loki -kubectl -n monitoring describe pod loki-0 -kubectl -n monitoring logs loki-0 --tail=200 -kubectl -n monitoring describe pvc -l app.kubernetes.io/name=loki -``` - -판단: - -- PVC attach/mount 실패면 Longhorn volume 상태를 먼저 봅니다. -- readiness만 실패하면 Loki config, retention, disk full, compactor error를 확인합니다. -- Loki가 내려가도 앱 요청 처리는 계속되지만 로그 수집/조회가 끊기므로 빠르게 복구합니다. - -### PrometheusUnavailable / PrometheusReplicaUnavailable / AlertmanagerUnavailable / AlertmanagerReplicaUnavailable - -증상: Prometheus 또는 Alertmanager Pod가 ready가 아니거나, Prometheus ready replica가 2개 미만입니다. - -확인: - -```sh -kubectl -n monitoring get pod,svc | grep -E 'prometheus|alertmanager' -kubectl -n monitoring describe pod prometheus-kube-prometheus-stack-prometheus-0 -kubectl -n monitoring describe pod prometheus-kube-prometheus-stack-prometheus-1 -kubectl -n monitoring describe pod alertmanager-kube-prometheus-stack-alertmanager-0 -kubectl -n monitoring logs prometheus-kube-prometheus-stack-prometheus-0 -c prometheus --tail=200 -kubectl -n monitoring logs alertmanager-kube-prometheus-stack-alertmanager-0 -c alertmanager --tail=200 -``` - -```promql -kube_pod_status_ready{namespace="monitoring", pod=~"prometheus-kube-prometheus-stack-prometheus-.*|alertmanager-kube-prometheus-stack-alertmanager-.*", condition="true"} -sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) -prometheus_notifications_alertmanagers_discovered -prometheus_rule_evaluation_failures_total -``` - -판단: - -- Prometheus ready replica가 1개만 남으면 metric 수집과 rule evaluation은 계속되지만 monitoring path HA를 잃은 상태입니다. 남은 replica까지 내려가기 전에 PVC attach, config reload, resource pressure, scheduling event를 확인합니다. -- Prometheus가 전부 down이면 metric 수집과 rule evaluation이 멈춥니다. Alertmanager가 살아 있어도 새 알림은 생성되지 않습니다. -- Alertmanager가 down이면 firing alert가 있어도 Discord/email 전달이 실패합니다. P0 수신 지연 원인으로 먼저 봅니다. -- PVC, config reload, rule syntax, resource pressure, operator reconciliation event를 함께 확인합니다. - -### LonghornManagerDown / LonghornManagerReplicaUnavailable / LonghornVolumeFaulted / LonghornVolumeReadOnly - -증상: Longhorn manager가 전부 ready가 아니거나, volume이 faulted/read-only 상태입니다. - -확인: - -```sh -kubectl -n longhorn-system get pod,svc -kubectl -n longhorn-system describe pod -l app=longhorn-manager -kubectl -n longhorn-system logs -l app=longhorn-manager --tail=200 -``` - -```promql -sum(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) -kube_daemonset_status_number_available{namespace="longhorn-system", daemonset="longhorn-manager"} -kube_daemonset_status_desired_number_scheduled{namespace="longhorn-system", daemonset="longhorn-manager"} -longhorn_volume_robustness -longhorn_volume_file_system_read_only -longhorn_volume_actual_size_bytes -longhorn_volume_capacity_bytes -``` - -판단: - -- manager가 전부 down이면 Longhorn UI/API와 metrics가 끊깁니다. node pressure, image pull, network, Longhorn system Pod 상태를 먼저 봅니다. -- volume faulted는 PVC-backed workload의 storage availability 문제입니다. affected `pvc_namespace`/`pvc`를 찾아 해당 workload 상태와 Longhorn replica 상태를 확인합니다. -- read-only volume은 쓰기 실패로 바로 이어질 수 있습니다. Loki/Tempo/PostgreSQL/Redis PVC가 대상이면 해당 관측/데이터 경로 장애로 취급합니다. - -### VLLMUnavailable - -증상: prod vLLM `/metrics` scrape가 2분 이상 실패합니다. AI 힌트 기능이 실패하거나 장시간 지연될 수 있습니다. - -확인: - -```sh -kubectl -n code-place-prod get pod,svc,endpoints,pvc | grep vllm -kubectl -n code-place-prod describe pod -l app=vllm -kubectl -n code-place-prod logs deploy/vllm --tail=200 -kubectl -n monitoring get servicemonitor vllm -o yaml -``` - -```promql -up{job="vllm", namespace="code-place-prod"} -kube_pod_status_ready{namespace="code-place-prod", pod=~"vllm-.*", condition="true"} -``` - -판단: - -- Pod가 pending이면 GPU node label, `runtimeClassName: nvidia`, GPU resource, Longhorn HF cache PVC attach 상태를 봅니다. -- Pod는 ready인데 scrape만 실패하면 Service port/path, vLLM `/metrics`, NetworkPolicy 여부를 확인합니다. -- backend AI hint API error가 같이 증가하면 사용자 영향이 있습니다. - -### GPUXIDError - -증상: NVIDIA DCGM이 GPU XID error를 1분 이상 보고합니다. vLLM이 ready여도 GPU driver, ECC, PCIe, memory, thermal 문제로 추론 장애가 날 수 있습니다. - -확인: - -```sh -kubectl -n monitoring logs daemonset/dcgm-exporter --tail=200 -kubectl -n code-place-prod get pod -l app=vllm -o wide -kubectl -n code-place-prod logs deploy/vllm --tail=200 -kubectl describe node -l workload.code-place.ai/vllm=true -``` - -```promql -DCGM_FI_DEV_XID_ERRORS{namespace="monitoring"} -DCGM_FI_DEV_GPU_TEMP{namespace="monitoring"} -DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"} -``` - -판단: - -- XID가 증가하고 vLLM latency/5xx가 같이 증가하면 AI hint 사용자 영향으로 봅니다. -- 같은 node에서 temperature나 utilization도 높으면 cooling, sustained load, `max-num-seqs`, KV cache 설정을 함께 낮춰 봅니다. -- XID가 반복되면 Pod 재시작만으로 닫지 말고 node driver/runtime 상태와 GPU hardware 상태를 확인합니다. - -## P1 - -### OTelCollectorUnavailable / OTelCollectorReplicaUnavailable / OTelCollectorRefusedSpans / OTelCollectorExportFailures / TempoUnavailable / TempoPVCAlmostFull - -확인: - -```sh -kubectl -n monitoring get pod,svc,pvc | grep -E 'otel-collector|tempo' -kubectl -n monitoring describe pod -l app=otel-collector -kubectl -n monitoring describe pod -l app=tempo -kubectl -n monitoring logs deploy/otel-collector --tail=200 -kubectl -n monitoring logs deploy/tempo --tail=200 -kubectl -n monitoring describe pvc tempo-data -``` - -PromQL: - -```promql -kube_pod_status_ready{namespace="monitoring", pod=~"otel-collector-.*|tempo-.*", condition="true"} -kube_deployment_status_replicas_available{namespace="monitoring", deployment="otel-collector"} -kube_deployment_spec_replicas{namespace="monitoring", deployment="otel-collector"} -max(kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) / -max(kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) -sum(rate(otelcol_receiver_accepted_spans_total{namespace="monitoring"}[5m])) -sum(rate(otelcol_receiver_refused_spans_total{namespace="monitoring"}[5m])) -sum(rate(otelcol_exporter_send_failed_spans_total{namespace="monitoring"}[5m])) -``` - -판단: - -- Collector가 down이면 dev backend/celery trace export가 실패합니다. 앱 요청 처리는 계속되지만 trace 관측이 끊깁니다. -- refused span이 증가하면 Collector receiver limit, malformed OTLP payload, backend/celery exporter 설정, collector 로그를 먼저 봅니다. -- export failure가 증가하면 Tempo readiness, `tempo.monitoring.svc.cluster.local:4317` 경로, Collector exporter queue/retry 로그를 봅니다. -- Tempo가 down이면 Collector가 span export 실패를 기록합니다. -- Tempo PVC가 85%를 넘으면 retention 단축, noisy trace sampling 조정, PVC 증설 중 하나를 먼저 결정합니다. -- Tempo는 단일 replica와 Longhorn RWO PVC 조합이므로 Deployment strategy가 `Recreate`여야 합니다. RollingUpdate로 바뀌면 업그레이드 중 새 Pod와 기존 Pod가 `tempo-data` PVC를 두고 경합해 rollout이 멈출 수 있습니다. -- 채점 병목은 Tempo에서 `judge_task`, `submission.judge`, `judge_server.select`, `judge_server.request`, `submission.result_save` span duration을 순서대로 봅니다. 선택 span이 길면 DB row lock/available judge 부족, request span이 길면 judge-server 실행/네트워크, result save span이 길면 DB lock/write 병목을 의심합니다. -- prod는 기본 `OTEL_ENABLED=0`입니다. prod trace가 없다고 해서 장애는 아니며, dev에서 Collector/Tempo를 먼저 확인한 뒤 prod 승격 여부를 결정합니다. - -### ApiLatencyHigh - -확인: - -```promql -histogram_quantile(0.95, sum by (le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=""}[5m]))) -sum by (namespace) (codeplace_waiting_queue_length{namespace=""}) -``` - -backend latency와 DB/Redis readiness, CPU/memory를 같이 봅니다. - -### PublicEndpointDown / PublicEndpointLatencyHigh / PublicEndpointTLSExpiringSoon / BlackboxExporterUnavailable / BlackboxExporterReplicaUnavailable - -확인: - -```promql -probe_success{probe_type="public-http"} -probe_duration_seconds{probe_type="public-http"} -probe_http_duration_seconds{probe_type="public-http"} -(probe_ssl_earliest_cert_expiry{probe_type="public-http"} - time()) / 86400 -kube_pod_status_ready{namespace="monitoring", pod=~"blackbox-exporter-.*", condition="true"} -kube_deployment_status_replicas_available{namespace="monitoring", deployment="blackbox-exporter"} -kube_deployment_spec_replicas{namespace="monitoring", deployment="blackbox-exporter"} -``` - -판단: - -- dev endpoint down은 배포 테스트 영향으로 P1입니다. prod endpoint down은 P0입니다. -- latency가 synthetic probe에서만 높으면 public route/TLS/frontend nginx를 먼저 보고, backend latency도 높으면 API/DB/Redis 병목을 같이 봅니다. -- `BlackboxExporterUnavailable`이면 probe 자체가 죽은 것이므로 공개 URL 상태를 판단할 수 없습니다. - -### GrafanaUnavailable / GrafanaMetricsTargetDown / MonitoringServiceMetricsTargetDown / PrometheusOperatorUnavailable / PrometheusConfigReloadFailed / AlertmanagerConfigReloadFailed / PrometheusRuleEvaluationFailures / PrometheusAlertmanagerDiscoveryFailed / AlertmanagerNotificationFailures / AlertmanagerWebhookSecretMissing - -확인: - -```sh -kubectl -n monitoring get pod | grep -E 'grafana|operator|prometheus|alertmanager' -kubectl -n monitoring get secret alertmanager-contact-points -kubectl -n monitoring describe pod -l app.kubernetes.io/name=grafana -kubectl -n monitoring describe pod -l app.kubernetes.io/name=prometheus-operator -kubectl -n monitoring logs deploy/kube-prometheus-stack-operator --tail=200 -kubectl -n monitoring logs alertmanager-kube-prometheus-stack-alertmanager-0 -c alertmanager --tail=200 -kubectl -n monitoring get prometheusrule,servicemonitor,podmonitor,alertmanagerconfig -``` - -PromQL: - -```promql -kube_pod_status_ready{namespace="monitoring", pod=~"kube-prometheus-stack-grafana-.*|kube-prometheus-stack-operator-.*", condition="true"} -up{namespace="monitoring", service="kube-prometheus-stack-grafana"} -up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy"} -sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) -sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) -prometheus_config_last_reload_successful{namespace="monitoring"} -alertmanager_config_last_reload_successful{namespace="monitoring"} -kube_secret_info{namespace="monitoring", secret="alertmanager-contact-points"} -prometheus_notifications_alertmanagers_discovered -sum by (integration) (increase(alertmanager_notifications_failed_total[5m])) -sum by (job, namespace) (up == 0) -``` - -판단: - -- Grafana만 down이면 알림은 계속 갈 수 있지만 dashboard/Explore 기반 원인 분석이 막힙니다. -- Grafana UI는 살아 있는데 metrics target만 down이면 Grafana 내부 health와 provisioning 상태를 Prometheus에서 못 봅니다. Grafana ServiceMonitor, Service port, `/metrics` 응답, Grafana 로그를 확인합니다. -- monitoring service metrics target이 down이면 Pod readiness가 정상이어도 ServiceMonitor selector, Service port name, generated Prometheus config, network path를 확인합니다. -- Operator가 down이면 새 ServiceMonitor/PodMonitor/PrometheusRule/AlertmanagerConfig 변경이 Prometheus/Alertmanager에 반영되지 않을 수 있습니다. -- rule evaluation failure가 있으면 최근 추가한 PromQL expression과 label cardinality, metric 존재 여부를 먼저 봅니다. -- Alertmanager discovery가 0이면 AlertmanagerConfig가 맞아도 Prometheus가 알림을 보낼 대상이 없습니다. Prometheus generated config와 Alertmanager Service/endpoints를 확인합니다. -- webhook Secret이 없으면 Discord 알림은 동작하지 않습니다. 이 경우 SealedSecret controller가 `alertmanager-contact-points` Secret을 복호화했는지, Secret key가 `webhook-url`인지 확인합니다. -- notification failure가 있으면 webhook Secret 값, Alertmanager generated config, Discord endpoint 응답, cluster outbound network, rate limit을 확인합니다. 로컬 `curl` 성공만으로 Alertmanager Pod의 egress 성공을 보장하지 않습니다. - -### FrontendRuntimeErrorsSpike - -확인: - -```promql -sum by (namespace, surface, error_type) (increase(codeplace_frontend_error_total{namespace=""}[10m])) -sum by (namespace, surface, error_type) (rate(codeplace_frontend_error_total{namespace=""}[5m])) -``` - -Grafana Explore: - -```logql -{namespace="", app="backend"} | json | logger="frontend.error" -{namespace="", app="backend"} | json | logger="frontend.error" | request_id="" -``` - -판단: - -- API 5xx/latency는 정상인데 frontend error만 증가하면 최근 frontend image, JS bundle, browser compatibility, feature flag를 먼저 봅니다. -- `surface="admin"`만 증가하면 admin bundle 또는 관리자 화면 배포 영향으로 좁힙니다. -- `error_type="unhandled_rejection"`이 많으면 API response handling, auth/session 만료 처리, network rejection을 봅니다. -- Sentry가 켜진 환경이면 같은 `release`, `environment`, `request_id`로 Sentry event와 Loki `frontend.error` 로그를 대조합니다. - -### JudgeWaitingQueueBacklog / CeleryBrokerQueueBacklog / CodePlaceCollectorFailed - -확인: - -```promql -codeplace_waiting_queue_length{namespace=""} -codeplace_celery_broker_queue_length{namespace=""} -codeplace_observability_collector_success{namespace=""} -sum by (namespace) (codeplace_judge_server_available{namespace=""}) -``` - -```sh -kubectl -n logs deploy/backend --tail=200 -kubectl -n logs deploy/celery-worker --tail=200 -kubectl -n get pod -l app=judge-server -``` - -판단: - -- queue 증가와 judge available 감소가 같이 보이면 judge-server 문제입니다. -- pending age만 증가하면 submission 생성 이후 Celery enqueue/worker dispatch/Redis queue 경로를 봅니다. -- judging age가 증가하면 judge-server 실행, judge-server `/judge` 호출, result save path, worker 로그의 `judge.tasks.judge_task`를 봅니다. -- pending/judging count는 낮지만 oldest age만 높으면 특정 제출이 고립된 상태입니다. 사용자 영향은 제한적일 수 있으나 결과 저장/상태 전환 실패를 확인합니다. - -### SubmissionCreateSystemFailures / JudgeTaskFailures - -확인: - -```promql -sum by (namespace, scope, status) (increase(codeplace_submission_create_outcome_total{namespace=""}[5m])) -sum by (namespace, scope, status) (increase(codeplace_judge_task_outcome_total{namespace=""}[5m])) -``` - -```sh -kubectl -n logs deploy/backend --tail=200 -kubectl -n logs deploy/celery-worker --tail=200 -``` - -판단: - -- `db_error`는 Submission row 생성이 실패한 상태입니다. PostgreSQL readiness, connection usage, lock wait, backend exception log를 봅니다. -- `enqueue_error`는 Submission row는 생성됐지만 judge task queue 등록이 실패한 상태입니다. Redis/Celery broker, worker readiness, queue metric을 먼저 봅니다. -- judge task의 `submission_missing`, `user_missing`, `error`는 Celery worker 내부 실패입니다. worker log의 exception, Submission/User row 상태, judge-server 호출, result save path를 봅니다. -- judge task의 `user_disabled`는 비활성 사용자 제출 skip으로 보고 시스템 장애 알림 대상에서 제외합니다. -- `throttled`, `invalid_captcha`, `problem_not_found`, `language_not_allowed`, `contest_permission_denied`는 사용자 입력/권한/문제 상태로 보고 시스템 장애 알림 대상에서 제외합니다. - -### CeleryWorkerRestarting / CeleryBeatDown - -확인: - -```sh -kubectl -n get pod -l app=celery-worker -kubectl -n get pod -l app=celery-beat -kubectl -n logs deploy/celery-worker --tail=200 -kubectl -n logs deploy/celery-beat --tail=200 -``` - -Grafana Explore: - -```logql -{namespace="", container=~"celery-worker|celery-beat"} -``` - -### CeleryTaskFailures / CeleryTaskRetries / CeleryTaskRuntimeHigh / CeleryMinuteScheduledTaskStale / CeleryDailyScheduledTaskStale / CeleryWeeklyScheduledTaskStale - -확인: - -```promql -sum by (task_name, status) (increase(codeplace_celery_task_total{namespace=""}[10m])) -histogram_quantile(0.95, sum by (task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=""}[10m]))) -codeplace_celery_task_last_runtime_seconds{namespace=""} -codeplace_celery_task_last_seen_age_seconds{namespace=""} -codeplace_celery_task_last_success_age_seconds{namespace=""} -``` - -Grafana Explore: - -```logql -{namespace="", container=~"celery-worker|celery-beat"} | json | task_status=~"failure|retry" -{namespace="", container=~"celery-worker|celery-beat"} | json | task_name="" -{namespace="", container=~"celery-worker|celery-beat"} | json | request_id="" -``` - -판단: - -- 특정 task만 실패하면 최근 코드 변경, 입력 데이터, 외부 의존성을 우선 확인합니다. -- runtime만 증가하면 DB/Redis 지연, lock, worker CPU/memory, queue backlog를 같이 봅니다. -- `judge.tasks.judge_task` 실패와 `JudgeWaitingQueueBacklog`가 같이 발생하면 judge-server availability와 Redis `waiting_queue`를 우선 확인합니다. -- scheduled task stale은 beat가 task를 발행하지 못했거나 worker가 성공 종료하지 못한 상태입니다. - `calculate_user_score_fluctuation`은 1분 주기라 10분 stale이면 빠르게 확인하고, `calculate_user_score_basis`는 일간, `update_weekly_stats`와 `update_bonus_problem`은 주간 기준으로 봅니다. -- `codeplace_celery_task_last_success_age_seconds` series가 아예 없으면 해당 task 성공이 한 번도 관측되지 않은 상태입니다. 배포 직후라면 beat schedule 등록, worker task autodiscovery, Redis metric write, backend `/metrics` collector를 순서대로 확인합니다. -- task metric은 worker가 Redis에 누적하고 backend `/metrics` collector가 노출합니다. worker 로그에는 `task_id`, `task_name`, `task_status`, `duration_ms`, `trace_id`가 포함됩니다. - -### PodCrashLooping / CodePlacePodNotReady - -확인: - -```sh -kubectl -n get pod -kubectl -n describe pod -kubectl -n logs --previous --tail=200 -``` - -Grafana Explore: - -```logql -{namespace="", pod=""} -{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} |= "" -``` - -이미지 pull, secret mount, readiness, resource limit, node scheduling을 확인합니다. - -### KubernetesEventExporterUnavailable - -확인: - -```sh -kubectl -n monitoring get pod,svc,servicemonitor | grep event-exporter -kubectl -n monitoring describe pod -l app.kubernetes.io/name=kubernetes-event-exporter -kubectl -n monitoring logs deploy/kubernetes-event-exporter --tail=200 -``` - -PromQL: - -```promql -kube_pod_status_ready{namespace="monitoring", pod=~"kubernetes-event-exporter-.*", condition="true"} -up{job="kubernetes-event-exporter", namespace="monitoring"} -``` - -판단: - -- exporter가 down이면 `CodePlace Kubernetes Events` dashboard의 Warning event 로그가 비어 있을 수 있습니다. -- kube-state-metrics 기반 `ImagePullBackOff`, `CrashLoopBackOff`, `OOMKilled`, `Pending` 알림은 exporter와 별개로 계속 동작합니다. -- event exporter 로그에 throttling 또는 discarded event가 보이면 `maxEventAgeSeconds`, API server 부하, 이벤트 폭주 원인을 확인합니다. - -### KubernetesPodImagePullBackOff / KubernetesPodCrashLoopBackOff / KubernetesPodOOMKilled / KubernetesPodUnschedulable - -확인: - -```sh -kubectl -n get pod -kubectl -n describe pod -kubectl -n logs --previous --tail=200 -kubectl describe node -``` - -PromQL: - -```promql -kube_pod_container_status_waiting_reason{namespace="", reason=~"ImagePullBackOff|ErrImagePull|CrashLoopBackOff|CreateContainerConfigError|CreateContainerError"} -max_over_time(kube_pod_container_status_last_terminated_reason{namespace="", reason="OOMKilled"}[10m]) -kube_pod_status_unschedulable{namespace=""} -kube_pod_status_phase{namespace="", phase="Pending"} -``` - -Grafana Explore: - -```logql -{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} |= "" -{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} |~ "FailedScheduling|ImagePullBackOff|ErrImagePull|BackOff|OOMKilling|Unhealthy|FailedMount|FailedAttachVolume" -{namespace="", pod=""} -``` - -판단: - -- `ImagePullBackOff`/`ErrImagePull`은 image tag, registry 접근, image pull secret, node DNS/egress를 먼저 봅니다. -- `CrashLoopBackOff`는 `--previous` 로그와 직전 Warning event를 같이 봅니다. readiness 실패만으로 재시작되는지, 프로세스가 종료되는지 구분합니다. -- `OOMKilled`는 memory limit, 최근 트래픽, 캐시/큐 증가, JVM/Python heap 설정을 확인합니다. -- `Pending`/`Unschedulable`은 node capacity, taint/toleration, nodeSelector/affinity, PVC binding, Longhorn volume attach event를 확인합니다. - -### CodePlaceDeploymentUnavailable / CodePlaceDeploymentRolloutStuck / CodePlacePDBAtRisk - -확인: - -```sh -kubectl -n get deploy,rs,pod -kubectl -n rollout status deploy/ -kubectl -n describe deploy -kubectl -n get event --sort-by=.lastTimestamp | tail -50 -``` - -PromQL: - -```promql -kube_deployment_status_replicas_unavailable{namespace=""} -kube_deployment_status_observed_generation{namespace=""} -kube_deployment_metadata_generation{namespace=""} -kube_poddisruptionbudget_status_current_healthy{namespace="", poddisruptionbudget=~"backend|frontend"} -kube_poddisruptionbudget_status_desired_healthy{namespace="", poddisruptionbudget=~"backend|frontend"} -kube_pod_container_status_waiting_reason{namespace=""} -``` - -판단: - -- unavailable replica가 늘면 readiness probe, image pull, crash loop, resource pressure, scheduling event를 먼저 봅니다. -- observed generation이 metadata generation을 따라가지 못하면 Deployment controller reconcile, ReplicaSet 생성 실패, admission/image pull/secret 문제를 확인합니다. -- PDB current healthy가 desired healthy 이하이면 node drain/upgrade 전에 backend/frontend replica readiness를 먼저 복구합니다. -- 배포 직후 API 5xx/latency와 같이 발생하면 사용자 영향이 있는 rollout 장애로 봅니다. - -### CodePlaceServiceNoReadyEndpoints / VLLMServiceNoReadyEndpoints - -확인: - -```sh -kubectl -n get svc,endpoints,endpointslice -kubectl -n describe svc -kubectl -n get pod --show-labels -kubectl -n get event --sort-by=.lastTimestamp | tail -50 -``` - -PromQL: - -```promql -sum by (namespace, endpoint) (kube_endpoint_address{namespace="", ready="true"}) -sum by (namespace, endpoint) (kube_endpoint_address{namespace="", ready="false"}) -kube_pod_status_ready{namespace="", condition="true"} -kube_deployment_status_replicas_unavailable{namespace=""} -``` - -판단: - -- Pod는 ready인데 endpoint가 0이면 Service selector label과 Pod label 불일치를 먼저 봅니다. -- Pod readiness가 같이 0이면 rollout/readiness/image pull/crash loop 문제로 봅니다. -- ingress 5xx와 같이 발생하면 Traefik route 자체보다 Service endpoint 부재가 직접 원인일 가능성이 큽니다. -- vLLM endpoint가 0이면 AI hint path만 영향받을 수 있으므로 `CodePlace AI Inference` dashboard와 GPU node scheduling 상태를 같이 봅니다. - -### CodePlaceContainerCPUHigh / CodePlaceContainerMemoryHigh - -확인: - -```promql -topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace="", container!=""}[5m]))) -topk(10, container_memory_working_set_bytes{namespace="", container!=""}) -``` - -CPU/memory가 지속적으로 높으면 replica 조정, limit 조정, 최근 배포 변경을 확인합니다. - -### PostgresCollectorError / PostgresHADegraded / PostgresReplicaUnavailable / RedisReplicaUnavailable / RedisMemoryHigh / RedisConnectedClientsHigh / RedisRejectedConnections - -확인: - -```promql -cnpg_collector_last_collection_error{namespace="", cluster="postgres"} -cnpg_collector_nodes_used{namespace="", cluster="postgres"} -sum by (namespace) (kube_pod_status_ready{namespace="", condition="true", pod=~"postgres-[0-9]+"}) -sum by (namespace) (kube_pod_status_ready{namespace="", condition="true", pod=~"redis-.*|redis-replication-.*"}) -redis_memory_used_bytes{namespace=""} -redis_memory_max_bytes{namespace=""} -codeplace_redis_connected_clients{namespace=""} -codeplace_redis_max_clients{namespace=""} -increase(codeplace_redis_rejected_connections_total{namespace=""}[5m]) -``` - -```sh -kubectl -n logs deploy/backend --tail=200 -kubectl -n get pod -l cnpg.io/cluster=postgres -o wide -kubectl -n get pod -l app=redis-replication -o wide -``` - -판단: - -- `PostgresCollectorError`는 metrics query 실패이므로 CNPG exporter 로그와 PostgreSQL 권한/상태를 확인합니다. -- `PostgresHADegraded`는 3개 instance가 서로 다른 node에 분산되지 않은 상태입니다. node 수, taint, Longhorn volume attach 상태를 확인합니다. -- `PostgresReplicaUnavailable` / `RedisReplicaUnavailable`은 3노드 분산 상태에서 남은 replica 수와 node placement를 같이 봅니다. 일부 replica만 down이면 failover 여지는 있지만 다음 장애에 취약합니다. -- PostgreSQL connection/long transaction/lock wait는 backend `/metrics`에서 직접 DB 조회하지 않습니다. 필요하면 CNPG exporter에 동일 의미의 metric이 있는지 운영 클러스터에서 먼저 확인한 뒤, 확정된 exporter metric만 dashboard/rule에 추가합니다. -- `RedisMemoryHigh`는 Redis `maxmemory`가 0보다 클 때만 동작합니다. `maxmemory=0`이면 Redis 자체 제한이 없으므로 container memory alert를 기준으로 보고, 제한을 둘지 별도로 결정합니다. -- `RedisConnectedClientsHigh`는 connection pool 설정, worker replica/concurrency, client leak를 확인합니다. -- `RedisRejectedConnections`는 Redis `maxclients`에 실제로 걸린 상태입니다. API/cache와 Celery queue가 동시에 영향을 받을 수 있습니다. -- Redis memory가 높으면 eviction 정책, queue/backlog, cache key 증가를 확인합니다. - -### PVCAlmostFull / MonitoringPVCAlmostFull - -확인: - -```promql -max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=""}) / -max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=""}) -max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim!~".*loki.*|tempo-data"}) / -max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim!~".*loki.*|tempo-data"}) -``` - -```sh -kubectl -n get pvc -kubectl -n describe pvc -``` - -### LonghornVolumeDegraded / LonghornNodeNotReady / LonghornDiskNotReady / LonghornNodeStorageHigh / LonghornDiskUsageHigh - -확인: - -```sh -kubectl -n longhorn-system get pod -kubectl get node -kubectl describe node -``` - -```promql -longhorn_volume_robustness == 2 -longhorn_node_status{condition="ready"} -longhorn_disk_status{condition="ready"} -(longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1) -(longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1) -``` - -판단: - -- degraded volume은 replica rebuild 중이거나 node/disk 여유가 부족한 상태일 수 있습니다. -- node/disk not ready는 앱 Pod 장애보다 먼저 storage layer 문제로 봅니다. -- storage usage가 85%를 넘으면 Loki/Tempo retention 축소, noisy 로그/trace 감소, Longhorn disk/PVC 증설 중 하나를 결정합니다. - 클라우드 object storage가 없는 현재 조건에서는 Longhorn 여유 용량이 로그/트레이스 retention의 상한입니다. - -### VLLMWaitingQueueHigh / VLLMKVCacheHigh / VLLMRequestLatencyHigh - -확인: - -```promql -{__name__="vllm:num_requests_running", namespace="code-place-prod"} -{__name__="vllm:num_requests_waiting", namespace="code-place-prod"} -{__name__="vllm:kv_cache_usage_perc", namespace="code-place-prod"} -histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) -histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:request_queue_time_seconds_bucket", namespace="code-place-prod"}[10m]))) -histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:time_to_first_token_seconds_bucket", namespace="code-place-prod"}[10m]))) -``` - -```sh -kubectl -n code-place-prod top pod -l app=vllm -kubectl -n code-place-prod describe pod -l app=vllm -kubectl -n code-place-prod logs deploy/vllm --tail=200 -kubectl -n code-place-prod describe pvc vllm-hf-cache -``` - -판단: - -- waiting queue와 queue p95가 같이 오르면 vLLM scheduling capacity 부족입니다. GPU saturation, `max-num-seqs`, KV cache pressure를 봅니다. -- KV cache가 90%를 넘으면 max model length, 동시 요청 수, prompt 길이, GPU memory utilization을 함께 봅니다. -- e2e latency만 증가하면 prompt/generation token 증가, backend streaming read timeout, vLLM 로그의 model/engine warning을 확인합니다. - 현재 vLLM은 prod 전용입니다. dev에서 AI hint를 별도 검증하려면 dev vLLM 배포와 ServiceMonitor 확장이 먼저 필요합니다. - -### AIHintAPIFailures / AIHintBackendErrors / AIHintBackendLatencyHigh - -확인: - -```promql -sum by (namespace, scope, status) (increase(codeplace_ai_hint_api_outcome_total{namespace=""}[5m])) -sum by (namespace, status) (increase(codeplace_ai_hint_requests_total{namespace=""}[5m])) -histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace="", status="success"}[10m]))) -{__name__="vllm:num_requests_waiting", namespace="code-place-prod"} -histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) -``` - -```sh -kubectl -n logs deploy/backend --tail=200 -kubectl -n code-place-prod get svc,endpoints vllm -kubectl -n code-place-prod logs deploy/vllm --tail=200 -``` - -판단: - -- `request_error`는 backend에서 vLLM Service까지의 endpoint, network, timeout, HTTP error를 먼저 확인합니다. -- `stream_parse_error`는 vLLM OpenAI-compatible stream format과 backend parser 호환성을 확인합니다. -- API outcome의 `permission_denied`, `problem_limit_exceeded`, `disabled`, `problem_not_found`는 시스템 장애보다 사용자/대회 상태로 먼저 봅니다. -- API outcome의 `llm_error`, `unexpected_error`, `empty_response`는 사용자 영향이 있는 실패입니다. -- backend duration과 vLLM queue/e2e latency가 같이 높으면 vLLM/GPU 병목으로 봅니다. -- backend duration만 높으면 client stream read behavior, backend worker saturation, proxy timeout을 확인합니다. - -### DCGMExporterUnavailable / GPUUtilizationHigh / GPUMemoryHigh / GPUTemperatureHigh - -확인: - -```sh -kubectl -n monitoring get daemonset,pod,svc,servicemonitor dcgm-exporter -o wide -kubectl -n monitoring describe daemonset dcgm-exporter -kubectl -n monitoring logs daemonset/dcgm-exporter --tail=200 -kubectl get node -l workload.code-place.ai/vllm=true -``` - -PromQL: - -```promql -kube_daemonset_status_number_available{namespace="monitoring", daemonset="dcgm-exporter"} -kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"} -DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"} -DCGM_FI_DEV_FB_USED{namespace="monitoring"} / clamp_min(DCGM_FI_DEV_FB_USED{namespace="monitoring"} + DCGM_FI_DEV_FB_FREE{namespace="monitoring"}, 1) -DCGM_FI_DEV_GPU_TEMP{namespace="monitoring"} -DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace="monitoring"} -DCGM_FI_PROF_DRAM_ACTIVE{namespace="monitoring"} -``` - -판단: - -- exporter가 unavailable이면 GPU 상태를 볼 수 없습니다. vLLM node label, `runtimeClassName: nvidia`, NVIDIA device plugin/runtime, pod scheduling event를 확인합니다. -- GPU utilization만 높고 memory 여유가 있으면 request concurrency와 vLLM scheduling capacity 문제일 가능성이 큽니다. -- framebuffer memory usage가 높으면 KV cache pressure, `max-num-seqs`, `max-model-len`, model load 상태를 우선 봅니다. -- temperature가 높으면 AI latency보다 node cooling/thermal throttling을 먼저 의심합니다. - -### LokiPVCAlmostFull - -확인: - -```promql -max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim=~".*loki.*"}) / -max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim=~".*loki.*"}) -``` - -```sh -kubectl -n monitoring get pvc | grep loki -kubectl -n monitoring describe pvc -``` - -조치: - -- dev/prod retention이 실제 기대대로 적용되는지 확인합니다. -- 당장 여유가 없으면 Longhorn PVC size를 증설합니다. -- 반복되면 retention 기간을 줄이거나 prod 로그 저장 방식을 내부 S3 호환 object storage로 분리합니다. - -### AlloyDaemonSetUnavailable / LokiGatewayUnavailable / LokiGatewayReplicaUnavailable / LokiIngestionStalled / LokiRequestErrors / LokiCanaryMissingEntries / AlloyLokiWriteDrops / AlloyLokiWriteRetries - -확인: - -```sh -kubectl -n monitoring get daemonset alloy -kubectl -n monitoring describe daemonset alloy -kubectl -n monitoring logs daemonset/alloy --tail=200 -kubectl -n monitoring get pod -l app.kubernetes.io/component=gateway -kubectl -n monitoring logs deploy/loki-gateway --tail=200 -kubectl -n monitoring logs loki-0 --tail=200 -``` - -PromQL: - -```promql -sum(rate(loki_distributor_lines_received_total{namespace="monitoring"}[5m])) -sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) -sum(rate(loki_request_duration_seconds_count{namespace="monitoring", status_code=~"5.."}[5m])) -sum(increase(loki_canary_missing_entries_total{namespace="monitoring"}[5m])) -sum(increase(loki_write_dropped_entries_total{namespace="monitoring"}[5m])) -sum(increase(loki_write_batch_retries_total{namespace="monitoring"}[5m])) -``` - -LogQL: - -```logql -sum by (namespace) (rate({namespace=~"code-place-dev|code-place-prod|monitoring|kube-system"}[5m])) -sum by (node) (rate({namespace=~"code-place-dev|code-place-prod|monitoring|kube-system"}[5m])) -sum by (namespace, app) (rate({namespace=~"code-place-dev|code-place-prod", app=~"backend|frontend"}[5m])) -sum by (namespace, container) (rate({namespace=~"code-place-dev|code-place-prod", container=~"celery-worker|celery-beat|judge-server|hub-auth|vllm"}[5m])) -``` - -판단: - -- Alloy 또는 LogQL node별 유입이 특정 node에서만 비면 해당 node의 scheduling, taint, resource pressure, `/var/log/pods` mount 상태를 봅니다. -- Loki gateway가 비면 Alloy write와 Grafana query가 실패할 수 있습니다. -- Loki가 ready인데 ingest rate가 0이면 Alloy discovery/relabel, node별 container log path, gateway write path를 먼저 확인합니다. -- Loki 전체 ingest rate는 정상인데 특정 namespace/app 로그만 0이면 해당 Pod stdout/stderr, container restart, Alloy relabel label, node별 Alloy target discovery를 확인합니다. -- Loki 5xx가 있으면 gateway와 `loki-0` 로그를 같이 보고, PVC full/Longhorn degraded/compactor error를 확인합니다. -- Loki canary missing entry는 write 성공 후 readback 또는 storage/query path가 깨진 상태일 수 있습니다. -- Alloy write drop/retry가 있으면 Loki gateway 응답, network, Loki backpressure, Alloy resource limit을 확인합니다. - -### CodePlaceNodePressure - -확인: - -```sh -kubectl get node -kubectl describe node -kubectl top node -kubectl top pod -A --sort-by=memory -``` - -node pressure는 앱 문제가 아니라 cluster resource 문제일 가능성이 큽니다. - -## Test Alert - -Discord webhook 자체는 curl로 확인할 수 있습니다. Alertmanager routing은 임시 `PrometheusRule`로 확인합니다. -Email fallback은 SMTP 정보가 정해진 환경에서만 켭니다. `kubernetes/monitoring/alertmanager-config-email.example.yaml`의 placeholder를 실제 `smarthost`, `from`, `to`, `authUsername`으로 바꾸고 `alertmanager-email` Secret을 만든 뒤 적용합니다. - -SealedSecret으로 webhook을 새로 만들 때는 운영자가 실제 cluster 공개키로 암호화합니다. - -```sh -mkdir -p kubernetes/monitoring/secrets - -kubectl -n monitoring create secret generic alertmanager-contact-points \ - --from-literal=webhook-url="$ALERT_WEBHOOK_URL" \ - --dry-run=client -o yaml \ - | kubeseal --controller-namespace kube-system --format yaml \ - > kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml - -kubectl apply -f kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml -``` - -```yaml -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: codeplace-test-alert - namespace: monitoring - labels: - release: kube-prometheus-stack -spec: - groups: - - name: codeplace.test - interval: 15s - rules: - - alert: CodePlaceTestAlert - expr: vector(1) - for: 30s - labels: - severity: critical - priority: P0 - namespace: code-place-dev - annotations: - summary: "CodePlace test alert" - description: "Discord alert routing test" -``` - -테스트 후 삭제합니다. - -```sh -kubectl -n monitoring delete prometheusrule codeplace-test-alert -``` diff --git a/kubernetes/monitoring/alertmanager-config.yaml b/kubernetes/monitoring/alertmanager-config.yaml index 99de5764..09a09951 100644 --- a/kubernetes/monitoring/alertmanager-config.yaml +++ b/kubernetes/monitoring/alertmanager-config.yaml @@ -56,7 +56,6 @@ spec: Overview: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-namespace={{ .CommonLabels.namespace }} Logs: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-namespace={{ .CommonLabels.namespace }} Monitoring: https://monitoring.code-place-dev.site/d/codeplace-monitoring-stack/codeplace-monitoring-stack?orgId=1 - Runbook: https://github.com/pnu-code-place/code-place/blob/develop/docs/content/developer/infra/observability-runbook/_index.md Source: {{ range .Alerts }}{{ .GeneratorURL }}{{ end }} username: CodePlace Alertmanager - name: dev-p1-muted @@ -76,6 +75,5 @@ spec: Overview: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-namespace={{ .CommonLabels.namespace }} Logs: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-namespace={{ .CommonLabels.namespace }} Monitoring: https://monitoring.code-place-dev.site/d/codeplace-monitoring-stack/codeplace-monitoring-stack?orgId=1 - Runbook: https://github.com/pnu-code-place/code-place/blob/develop/docs/content/developer/infra/observability-runbook/_index.md Source: {{ range .Alerts }}{{ .GeneratorURL }}{{ end }} username: CodePlace Alertmanager From 7065c80b039c8ef16c674e9bba6bdce18da44ed9 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:48:35 +0900 Subject: [PATCH 134/147] =?UTF-8?q?Alertmanager=20=EC=9D=B4=EB=A9=94?= =?UTF-8?q?=EC=9D=BC=20=EC=98=88=EC=8B=9C=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../alertmanager-config-email.example.yaml | 73 ------------------- 1 file changed, 73 deletions(-) delete mode 100644 kubernetes/monitoring/alertmanager-config-email.example.yaml diff --git a/kubernetes/monitoring/alertmanager-config-email.example.yaml b/kubernetes/monitoring/alertmanager-config-email.example.yaml deleted file mode 100644 index f15a2658..00000000 --- a/kubernetes/monitoring/alertmanager-config-email.example.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# Optional email fallback for CodePlace alerts. -# -# Do not add this file to kustomization.yaml until SMTP values are available. -# Required secret: -# -# kubectl -n monitoring create secret generic alertmanager-email \ -# --from-literal=smtp-auth-password="$SMTP_AUTH_PASSWORD" \ -# --dry-run=client -o yaml | kubectl apply -f - -# -# Replace smarthost, from, to, and authUsername before applying. -apiVersion: monitoring.coreos.com/v1alpha1 -kind: AlertmanagerConfig -metadata: - name: codeplace-alert-email-fallback - labels: - alertmanagerConfig: codeplace -spec: - route: - groupBy: - - alertname - - priority - - namespace - groupWait: 30s - groupInterval: 5m - repeatInterval: 1h - receiver: unmatched-muted - routes: - - matchers: - - name: priority - value: P0 - groupWait: 10s - groupInterval: 1m - repeatInterval: 15m - receiver: p0-email - - matchers: - - name: priority - value: P1 - groupWait: 30s - groupInterval: 5m - repeatInterval: 1h - receiver: p1-email - receivers: - - name: unmatched-muted - - name: p0-email - emailConfigs: - - sendResolved: true - smarthost: smtp.example.com:587 - from: codeplace-alerts@example.com - to: oncall@example.com - authUsername: codeplace-alerts@example.com - authPassword: - name: alertmanager-email - key: smtp-auth-password - requireTLS: true - headers: - - key: Subject - value: '[{{ .Status }}][P0] {{ .CommonLabels.alertname }}' - html: '{{ .CommonAnnotations.summary }}
{{ .CommonAnnotations.description }}' - - name: p1-email - emailConfigs: - - sendResolved: true - smarthost: smtp.example.com:587 - from: codeplace-alerts@example.com - to: oncall@example.com - authUsername: codeplace-alerts@example.com - authPassword: - name: alertmanager-email - key: smtp-auth-password - requireTLS: true - headers: - - key: Subject - value: '[{{ .Status }}][P1] {{ .CommonLabels.alertname }}' - html: '{{ .CommonAnnotations.summary }}
{{ .CommonAnnotations.description }}' From bf022c243416edffae49797642c21a90a063266f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:51:39 +0900 Subject: [PATCH 135/147] =?UTF-8?q?Sentry=20=EB=A6=B4=EB=A6=AC=EC=A6=88=20?= =?UTF-8?q?=ED=95=98=EB=93=9C=EC=BD=94=EB=94=A9=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kubernetes/overlays/dev/backend-deployment-patch.yaml | 2 -- kubernetes/overlays/dev/celery-beat-deployment-patch.yaml | 2 -- kubernetes/overlays/dev/celery-worker-deployment-patch.yaml | 2 -- kubernetes/overlays/prod/backend-deployment-patch.yaml | 2 -- kubernetes/overlays/prod/celery-beat-deployment-patch.yaml | 2 -- kubernetes/overlays/prod/celery-worker-deployment-patch.yaml | 2 -- 6 files changed, 12 deletions(-) diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index 7a98516d..78acaaa2 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -17,7 +17,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" - - name: SENTRY_RELEASE - value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index 3712eff3..8bbe2697 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -20,7 +20,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" - - name: SENTRY_RELEASE - value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index 0a8b217a..82089303 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -23,7 +23,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" - - name: SENTRY_RELEASE - value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/prod/backend-deployment-patch.yaml b/kubernetes/overlays/prod/backend-deployment-patch.yaml index e4468851..de88b416 100644 --- a/kubernetes/overlays/prod/backend-deployment-patch.yaml +++ b/kubernetes/overlays/prod/backend-deployment-patch.yaml @@ -24,5 +24,3 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" - - name: SENTRY_RELEASE - value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml index bb4e4f49..db60527a 100644 --- a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml @@ -20,5 +20,3 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" - - name: SENTRY_RELEASE - value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml index cb654565..f98ee708 100644 --- a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml @@ -23,5 +23,3 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" - - name: SENTRY_RELEASE - value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" From 80ae303dbbe904f196a7246c5c9e5646a3e08486 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:55:49 +0900 Subject: [PATCH 136/147] =?UTF-8?q?=EB=B6=88=ED=95=84=EC=9A=94=ED=95=9C=20?= =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=98=88=EC=99=B8=EC=B2=98?= =?UTF-8?q?=EB=A6=AC=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/observability_tracing.py | 49 +++++-------------- backend/utils/tests.py | 28 ----------- frontend/src/utils/client_error_reporter.js | 52 ++++++++++----------- 3 files changed, 36 insertions(+), 93 deletions(-) diff --git a/backend/utils/observability_tracing.py b/backend/utils/observability_tracing.py index d5ae480e..c3c76817 100644 --- a/backend/utils/observability_tracing.py +++ b/backend/utils/observability_tracing.py @@ -1,5 +1,6 @@ import logging -from contextlib import nullcontext + +from opentelemetry import trace from utils.shortcuts import get_env @@ -7,32 +8,11 @@ _OTEL_CONFIGURED = False -class _NoOpSpan: - - def set_attribute(self, key, value): - return None - - -class _NoOpTracer: - - def start_as_current_span(self, name): - return nullcontext(_NoOpSpan()) - - def get_tracer(name): - try: - from opentelemetry import trace - except Exception: - return _NoOpTracer() return trace.get_tracer(name) def get_current_trace_context(): - try: - from opentelemetry import trace - except Exception: - return {} - try: span_context = trace.get_current_span().get_span_context() except Exception: @@ -52,21 +32,16 @@ def configure_opentelemetry(service_name): if _OTEL_CONFIGURED: return - try: - from opentelemetry import trace - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter - from opentelemetry.instrumentation.celery import CeleryInstrumentor - from opentelemetry.instrumentation.django import DjangoInstrumentor - from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor - from opentelemetry.instrumentation.redis import RedisInstrumentor - from opentelemetry.instrumentation.requests import RequestsInstrumentor - from opentelemetry.sdk.resources import Resource - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased - except Exception: - logger.exception("OpenTelemetry packages are unavailable") - return + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + from opentelemetry.instrumentation.celery import CeleryInstrumentor + from opentelemetry.instrumentation.django import DjangoInstrumentor + from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor + from opentelemetry.instrumentation.redis import RedisInstrumentor + from opentelemetry.instrumentation.requests import RequestsInstrumentor + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased try: sampler_ratio = float(get_env("OTEL_TRACES_SAMPLER_ARG", "0.05")) diff --git a/backend/utils/tests.py b/backend/utils/tests.py index df6f7011..69316f7e 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -18,7 +18,6 @@ ) from utils.observability_metrics import CodePlaceCollector from utils import observability_tracing -from utils.observability_tracing import get_tracer from utils.testcase_cache import TestCaseCacheManager @@ -405,33 +404,6 @@ class ObservabilityTracingTest(SimpleTestCase): def tearDown(self): observability_tracing._OTEL_CONFIGURED = False - def test_get_tracer_returns_noop_tracer_when_opentelemetry_is_unavailable(self): - real_import = __import__ - - def fake_import(name, *args, **kwargs): - if name == "opentelemetry": - raise ImportError("missing opentelemetry") - return real_import(name, *args, **kwargs) - - with patch("builtins.__import__", side_effect=fake_import): - tracer = get_tracer("test") - - with tracer.start_as_current_span("span") as span: - self.assertIsNone(span.set_attribute("key", "value")) - - def test_get_current_trace_context_returns_empty_when_opentelemetry_is_unavailable(self): - from utils.observability_tracing import get_current_trace_context - - real_import = __import__ - - def fake_import(name, *args, **kwargs): - if name == "opentelemetry": - raise ImportError("missing opentelemetry") - return real_import(name, *args, **kwargs) - - with patch("builtins.__import__", side_effect=fake_import): - self.assertEqual(get_current_trace_context(), {}) - def test_configure_opentelemetry_is_idempotent(self): observability_tracing._OTEL_CONFIGURED = True real_import = __import__ diff --git a/frontend/src/utils/client_error_reporter.js b/frontend/src/utils/client_error_reporter.js index 62591f91..8993ca33 100644 --- a/frontend/src/utils/client_error_reporter.js +++ b/frontend/src/utils/client_error_reporter.js @@ -20,37 +20,33 @@ function currentRoute(router) { } function reportClientError(payload) { - try { - const requestId = currentRequestId() - const body = JSON.stringify({ - ...payload, - request_id: requestId, - release: process.env.VERSION, - }) - const headers = { - "Content-Type": "application/json", - } - if (requestId) { - headers["X-Request-ID"] = requestId - } + const requestId = currentRequestId() + const body = JSON.stringify({ + ...payload, + request_id: requestId, + release: process.env.VERSION, + }) + const headers = { + "Content-Type": "application/json", + } + if (requestId) { + headers["X-Request-ID"] = requestId + } - if (navigator.sendBeacon) { - const blob = new Blob([body], { type: "application/json" }) - if (navigator.sendBeacon(CLIENT_ERROR_ENDPOINT, blob)) { - return - } + if (navigator.sendBeacon) { + const blob = new Blob([body], { type: "application/json" }) + if (navigator.sendBeacon(CLIENT_ERROR_ENDPOINT, blob)) { + return } - - window.fetch(CLIENT_ERROR_ENDPOINT, { - method: "POST", - headers, - body, - credentials: "same-origin", - keepalive: true, - }).catch(() => {}) - } catch (error) { - console.error(error) } + + window.fetch(CLIENT_ERROR_ENDPOINT, { + method: "POST", + headers, + body, + credentials: "same-origin", + keepalive: true, + }).catch(() => {}) } function errorMessage(error) { From 753a9ddf9080a56729537cd60ceea31a7046b7b0 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 02:59:34 +0900 Subject: [PATCH 137/147] =?UTF-8?q?=EA=B3=BC=ED=95=9C=20=EA=B4=80=EC=B8=A1?= =?UTF-8?q?=EC=84=B1=20=EC=95=8C=EB=A6=BC=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../observability-improvement-plan/_index.md | 4 +- kubernetes/monitoring/prometheus-rules.yaml | 434 +----------------- 2 files changed, 10 insertions(+), 428 deletions(-) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index d05d15c2..50b3f5d1 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -114,8 +114,8 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `Ingress5xxSpike`: Traefik 5xx 비율 5% 초과 2분 지속. - `JudgeAllUnavailable`: 사용 가능한 judge-server 0개 1분 지속. - `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. -- `PostgresUnavailable`: PostgreSQL Pod not ready 또는 exporter target down 1분 지속. -- `RedisUnavailable`: Redis Pod not ready 또는 exporter target down 1분 지속. +- `PostgresUnavailable`: PostgreSQL Pod not ready 또는 readiness metric missing 1분 지속. +- `RedisUnavailable`: Redis/Sentinel Pod not ready 또는 readiness metric missing 1분 지속. - `LokiUnavailable`: Loki Pod not ready 1분 지속. ### P1 diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index f7f00dec..005821cb 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -101,16 +101,6 @@ spec: annotations: summary: "Public endpoint is down in prod" description: "A production synthetic HTTPS probe has failed for 1 minute." - - alert: PublicEndpointTLSCritical - expr: (probe_ssl_earliest_cert_expiry{probe_type="public-http", environment="prod"} - time()) < 259200 - for: 15m - labels: - severity: critical - priority: P0 - namespace: code-place-prod - annotations: - summary: "Public endpoint TLS certificate expires within 3 days" - description: "The production CodePlace public HTTPS certificate expires within 3 days." - alert: JudgeAllUnavailable expr: (sum by (namespace) (codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) == 0) or (max by (namespace) (up{job="backend", namespace=~"code-place-(dev|prod)"}) unless on (namespace) codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) for: 1m @@ -169,46 +159,6 @@ spec: annotations: summary: "Redis pod is not ready in prod" description: "No prod Redis/Sentinel pod has been ready, or Redis pod readiness metrics are missing, for 1 minute." - - alert: PostgresExporterDown - expr: (min(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) == 0) or absent(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) - for: 2m - labels: - severity: critical - priority: P0 - namespace: code-place-dev - annotations: - summary: "PostgreSQL metrics exporter is down" - description: "Prometheus cannot collect CNPG PostgreSQL metrics for code-place-dev for 2 minutes." - - alert: PostgresExporterDown - expr: (min(cnpg_collector_up{namespace="code-place-prod", cluster="postgres"}) == 0) or absent(cnpg_collector_up{namespace="code-place-prod", cluster="postgres"}) - for: 2m - labels: - severity: critical - priority: P0 - namespace: code-place-prod - annotations: - summary: "PostgreSQL metrics exporter is down" - description: "Prometheus cannot collect CNPG PostgreSQL metrics for code-place-prod for 2 minutes." - - alert: RedisExporterDown - expr: (min(redis_up{namespace="code-place-dev"}) == 0) or absent(redis_up{namespace="code-place-dev"}) - for: 2m - labels: - severity: critical - priority: P0 - namespace: code-place-dev - annotations: - summary: "Redis metrics exporter is down" - description: "Prometheus cannot collect Redis exporter metrics for code-place-dev for 2 minutes." - - alert: RedisExporterDown - expr: (min(redis_up{namespace="code-place-prod"}) == 0) or absent(redis_up{namespace="code-place-prod"}) - for: 2m - labels: - severity: critical - priority: P0 - namespace: code-place-prod - annotations: - summary: "Redis metrics exporter is down" - description: "Prometheus cannot collect Redis exporter metrics for code-place-prod for 2 minutes." - alert: LokiUnavailable expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) for: 1m @@ -219,26 +169,6 @@ spec: annotations: summary: "Loki is not ready" description: "No Loki single-binary pod has been ready for 1 minute. Log ingestion and log queries may be unavailable." - - alert: LonghornManagerDown - expr: (sum(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) == 0) or absent(kube_pod_status_ready{namespace="longhorn-system", condition="true", pod=~"longhorn-manager-.*"}) - for: 2m - labels: - severity: critical - priority: P0 - namespace: longhorn-system - annotations: - summary: "Longhorn managers are not ready" - description: "No Longhorn manager pod has been ready for 2 minutes. PVC health and Longhorn metrics may be unavailable." - - alert: LonghornManagerReplicaUnavailable - expr: kube_daemonset_status_number_available{namespace="longhorn-system", daemonset="longhorn-manager"} < kube_daemonset_status_desired_number_scheduled{namespace="longhorn-system", daemonset="longhorn-manager"} - for: 5m - labels: - severity: warning - priority: P1 - namespace: longhorn-system - annotations: - summary: "Longhorn manager replica is unavailable" - description: "At least one Longhorn manager replica is unavailable for 5 minutes. Storage health visibility and volume operations are degraded." - alert: LonghornVolumeFaulted expr: longhorn_volume_robustness == 3 for: 1m @@ -279,36 +209,6 @@ spec: annotations: summary: "GPU driver reported an XID error" description: "DCGM reports a non-zero NVIDIA XID error for 1 minute. vLLM GPU health and node driver state should be checked." - - alert: PrometheusUnavailable - expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) - for: 1m - labels: - severity: critical - priority: P0 - namespace: monitoring - annotations: - summary: "Prometheus is not ready" - description: "The kube-prometheus-stack Prometheus pod has not been ready for 1 minute. Metric collection, rule evaluation, and alert delivery may be unavailable." - - alert: AlertmanagerUnavailable - expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) - for: 1m - labels: - severity: critical - priority: P0 - namespace: monitoring - annotations: - summary: "Alertmanager is not ready" - description: "The kube-prometheus-stack Alertmanager pod has not been ready for 1 minute. Fast alert delivery may be unavailable." - - alert: AlertmanagerWebhookSecretMissing - expr: absent(kube_secret_info{namespace="monitoring", secret="alertmanager-contact-points"}) - for: 1m - labels: - severity: critical - priority: P0 - namespace: monitoring - annotations: - summary: "Alertmanager Discord webhook secret is missing" - description: "The monitoring namespace does not expose the alertmanager-contact-points Secret through kube-state-metrics. Discord notifications cannot work until the SealedSecret is decrypted into this Secret." - name: codeplace.p1 interval: 30s rules: @@ -332,7 +232,7 @@ spec: summary: "Public endpoint is down in dev" description: "A dev synthetic HTTPS probe has failed for 2 minutes." - alert: PublicEndpointLatencyHigh - expr: probe_duration_seconds{probe_type="public-http", namespace=~"code-place-(dev|prod)|monitoring"} > 2 + expr: probe_duration_seconds{probe_type="public-http", namespace=~"code-place-(dev|prod)"} > 2 for: 5m labels: severity: warning @@ -340,15 +240,6 @@ spec: annotations: summary: "Public frontend endpoint latency is high" description: "A synthetic HTTPS probe has taken more than 2 seconds for 5 minutes." - - alert: PublicEndpointTLSExpiringSoon - expr: (probe_ssl_earliest_cert_expiry{probe_type="public-http", namespace=~"code-place-(dev|prod)|monitoring"} - time()) < 1209600 - for: 1h - labels: - severity: warning - priority: P1 - annotations: - summary: "Public endpoint TLS certificate expires within 14 days" - description: "A CodePlace public HTTPS certificate expires within 14 days." - alert: FrontendRuntimeErrorsSpike expr: sum by (namespace, surface) (increase(codeplace_frontend_error_total{namespace=~"code-place-(dev|prod)"}[5m])) > 10 for: 5m @@ -358,156 +249,6 @@ spec: annotations: summary: "Frontend runtime errors increased" description: "Browser clients reported more than 10 frontend runtime errors in 5 minutes. Check recent deploys, Sentry events, and frontend.error logs." - - alert: BlackboxExporterUnavailable - expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"blackbox-exporter-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"blackbox-exporter-.*"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Blackbox exporter is not ready" - description: "The synthetic public endpoint probe exporter has not been ready for 2 minutes." - - alert: BlackboxExporterReplicaUnavailable - expr: kube_deployment_status_replicas_available{namespace="monitoring", deployment="blackbox-exporter"} < kube_deployment_spec_replicas{namespace="monitoring", deployment="blackbox-exporter"} - for: 5m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Blackbox exporter replica is unavailable" - description: "At least one synthetic public endpoint probe exporter replica is unavailable for 5 minutes. Public endpoint probing is degraded." - - alert: GrafanaPublicEndpointDown - expr: (probe_success{environment="monitoring", service="grafana", probe_type="public-http"} == 0) or absent(probe_success{environment="monitoring", service="grafana", probe_type="public-http"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Grafana public endpoint is down" - description: "The public Grafana monitoring endpoint synthetic HTTPS probe has failed for 2 minutes. Dashboards may be unavailable even if the Grafana pod is ready." - - alert: KubernetesEventExporterUnavailable - expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kubernetes-event-exporter-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kubernetes-event-exporter-.*"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Kubernetes event exporter is not ready" - description: "Kubernetes Warning events may not be available in Loki for deployment and scheduling failure triage." - - alert: GrafanaUnavailable - expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-grafana-.*"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Grafana is not ready" - description: "No kube-prometheus-stack Grafana pod in the monitoring namespace has been ready for 2 minutes. Dashboards and Explore may be unavailable." - - alert: GrafanaMetricsTargetDown - expr: (up{namespace="monitoring", service="kube-prometheus-stack-grafana"} == 0) or absent(up{namespace="monitoring", service="kube-prometheus-stack-grafana"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Grafana metrics target is down" - description: "Prometheus cannot scrape Grafana /metrics for 2 minutes. Grafana may still serve UI, but dashboard/datasource/provisioning health visibility is degraded." - - alert: MonitoringServiceMetricsTargetDown - expr: (up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy"} == 0) or (kube_service_info{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy"} unless on (namespace, service) up{namespace="monitoring", service=~"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "A monitoring service metrics target is down" - description: "A monitoring Service exists but Prometheus cannot scrape its metrics target for 2 minutes, or the target series is missing. Check ServiceMonitor selection, Service port names, Pod readiness, and generated Prometheus config." - - alert: PrometheusOperatorUnavailable - expr: (max(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-operator-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"kube-prometheus-stack-operator-.*"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Prometheus Operator is not ready" - description: "The kube-prometheus-stack operator has not been ready for 2 minutes. ServiceMonitor, PodMonitor, PrometheusRule, and AlertmanagerConfig reconciliation may be stalled." - - alert: PrometheusRuleEvaluationFailures - expr: sum(increase(prometheus_rule_evaluation_failures_total[5m])) > 0 - for: 5m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Prometheus rule evaluation is failing" - description: "Prometheus has reported rule evaluation failures for 5 minutes. Alert and recording rule expressions should be checked." - - alert: PrometheusConfigReloadFailed - expr: (max(prometheus_config_last_reload_successful{namespace="monitoring"}) == 0) or absent(prometheus_config_last_reload_successful{namespace="monitoring"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Prometheus config reload is failing" - description: "Prometheus has not successfully reloaded its generated configuration for 2 minutes, or the reload metric is missing. ServiceMonitor, Probe, and rule changes may not be active." - - alert: PrometheusAlertmanagerDiscoveryFailed - expr: (max(prometheus_notifications_alertmanagers_discovered) < 1) or absent(prometheus_notifications_alertmanagers_discovered) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Prometheus has not discovered Alertmanager" - description: "Prometheus has discovered zero Alertmanager targets for 2 minutes. Alert delivery may be broken even if alert rules are firing." - - alert: PrometheusReplicaUnavailable - expr: sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-.*"}) < 2 - for: 5m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Prometheus replica count is degraded" - description: "Fewer than 2 Prometheus replicas have been ready for 5 minutes. Metric collection and alert evaluation still work, but the monitoring path has lost HA." - - alert: AlertmanagerReplicaUnavailable - expr: sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"}) < 2 - for: 5m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Alertmanager replica count is degraded" - description: "Fewer than 2 Alertmanager replicas have been ready for 5 minutes. Alert delivery still works, but the notification path has lost HA." - - alert: AlertmanagerConfigReloadFailed - expr: (max(alertmanager_config_last_reload_successful{namespace="monitoring"}) == 0) or absent(alertmanager_config_last_reload_successful{namespace="monitoring"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Alertmanager config reload is failing" - description: "Alertmanager has not successfully reloaded its generated configuration for 2 minutes, or the reload metric is missing. Check AlertmanagerConfig, webhook Secret references, and operator reconciliation." - - alert: AlertmanagerNotificationFailures - expr: sum(increase(alertmanager_notifications_failed_total[5m])) > 0 - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Alertmanager notification delivery is failing" - description: "Alertmanager has failed to send one or more notifications in the last 5 minutes. Check Discord webhook secret, receiver config, outbound network, and Alertmanager logs." - alert: JudgeWaitingQueueBacklog expr: codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"} > 5 for: 3m @@ -598,10 +339,7 @@ spec: summary: "Celery task runtime is high" description: "Celery task p95 runtime has been above 120 seconds for 10 minutes." - alert: CeleryMinuteScheduledTaskStale - expr: >- - (codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_fluctuation"} > 600) - or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-dev", task_name="account.tasks.calculate_user_score_fluctuation"}) - or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-prod", task_name="account.tasks.calculate_user_score_fluctuation"}) + expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_fluctuation"} > 600 for: 5m labels: severity: warning @@ -610,10 +348,7 @@ spec: summary: "Minute scheduled Celery task is stale" description: "calculate_user_score_fluctuation has not completed successfully for more than 10 minutes, or no success metric has ever been observed. Check celery-beat, celery-worker, Redis broker, and task logs." - alert: CeleryDailyScheduledTaskStale - expr: >- - (codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_basis"} > 93600) - or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-dev", task_name="account.tasks.calculate_user_score_basis"}) - or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-prod", task_name="account.tasks.calculate_user_score_basis"}) + expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_basis"} > 93600 for: 30m labels: severity: warning @@ -622,12 +357,7 @@ spec: summary: "Daily scheduled Celery task is stale" description: "calculate_user_score_basis has not completed successfully for more than 26 hours, or no success metric has ever been observed. Check celery-beat, celery-worker, Redis broker, and task logs." - alert: CeleryWeeklyScheduledTaskStale - expr: >- - (codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name=~"problem.tasks.update_weekly_stats|problem.tasks.update_bonus_problem"} > 691200) - or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-dev", task_name="problem.tasks.update_weekly_stats"}) - or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-prod", task_name="problem.tasks.update_weekly_stats"}) - or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-dev", task_name="problem.tasks.update_bonus_problem"}) - or absent(codeplace_celery_task_last_success_age_seconds{namespace="code-place-prod", task_name="problem.tasks.update_bonus_problem"}) + expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name=~"problem.tasks.update_weekly_stats|problem.tasks.update_bonus_problem"} > 691200 for: 1h labels: severity: warning @@ -654,7 +384,7 @@ spec: summary: "CodePlace pod is not ready" description: "A CodePlace application or datastore pod has not been ready for 5 minutes." - alert: KubernetesPodImagePullBackOff - expr: max by (namespace, pod, container, reason) (kube_pod_container_status_waiting_reason{namespace=~"code-place-(dev|prod)|monitoring", reason=~"ImagePullBackOff|ErrImagePull"}) == 1 + expr: max by (namespace, pod, container, reason) (kube_pod_container_status_waiting_reason{namespace=~"code-place-(dev|prod)", reason=~"ImagePullBackOff|ErrImagePull"}) == 1 for: 2m labels: severity: warning @@ -663,7 +393,7 @@ spec: summary: "A Kubernetes pod cannot pull its image" description: "A pod has been waiting with ImagePullBackOff or ErrImagePull for 2 minutes. Check image tag, registry access, and image pull secret." - alert: KubernetesPodCrashLoopBackOff - expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{namespace=~"code-place-(dev|prod)|monitoring", reason="CrashLoopBackOff"}) == 1 + expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{namespace=~"code-place-(dev|prod)", reason="CrashLoopBackOff"}) == 1 for: 5m labels: severity: warning @@ -672,7 +402,7 @@ spec: summary: "A Kubernetes pod is in CrashLoopBackOff" description: "A pod has been waiting with CrashLoopBackOff for 5 minutes. Check the previous container logs and Kubernetes Warning events." - alert: KubernetesPodOOMKilled - expr: max by (namespace, pod, container) (max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~"code-place-(dev|prod)|monitoring", reason="OOMKilled"}[10m])) == 1 + expr: max by (namespace, pod, container) (max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~"code-place-(dev|prod)", reason="OOMKilled"}[10m])) == 1 for: 2m labels: severity: warning @@ -681,7 +411,7 @@ spec: summary: "A Kubernetes container was OOMKilled" description: "A container was terminated by OOMKilled within the last 10 minutes. Check memory limits, recent traffic, and heap/cache growth." - alert: KubernetesPodUnschedulable - expr: (max by (namespace, pod) (kube_pod_status_unschedulable{namespace=~"code-place-(dev|prod)|monitoring"}) == 1) or (max by (namespace, pod) (kube_pod_status_phase{namespace=~"code-place-(dev|prod)|monitoring", phase="Pending"}) == 1) + expr: (max by (namespace, pod) (kube_pod_status_unschedulable{namespace=~"code-place-(dev|prod)"}) == 1) or (max by (namespace, pod) (kube_pod_status_phase{namespace=~"code-place-(dev|prod)", phase="Pending"}) == 1) for: 10m labels: severity: warning @@ -698,24 +428,6 @@ spec: annotations: summary: "CodePlace deployment has unavailable replicas" description: "A CodePlace deployment has had unavailable replicas for 5 minutes. Check rollout status, pod events, readiness probes, image pulls, and resource pressure." - - alert: CodePlaceDeploymentRolloutStuck - expr: max by (namespace, deployment) (kube_deployment_status_observed_generation{namespace=~"code-place-(dev|prod)", deployment=~"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm"} != on (namespace, deployment) kube_deployment_metadata_generation{namespace=~"code-place-(dev|prod)", deployment=~"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm"}) > 0 - for: 10m - labels: - severity: warning - priority: P1 - annotations: - summary: "CodePlace deployment rollout is stuck" - description: "A CodePlace deployment observed generation has not caught up with metadata generation for 10 minutes. Check deployment events, ReplicaSet state, and rollout history." - - alert: CodePlacePDBAtRisk - expr: kube_poddisruptionbudget_status_current_healthy{namespace=~"code-place-(dev|prod)", poddisruptionbudget=~"backend|frontend"} <= kube_poddisruptionbudget_status_desired_healthy{namespace=~"code-place-(dev|prod)", poddisruptionbudget=~"backend|frontend"} - for: 10m - labels: - severity: warning - priority: P1 - annotations: - summary: "CodePlace PDB has no disruption headroom" - description: "A backend or frontend PodDisruptionBudget has been at or below its desired healthy count for 10 minutes. Node drain or voluntary disruption may reduce user-path availability." - alert: CodePlaceServiceNoReadyEndpoints expr: (sum by (namespace, endpoint) (kube_endpoint_address{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server", ready="true"}) < 1) or (label_replace(kube_service_info{namespace=~"code-place-(dev|prod)", service=~"backend|frontend|hub-auth|judge-server"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server", ready="true"}) for: 2m @@ -835,16 +547,6 @@ spec: annotations: summary: "Loki PVC usage is above 85%" description: "The Loki Longhorn PVC has been above 85% usage for 10 minutes. Log retention or PVC size must be adjusted before ingestion stops." - - alert: MonitoringPVCAlmostFull - expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim!~".*loki.*|tempo-data"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim!~".*loki.*|tempo-data"}), 1) > 0.85 - for: 10m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Monitoring PVC usage is above 85%" - description: "A Prometheus, Alertmanager, Grafana, or monitoring support PVC has been above 85% usage for 10 minutes. Check retention, WAL growth, and Longhorn capacity." - alert: AlloyDaemonSetUnavailable expr: (kube_daemonset_status_number_available{namespace="monitoring", daemonset="alloy"} < kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="alloy"}) or absent(kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="alloy"}) for: 5m @@ -865,26 +567,6 @@ spec: annotations: summary: "Loki gateway is not ready" description: "No Loki gateway replica has been ready for 2 minutes. Alloy writes and Grafana Loki queries may fail." - - alert: LokiGatewayReplicaUnavailable - expr: sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) < 2 - for: 5m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Loki gateway replica count is degraded" - description: "Fewer than 2 Loki gateway replicas have been ready for 5 minutes. Loki writes and queries still work, but the gateway path has lost HA." - - alert: LokiIngestionStalled - expr: (sum(rate(loki_distributor_lines_received_total{namespace="monitoring"}[5m])) == 0) or absent(loki_distributor_lines_received_total{namespace="monitoring"}) - for: 10m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Loki is receiving no log lines" - description: "Loki has received zero log lines for 10 minutes. Check Alloy discovery, Loki gateway writes, and Loki ingestion health." - alert: LokiRequestErrors expr: sum(rate(loki_request_duration_seconds_count{namespace="monitoring", status_code=~"5.."}[5m])) > 0 for: 5m @@ -895,96 +577,6 @@ spec: annotations: summary: "Loki is returning 5xx responses" description: "Loki has returned 5xx responses for 5 minutes. Log writes or Grafana log queries may be failing." - - alert: LokiCanaryMissingEntries - expr: sum(increase(loki_canary_missing_entries_total{namespace="monitoring"}[5m])) > 0 - for: 5m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Loki canary cannot read back written logs" - description: "Loki canary reports missing entries. Loki may be accepting writes while queries or storage are unhealthy." - - alert: AlloyLokiWriteDrops - expr: sum(increase(loki_write_dropped_entries_total{namespace="monitoring"}[5m])) > 0 - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Alloy is dropping log entries" - description: "Alloy dropped log entries before writing to Loki. Check Alloy limits, Loki gateway errors, and backpressure." - - alert: AlloyLokiWriteRetries - expr: sum(increase(loki_write_batch_retries_total{namespace="monitoring"}[5m])) > 0 - for: 5m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Alloy is retrying Loki writes" - description: "Alloy has retried Loki write batches for 5 minutes. Loki writes may be slow or intermittently failing." - - alert: OTelCollectorUnavailable - expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"otel-collector-.*"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "OpenTelemetry Collector is not ready" - description: "No OpenTelemetry Collector replica has been ready for 2 minutes. Enabled services cannot export traces." - - alert: OTelCollectorReplicaUnavailable - expr: kube_deployment_status_replicas_available{namespace="monitoring", deployment="otel-collector"} < kube_deployment_spec_replicas{namespace="monitoring", deployment="otel-collector"} - for: 5m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "OpenTelemetry Collector replica is unavailable" - description: "At least one OpenTelemetry Collector replica is unavailable for 5 minutes. Trace ingestion is still degraded even if another replica is serving traffic." - - alert: OTelCollectorRefusedSpans - expr: sum(increase(otelcol_receiver_refused_spans_total{namespace="monitoring"}[5m])) > 0 - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "OpenTelemetry Collector is refusing spans" - description: "The OpenTelemetry Collector has refused incoming spans in the last 5 minutes. Check collector limits, receiver errors, and backend/celery OTLP client logs." - - alert: OTelCollectorExportFailures - expr: sum(increase(otelcol_exporter_send_failed_spans_total{namespace="monitoring"}[5m])) > 0 - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "OpenTelemetry Collector cannot export spans" - description: "The OpenTelemetry Collector has failed to export spans to Tempo in the last 5 minutes. Check Tempo readiness, collector exporter logs, and network path." - - alert: TempoUnavailable - expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"tempo-.*"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Tempo is not ready" - description: "No Tempo pod has been ready for 2 minutes. Trace queries and trace ingestion may fail." - - alert: TempoPVCAlmostFull - expr: max(kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim="tempo-data"}), 1) > 0.85 - for: 10m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "Tempo PVC usage is above 85%" - description: "The Tempo Longhorn PVC has been above 85% usage for 10 minutes. Trace retention or PVC size must be adjusted." - alert: LonghornVolumeDegraded expr: longhorn_volume_robustness == 2 for: 5m @@ -1092,16 +684,6 @@ spec: annotations: summary: "User-facing AI hint API failures detected" description: "AI hint API returned LLM errors, unexpected errors, or empty responses in the last 5 minutes. Compare API outcome status with backend stream and vLLM panels." - - alert: DCGMExporterUnavailable - expr: (kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"} == 0) or (kube_daemonset_status_number_available{namespace="monitoring", daemonset="dcgm-exporter"} < kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"}) or absent(kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="dcgm-exporter"}) - for: 2m - labels: - severity: warning - priority: P1 - namespace: monitoring - annotations: - summary: "DCGM exporter is not running on the GPU node" - description: "The NVIDIA DCGM exporter DaemonSet is unavailable for 2 minutes. GPU utilization, memory, temperature, and XID metrics may be missing." - alert: GPUUtilizationHigh expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"}) > 95 for: 15m From 582d02c10aa642cb1a78d9fc0add95b264abc836 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 03:10:35 +0900 Subject: [PATCH 138/147] =?UTF-8?q?judge=20=EC=84=9C=EB=B2=84=20=EC=9A=B0?= =?UTF-8?q?=ED=9A=8C=20=EB=A9=94=ED=8A=B8=EB=A6=AD=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/account/tests.py | 2 +- backend/conf/views.py | 12 +- backend/judge/dispatcher.py | 4 - backend/utils/constants.py | 1 - backend/utils/judge_server_observability.py | 106 ------------------ backend/utils/observability_metrics.py | 48 -------- backend/utils/tests.py | 78 ------------- .../observability-improvement-plan/_index.md | 7 +- .../grafana-dashboard-codeplace.yaml | 54 --------- kubernetes/monitoring/prometheus-rules.yaml | 20 ---- 10 files changed, 4 insertions(+), 328 deletions(-) delete mode 100644 backend/utils/judge_server_observability.py diff --git a/backend/account/tests.py b/backend/account/tests.py index df046a5d..5dd80383 100644 --- a/backend/account/tests.py +++ b/backend/account/tests.py @@ -8,7 +8,7 @@ from django.contrib import auth from django.db import DatabaseError from django.http.response import JsonResponse -from django.test import RequestFactory, SimpleTestCase, TestCase +from django.test import RequestFactory, SimpleTestCase from django.utils import decorators from django.utils.timezone import now from utils.api.tests import APIClient, APITestCase diff --git a/backend/conf/views.py b/backend/conf/views.py index 8bd42c09..5593c8d2 100644 --- a/backend/conf/views.py +++ b/backend/conf/views.py @@ -22,7 +22,6 @@ from problem.models import Problem from submission.models import Submission from utils.api import APIView, CSRFExemptAPIView, validate_serializer -from utils.judge_server_observability import remove_judge_server_snapshot, sync_judge_server_snapshot from utils.shortcuts import send_email, get_env from utils.xss_filter import XSSHtml from .models import JudgeServer @@ -134,18 +133,13 @@ def delete(self, request): hostname = request.GET.get("hostname") if hostname: JudgeServer.objects.filter(hostname=hostname).delete() - remove_judge_server_snapshot(hostname) return self.success() @validate_serializer(EditJudgeServerSerializer) @super_admin_required def put(self, request): is_disabled = request.data.get("is_disabled", False) - server = JudgeServer.objects.filter(id=request.data["id"]).first() - if server: - server.is_disabled = is_disabled - server.save(update_fields=["is_disabled"]) - sync_judge_server_snapshot(server) + JudgeServer.objects.filter(id=request.data["id"]).update(is_disabled=is_disabled) if not is_disabled: process_pending_task() return self.success() @@ -171,9 +165,8 @@ def post(self, request): server.last_heartbeat = timezone.now() server.save( update_fields=["judger_version", "cpu_core", "memory_usage", "service_url", "ip", "last_heartbeat"]) - sync_judge_server_snapshot(server) except JudgeServer.DoesNotExist: - server = JudgeServer.objects.create( + JudgeServer.objects.create( hostname=data["hostname"], judger_version=data["judger_version"], cpu_core=data["cpu_core"], @@ -183,7 +176,6 @@ def post(self, request): service_url=data["service_url"], last_heartbeat=timezone.now(), ) - sync_judge_server_snapshot(server) # 新server上线 处理队列中的,防止没有新的提交而导致一直waiting process_pending_task() diff --git a/backend/judge/dispatcher.py b/backend/judge/dispatcher.py index 58273040..9b995c88 100644 --- a/backend/judge/dispatcher.py +++ b/backend/judge/dispatcher.py @@ -20,7 +20,6 @@ from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.constants import CacheKey, ProblemScore, ProblemField, Tier -from utils.judge_server_observability import increment_judge_server_task_snapshot, sync_judge_server_snapshot from utils.observability_metrics import record_judge_duration from utils.observability_tracing import get_tracer @@ -53,10 +52,8 @@ def __enter__(self) -> [JudgeServer, None]: span.set_attribute("codeplace.judge_server.candidates", len(servers)) for server in servers: if server.task_number <= server.cpu_core * 2: - selected_task_number = server.task_number + 1 server.task_number = F("task_number") + 1 server.save(update_fields=["task_number"]) - sync_judge_server_snapshot(server, task_number=selected_task_number) self.server = server span.set_attribute("codeplace.judge_server.selected", True) span.set_attribute("codeplace.judge_server.hostname", server.hostname) @@ -67,7 +64,6 @@ def __enter__(self) -> [JudgeServer, None]: def __exit__(self, exc_type, exc_val, exc_tb): if self.server: JudgeServer.objects.filter(id=self.server.id).update(task_number=F("task_number") - 1) - increment_judge_server_task_snapshot(self.server.hostname, -1) class DispatcherBase(object): diff --git a/backend/utils/constants.py b/backend/utils/constants.py index 7cad629e..ad2836a1 100644 --- a/backend/utils/constants.py +++ b/backend/utils/constants.py @@ -24,7 +24,6 @@ class ContestRuleType(Choices): class CacheKey: waiting_queue = "waiting_queue" - judge_server_observability = "codeplace:judge_server_observability" contest_rank_cache = "contest_rank_cache" website_config = "website_config" diff --git a/backend/utils/judge_server_observability.py b/backend/utils/judge_server_observability.py deleted file mode 100644 index 70101fe5..00000000 --- a/backend/utils/judge_server_observability.py +++ /dev/null @@ -1,106 +0,0 @@ -import json -import logging - -from django.utils import timezone - -from utils.cache import cache -from utils.constants import CacheKey - -logger = logging.getLogger(__name__) - -JUDGE_SERVER_SNAPSHOT_RETENTION_SECONDS = 24 * 60 * 60 - - -def sync_judge_server_snapshot(server, is_disabled=None, task_number=None): - hostname = getattr(server, "hostname", None) - if not hostname: - return - - disabled = getattr(server, "is_disabled", False) if is_disabled is None else is_disabled - if disabled: - remove_judge_server_snapshot(hostname) - return - - last_heartbeat = getattr(server, "last_heartbeat", None) - if last_heartbeat is None: - last_heartbeat_timestamp = 0 - else: - last_heartbeat_timestamp = last_heartbeat.timestamp() - - payload = { - "last_heartbeat_timestamp": last_heartbeat_timestamp, - "task_number": int(task_number if task_number is not None else getattr(server, "task_number", 0) or 0), - } - try: - cache.hset(CacheKey.judge_server_observability, hostname, json.dumps(payload)) - except Exception: - logger.exception("Failed to sync judge-server observability snapshot") - - -def remove_judge_server_snapshot(hostname): - if not hostname: - return - try: - cache.hdel(CacheKey.judge_server_observability, hostname) - except Exception: - logger.exception("Failed to remove judge-server observability snapshot") - - -def increment_judge_server_task_snapshot(hostname, delta): - if not hostname: - return - try: - raw = cache.hget(CacheKey.judge_server_observability, hostname) - if raw is None: - return - payload = _loads_snapshot(raw) - payload["task_number"] = max(int(payload.get("task_number", 0)) + int(delta), 0) - cache.hset(CacheKey.judge_server_observability, hostname, json.dumps(payload)) - except Exception: - logger.exception("Failed to increment judge-server task snapshot") - - -def load_judge_server_snapshots(): - snapshots = [] - try: - raw_snapshots = cache.hgetall(CacheKey.judge_server_observability) or {} - except Exception: - logger.exception("Failed to load judge-server observability snapshots") - return None - - now_timestamp = timezone.now().timestamp() - for raw_hostname, raw_payload in raw_snapshots.items(): - hostname = _decode(raw_hostname) - payload = _loads_snapshot(raw_payload) - last_heartbeat_timestamp = _float_or_zero(payload.get("last_heartbeat_timestamp")) - heartbeat_age = max(now_timestamp - last_heartbeat_timestamp, 0) if last_heartbeat_timestamp else float("inf") - if heartbeat_age > JUDGE_SERVER_SNAPSHOT_RETENTION_SECONDS: - remove_judge_server_snapshot(hostname) - continue - snapshots.append({ - "hostname": hostname, - "heartbeat_age": heartbeat_age, - "available": 1 if heartbeat_age <= 6 else 0, - "task_number": max(int(_float_or_zero(payload.get("task_number"))), 0), - }) - return snapshots - - -def _loads_snapshot(raw): - try: - return json.loads(_decode(raw)) - except (TypeError, ValueError): - return {} - - -def _decode(value): - if isinstance(value, bytes): - return value.decode() - return str(value) - - -def _float_or_zero(value): - try: - return float(value) - except (TypeError, ValueError): - return 0 diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 08695947..b590e616 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -16,7 +16,6 @@ CELERY_TASK_TOTAL_KEY, ) from utils.constants import CacheKey -from utils.judge_server_observability import load_judge_server_snapshots logger = logging.getLogger(__name__) @@ -81,21 +80,6 @@ def describe(self): "codeplace_celery_broker_queue_length", "Number of Celery tasks waiting in the Redis broker default queue.", ) - yield GaugeMetricFamily( - "codeplace_judge_server_available", - "Whether each judge-server is enabled and has a fresh heartbeat.", - labels=["hostname"], - ) - yield GaugeMetricFamily( - "codeplace_judge_server_last_heartbeat_age_seconds", - "Seconds since the latest judge-server heartbeat.", - labels=["hostname"], - ) - yield GaugeMetricFamily( - "codeplace_judge_server_task_number", - "Current task_number recorded for each judge-server.", - labels=["hostname"], - ) yield HistogramMetricFamily( "codeplace_submission_judge_duration_seconds", "Judge duration for submissions completed in the last 10 minutes.", @@ -142,7 +126,6 @@ def collect(self): self._collector_success = {} yield self._waiting_queue_length() yield self._celery_broker_queue_length() - yield from self._judge_server_metrics() yield self._judge_duration_histogram() yield from self._celery_task_metrics() yield from self._redis_health_metrics() @@ -178,36 +161,6 @@ def _celery_broker_queue_length(self): self._mark_collector_success("celery_broker_queue", True) return metric - def _judge_server_metrics(self): - available_metric = GaugeMetricFamily( - "codeplace_judge_server_available", - "Whether each judge-server is enabled and has a fresh heartbeat.", - labels=["hostname"], - ) - heartbeat_metric = GaugeMetricFamily( - "codeplace_judge_server_last_heartbeat_age_seconds", - "Seconds since the latest judge-server heartbeat.", - labels=["hostname"], - ) - task_metric = GaugeMetricFamily( - "codeplace_judge_server_task_number", - "Current task_number recorded for each judge-server.", - labels=["hostname"], - ) - snapshots = load_judge_server_snapshots() - if snapshots is None: - self._mark_collector_success("judge_server", False) - else: - self._mark_collector_success("judge_server", True) - for snapshot in snapshots or []: - hostname = snapshot["hostname"] or "unknown" - available_metric.add_metric([hostname], snapshot["available"]) - heartbeat_metric.add_metric([hostname], snapshot["heartbeat_age"]) - task_metric.add_metric([hostname], snapshot["task_number"]) - yield available_metric - yield heartbeat_metric - yield task_metric - def _judge_duration_histogram(self): metric = HistogramMetricFamily( "codeplace_submission_judge_duration_seconds", @@ -363,7 +316,6 @@ def _collector_success_metric(self): collectors = ( "waiting_queue", "celery_broker_queue", - "judge_server", "judge_duration", "celery_task", "redis", diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 69316f7e..17b77067 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -1,7 +1,6 @@ import json import logging import os -from datetime import timedelta from unittest.mock import patch, mock_open, MagicMock from django.test import SimpleTestCase, TestCase, override_settings from django.core.cache import cache @@ -12,10 +11,6 @@ from utils.json_logging import CodePlaceJsonFormatter from utils.observability_context import get_request_id, reset_request_id, set_request_id from utils.constants import CacheKey -from utils.judge_server_observability import ( - JUDGE_SERVER_SNAPSHOT_RETENTION_SECONDS, - increment_judge_server_task_snapshot, -) from utils.observability_metrics import CodePlaceCollector from utils import observability_tracing from utils.testcase_cache import TestCaseCacheManager @@ -26,71 +21,6 @@ class CodePlaceCollectorTest(SimpleTestCase): def _samples_by_metric(self, metrics): return {metric.name: metric.samples for metric in metrics} - def test_judge_server_metrics_emit_snapshot_servers(self): - now = timezone.now() - snapshots = { - b"enabled-judge": json.dumps({ - "last_heartbeat_timestamp": now.timestamp(), - "task_number": 2, - }).encode(), - } - - with patch("utils.judge_server_observability.cache.hgetall", return_value=snapshots): - samples = self._samples_by_metric(list(CodePlaceCollector()._judge_server_metrics())) - - for metric_samples in samples.values(): - hostnames = {sample.labels["hostname"] for sample in metric_samples} - self.assertIn("enabled-judge", hostnames) - self.assertNotIn("disabled-judge", hostnames) - - def test_judge_server_metrics_mark_stale_enabled_server_unavailable(self): - snapshots = { - b"stale-judge": json.dumps({ - "last_heartbeat_timestamp": (timezone.now() - timedelta(minutes=10)).timestamp(), - "task_number": 1, - }).encode(), - } - - with patch("utils.judge_server_observability.cache.hgetall", return_value=snapshots): - samples = self._samples_by_metric(list(CodePlaceCollector()._judge_server_metrics())) - - available = samples["codeplace_judge_server_available"][0] - heartbeat_age = samples["codeplace_judge_server_last_heartbeat_age_seconds"][0] - self.assertEqual(available.labels["hostname"], "stale-judge") - self.assertEqual(available.value, 0) - self.assertGreaterEqual(heartbeat_age.value, 599) - - def test_judge_server_metrics_are_loaded_from_redis_snapshot(self): - with patch("utils.judge_server_observability.cache.hgetall", return_value={}) as hgetall: - list(CodePlaceCollector()._judge_server_metrics()) - - hgetall.assert_called_once_with(CacheKey.judge_server_observability) - - def test_judge_server_metrics_prune_retired_snapshot(self): - retired_heartbeat = timezone.now() - timedelta(seconds=JUDGE_SERVER_SNAPSHOT_RETENTION_SECONDS + 1) - snapshots = { - b"retired-judge": json.dumps({ - "last_heartbeat_timestamp": retired_heartbeat.timestamp(), - "task_number": 0, - }).encode(), - } - - with patch("utils.judge_server_observability.cache.hgetall", return_value=snapshots), \ - patch("utils.judge_server_observability.cache.hdel") as hdel: - samples = self._samples_by_metric(list(CodePlaceCollector()._judge_server_metrics())) - - for metric_samples in samples.values(): - hostnames = {sample.labels["hostname"] for sample in metric_samples} - self.assertNotIn("retired-judge", hostnames) - hdel.assert_called_once_with(CacheKey.judge_server_observability, "retired-judge") - - def test_judge_server_task_increment_does_not_create_snapshot_without_heartbeat(self): - with patch("utils.judge_server_observability.cache.hget", return_value=None), \ - patch("utils.judge_server_observability.cache.hset") as hset: - increment_judge_server_task_snapshot("judge-1", 1) - - hset.assert_not_called() - def test_celery_broker_queue_length_is_collected_from_redis(self): with patch("utils.observability_metrics.cache.llen", return_value=7) as llen: samples = self._samples_by_metric([CodePlaceCollector()._celery_broker_queue_length()]) @@ -318,12 +248,6 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): "Number of Celery tasks waiting in the Redis broker default queue.", ) broker_queue_metric.add_metric([], 0) - judge_metric = GaugeMetricFamily( - "codeplace_judge_server_available", - "Whether each judge-server is enabled and has a fresh heartbeat.", - labels=["hostname"], - ) - judge_metric.add_metric(["judge-1"], 1) judge_duration_metric = HistogramMetricFamily( "codeplace_submission_judge_duration_seconds", "Judge duration for submissions completed in the last 10 minutes.", @@ -332,7 +256,6 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): with patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ patch.object(CodePlaceCollector, "_celery_broker_queue_length", return_value=broker_queue_metric), \ - patch.object(CodePlaceCollector, "_judge_server_metrics", return_value=[judge_metric]), \ patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric), \ patch.object(CodePlaceCollector, "_celery_task_metrics", return_value=[]), \ patch.object(CodePlaceCollector, "_redis_health_metrics", return_value=[]): @@ -343,7 +266,6 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): self.assertIn("django_http_requests_total_by_method_total", body) self.assertIn("codeplace_waiting_queue_length", body) self.assertIn("codeplace_celery_broker_queue_length", body) - self.assertIn("codeplace_judge_server_available", body) class ClientErrorReportAPITest(SimpleTestCase): diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 50b3f5d1..739cc7bc 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -37,9 +37,6 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_celery_broker_queue_length` - `codeplace_celery_task_total{task_name,status}` - `codeplace_celery_task_runtime_seconds{task_name}` -- `codeplace_judge_server_available{hostname}` -- `codeplace_judge_server_last_heartbeat_age_seconds{hostname}` -- `codeplace_judge_server_task_number{hostname}` - `codeplace_frontend_error_total{surface,error_type}` `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. @@ -112,8 +109,6 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `BackendTargetDown`: backend scrape 실패 1분 지속. - `Api5xxSpike`: backend 5xx 비율 5% 초과 2분 지속. - `Ingress5xxSpike`: Traefik 5xx 비율 5% 초과 2분 지속. -- `JudgeAllUnavailable`: 사용 가능한 judge-server 0개 1분 지속. -- `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. - `PostgresUnavailable`: PostgreSQL Pod not ready 또는 readiness metric missing 1분 지속. - `RedisUnavailable`: Redis/Sentinel Pod not ready 또는 readiness metric missing 1분 지속. - `LokiUnavailable`: Loki Pod not ready 1분 지속. @@ -141,7 +136,7 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. 4. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` 5. Prometheus target에서 `backend`, `postgres`, `redis`, `blackbox-exporter`, `loki`, `alloy` target이 healthy인지 확인합니다. -6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission/judge outcome, waiting queue, judge heartbeat panel을 확인합니다. +6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission/judge outcome, waiting queue panel을 확인합니다. 7. Grafana의 `CodePlace Logs` dashboard 또는 Explore에서 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. 8. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 280b1079..e08fd5ce 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -141,28 +141,6 @@ data: {"expr": "codeplace:judge_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} ] }, - { - "type": "stat", - "title": "Judge Available", - "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} - ] - } - } - }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "codeplace:judge_available_total{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} - ] - }, { "type": "timeseries", "title": "Backend Requests by Status", @@ -267,38 +245,6 @@ data: {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_submission_judge_duration_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} judge p95"} ] }, - { - "type": "timeseries", - "title": "Judge Heartbeat Max Age", - "gridPos": {"x": 8, "y": 20, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "green", "value": null}, - {"color": "orange", "value": 120}, - {"color": "red", "value": 180} - ] - } - } - }, - "targets": [ - {"expr": "max by (namespace) (codeplace_judge_server_last_heartbeat_age_seconds{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} max age"} - ] - }, - { - "type": "timeseries", - "title": "Judge Server Load", - "gridPos": {"x": 16, "y": 20, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "topk(10, codeplace_judge_server_task_number{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{hostname}} tasks"} - ] - }, { "type": "timeseries", "title": "Submission / Judge Outcomes", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 005821cb..f441a531 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -35,8 +35,6 @@ spec: namespace: code-place-prod - record: codeplace:public_endpoint_availability5m expr: avg by (environment, namespace, service, instance) (avg_over_time(probe_success{probe_type="public-http"}[5m])) - - record: codeplace:judge_available_total - expr: sum by (namespace) (codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) - record: codeplace:judge_waiting_queue_length expr: max by (namespace) (codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"}) - name: codeplace.p0 @@ -101,24 +99,6 @@ spec: annotations: summary: "Public endpoint is down in prod" description: "A production synthetic HTTPS probe has failed for 1 minute." - - alert: JudgeAllUnavailable - expr: (sum by (namespace) (codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) == 0) or (max by (namespace) (up{job="backend", namespace=~"code-place-(dev|prod)"}) unless on (namespace) codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) - for: 1m - labels: - severity: critical - priority: P0 - annotations: - summary: "No judge-server is available" - description: "No enabled judge-server has a fresh heartbeat." - - alert: JudgeHeartbeatCritical - expr: max by (namespace) (codeplace_judge_server_last_heartbeat_age_seconds{namespace=~"code-place-(dev|prod)"}) > 180 - for: 1m - labels: - severity: critical - priority: P0 - annotations: - summary: "Judge-server heartbeat is stale" - description: "A judge-server heartbeat is older than 180 seconds." - alert: PostgresUnavailable expr: (sum(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) for: 1m From e1ee6f5d84f7e17e52ae492d3a00c332b085600f Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 03:28:53 +0900 Subject: [PATCH 139/147] =?UTF-8?q?Redis=20=EB=88=84=EC=A0=81=20Celery=20?= =?UTF-8?q?=EC=83=81=EC=84=B8=20=EB=A9=94=ED=8A=B8=EB=A6=AD=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/oj/celery.py | 2 - backend/utils/celery_observability.py | 120 --------------- backend/utils/observability_metrics.py | 123 ---------------- backend/utils/tests.py | 139 ------------------ .../observability-improvement-plan/_index.md | 2 - .../grafana-dashboard-codeplace.yaml | 34 +---- kubernetes/monitoring/prometheus-rules.yaml | 54 ------- 7 files changed, 1 insertion(+), 473 deletions(-) delete mode 100644 backend/utils/celery_observability.py diff --git a/backend/oj/celery.py b/backend/oj/celery.py index f5ca162b..b9890384 100644 --- a/backend/oj/celery.py +++ b/backend/oj/celery.py @@ -1,7 +1,6 @@ import os import celery -from utils.celery_observability import configure_celery_observability from utils.observability_tracing import configure_opentelemetry os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oj.settings') @@ -10,4 +9,3 @@ app = celery.Celery('scheduler') app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks() -configure_celery_observability() diff --git a/backend/utils/celery_observability.py b/backend/utils/celery_observability.py deleted file mode 100644 index 5f64ec00..00000000 --- a/backend/utils/celery_observability.py +++ /dev/null @@ -1,120 +0,0 @@ -import logging -import time - -from celery import signals - -from utils.cache import cache -from utils.observability_context import get_request_id, reset_request_id, set_request_id - -logger = logging.getLogger("codeplace.celery_task") - -CELERY_TASK_TOTAL_KEY = "observability:celery:task_total" -CELERY_TASK_RUNTIME_BUCKET_KEY = "observability:celery:task_runtime_bucket" -CELERY_TASK_RUNTIME_COUNT_KEY = "observability:celery:task_runtime_count" -CELERY_TASK_RUNTIME_SUM_KEY = "observability:celery:task_runtime_sum" -CELERY_TASK_RUNTIME_LAST_KEY = "observability:celery:task_runtime_last" -CELERY_TASK_LAST_SEEN_AT_KEY = "observability:celery:task_last_seen_at" -CELERY_TASK_RUNTIME_BUCKETS = (1, 3, 5, 10, 30, 60, 120, 300, 600, float("inf")) - -_task_start_times = {} -_request_id_tokens = {} -_configured = False - - -def configure_celery_observability(): - global _configured - if _configured: - return - signals.before_task_publish.connect(_on_before_task_publish, weak=False) - signals.task_prerun.connect(_on_task_prerun, weak=False) - signals.task_postrun.connect(_on_task_postrun, weak=False) - signals.task_retry.connect(_on_task_retry, weak=False) - _configured = True - - -def _task_name(task): - return getattr(task, "name", None) or getattr(task, "__name__", None) or "unknown" - - -def _field(*parts): - return "|".join(str(part) for part in parts) - - -def _hincrbyfloat(key, field, amount): - client = cache.client.get_client(write=True) - return client.hincrbyfloat(key, field, amount) - - -def _hsetfloat(key, field, value): - client = cache.client.get_client(write=True) - return client.hset(key, field, value) - - -def _record_task(task_name, status, duration_seconds=None): - status = (status or "unknown").lower() - try: - cache.hincrby(CELERY_TASK_TOTAL_KEY, _field(task_name, status), 1) - _hsetfloat(CELERY_TASK_LAST_SEEN_AT_KEY, _field(task_name, status), time.time()) - if duration_seconds is not None: - _hincrbyfloat(CELERY_TASK_RUNTIME_SUM_KEY, task_name, duration_seconds) - cache.hincrby(CELERY_TASK_RUNTIME_COUNT_KEY, task_name, 1) - _hsetfloat(CELERY_TASK_RUNTIME_LAST_KEY, _field(task_name, status), duration_seconds) - for bucket in CELERY_TASK_RUNTIME_BUCKETS: - if duration_seconds <= bucket: - bucket_label = "+Inf" if bucket == float("inf") else str(bucket) - cache.hincrby(CELERY_TASK_RUNTIME_BUCKET_KEY, _field(task_name, bucket_label), 1) - except Exception: - logger.exception("failed to record celery task metric", extra={ - "task_name": task_name, - "task_status": status, - }) - - -def _on_before_task_publish(headers=None, **kwargs): - request_id = get_request_id() - if request_id and headers is not None: - headers["x-request-id"] = request_id - - -def _request_headers(task): - request = getattr(task, "request", None) - headers = getattr(request, "headers", None) or {} - return headers if isinstance(headers, dict) else {} - - -def _on_task_prerun(task_id=None, task=None, **kwargs): - if task_id: - _task_start_times[task_id] = time.monotonic() - request_id = _request_headers(task).get("x-request-id") - if request_id and task_id: - _request_id_tokens[task_id] = set_request_id(request_id) - - -def _on_task_postrun(task_id=None, task=None, state=None, **kwargs): - task_name = _task_name(task) - started_at = _task_start_times.pop(task_id, None) - duration_seconds = None - if started_at is not None: - duration_seconds = max(time.monotonic() - started_at, 0) - _record_task(task_name, state, duration_seconds) - logger.info("celery task completed", extra={ - "task_id": task_id, - "task_name": task_name, - "task_status": (state or "unknown").lower(), - "duration_ms": round(duration_seconds * 1000, 2) if duration_seconds is not None else None, - }) - token = _request_id_tokens.pop(task_id, None) - if token is not None: - reset_request_id(token) - - -def _on_task_retry(request=None, reason=None, **kwargs): - task_name = getattr(request, "task", None) or "unknown" - task_id = getattr(request, "id", None) - _record_task(task_name, "retry") - logger.warning("celery task retry", extra={ - "task_id": task_id, - "task_name": task_name, - "task_status": "retry", - "retry_reason": str(reason) if reason else None, - }) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index b590e616..723a5860 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -1,20 +1,9 @@ import logging -from datetime import timedelta -from django.utils import timezone from prometheus_client import Counter, Histogram, REGISTRY from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, HistogramMetricFamily from utils.cache import cache -from utils.celery_observability import ( - CELERY_TASK_RUNTIME_BUCKET_KEY, - CELERY_TASK_RUNTIME_BUCKETS, - CELERY_TASK_RUNTIME_COUNT_KEY, - CELERY_TASK_LAST_SEEN_AT_KEY, - CELERY_TASK_RUNTIME_LAST_KEY, - CELERY_TASK_RUNTIME_SUM_KEY, - CELERY_TASK_TOTAL_KEY, -) from utils.constants import CacheKey logger = logging.getLogger(__name__) @@ -84,21 +73,6 @@ def describe(self): "codeplace_submission_judge_duration_seconds", "Judge duration for submissions completed in the last 10 minutes.", ) - yield CounterMetricFamily( - "codeplace_celery_task_total", - "Total Celery tasks grouped by task name and status.", - labels=["task_name", "status"], - ) - yield HistogramMetricFamily( - "codeplace_celery_task_runtime_seconds", - "Celery task runtime grouped by task name.", - labels=["task_name"], - ) - yield GaugeMetricFamily( - "codeplace_celery_task_last_runtime_seconds", - "Latest observed Celery task runtime grouped by task name and status.", - labels=["task_name", "status"], - ) yield GaugeMetricFamily( "codeplace_redis_connected_clients", "Current Redis connected clients from INFO clients.", @@ -111,11 +85,6 @@ def describe(self): "codeplace_redis_rejected_connections_total", "Total Redis rejected connections from INFO stats.", ) - yield GaugeMetricFamily( - "codeplace_celery_task_last_success_age_seconds", - "Seconds since each Celery task last completed successfully.", - labels=["task_name"], - ) yield GaugeMetricFamily( "codeplace_observability_collector_success", "Whether each CodePlace custom metrics collector completed successfully.", @@ -127,7 +96,6 @@ def collect(self): yield self._waiting_queue_length() yield self._celery_broker_queue_length() yield self._judge_duration_histogram() - yield from self._celery_task_metrics() yield from self._redis_health_metrics() yield self._collector_success_metric() @@ -184,85 +152,6 @@ def _judge_duration_histogram(self): metric.add_metric([], cumulative_buckets, duration_sum) return metric - def _celery_task_metrics(self): - total_metric = CounterMetricFamily( - "codeplace_celery_task_total", - "Total Celery tasks grouped by task name and status.", - labels=["task_name", "status"], - ) - runtime_metric = HistogramMetricFamily( - "codeplace_celery_task_runtime_seconds", - "Celery task runtime grouped by task name.", - labels=["task_name"], - ) - last_runtime_metric = GaugeMetricFamily( - "codeplace_celery_task_last_runtime_seconds", - "Latest observed Celery task runtime grouped by task name and status.", - labels=["task_name", "status"], - ) - last_success_age_metric = GaugeMetricFamily( - "codeplace_celery_task_last_success_age_seconds", - "Seconds since each Celery task last completed successfully.", - labels=["task_name"], - ) - try: - task_totals = self._redis_hash(CELERY_TASK_TOTAL_KEY) - runtime_buckets = self._redis_hash(CELERY_TASK_RUNTIME_BUCKET_KEY) - runtime_counts = self._redis_hash(CELERY_TASK_RUNTIME_COUNT_KEY) - runtime_sums = self._redis_hash(CELERY_TASK_RUNTIME_SUM_KEY) - last_runtimes = self._redis_hash(CELERY_TASK_RUNTIME_LAST_KEY) - last_seen_at = self._redis_hash(CELERY_TASK_LAST_SEEN_AT_KEY) - except Exception as e: - logger.warning("Failed to collect Celery task metrics: %s", e) - self._mark_collector_success("celery_task", False) - yield total_metric - yield runtime_metric - yield last_runtime_metric - yield last_success_age_metric - return - else: - self._mark_collector_success("celery_task", True) - - for field, value in task_totals.items(): - task_name, status = self._split_field(field, 2) - total_metric.add_metric([task_name, status], self._float_or_zero(value)) - - task_names = set(runtime_counts) | set(runtime_sums) - for field in runtime_buckets: - task_name, _ = self._split_field(field, 2) - task_names.add(task_name) - for task_name in sorted(task_names): - cumulative_buckets = [] - for bucket in CELERY_TASK_RUNTIME_BUCKETS: - bucket_label = "+Inf" if bucket == float("inf") else str(bucket) - value = self._float_or_zero(runtime_buckets.get(self._join_field(task_name, bucket_label), 0)) - cumulative_buckets.append((bucket_label, value)) - runtime_metric.add_metric( - [task_name], - cumulative_buckets, - self._float_or_zero(runtime_sums.get(task_name, 0)), - ) - - for field, value in last_runtimes.items(): - task_name, status = self._split_field(field, 2) - last_runtime_metric.add_metric([task_name, status], self._float_or_zero(value)) - - now_timestamp = timezone.now().timestamp() - for field, value in last_seen_at.items(): - task_name, status = self._split_field(field, 2) - seen_timestamp = self._float_or_none(value) - if seen_timestamp is None: - logger.warning("Skipping invalid Celery task last_seen_at metric: %s=%s", field, value) - continue - age_seconds = max(now_timestamp - seen_timestamp, 0) - if status == "success": - last_success_age_metric.add_metric([task_name], age_seconds) - - yield total_metric - yield runtime_metric - yield last_runtime_metric - yield last_success_age_metric - def _redis_health_metrics(self): connected_clients_metric = GaugeMetricFamily( "codeplace_redis_connected_clients", @@ -317,7 +206,6 @@ def _collector_success_metric(self): "waiting_queue", "celery_broker_queue", "judge_duration", - "celery_task", "redis", ) for collector in collectors: @@ -343,17 +231,6 @@ def _decode(value): return value.decode() return str(value) - @staticmethod - def _split_field(field, expected_parts): - parts = field.split("|", expected_parts - 1) - if len(parts) != expected_parts: - return tuple(parts + ["unknown"] * (expected_parts - len(parts))) - return tuple(parts) - - @staticmethod - def _join_field(*parts): - return "|".join(str(part) for part in parts) - @staticmethod def _float_or_zero(value): parsed = CodePlaceCollector._float_or_none(value) diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 17b77067..26aebe07 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -4,12 +4,9 @@ from unittest.mock import patch, mock_open, MagicMock from django.test import SimpleTestCase, TestCase, override_settings from django.core.cache import cache -from django.utils import timezone from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily -from utils import celery_observability from utils.json_logging import CodePlaceJsonFormatter -from utils.observability_context import get_request_id, reset_request_id, set_request_id from utils.constants import CacheKey from utils.observability_metrics import CodePlaceCollector from utils import observability_tracing @@ -47,84 +44,6 @@ def test_judge_duration_histogram_is_collected_from_redis(self): for sample in samples["codeplace_submission_judge_duration_seconds"] )) - def test_celery_task_metrics_are_collected_from_redis(self): - hashes = { - celery_observability.CELERY_TASK_TOTAL_KEY: { - b"judge.tasks.judge_task|success": b"3", - b"judge.tasks.judge_task|failure": b"1", - }, - celery_observability.CELERY_TASK_RUNTIME_BUCKET_KEY: { - b"judge.tasks.judge_task|1": b"2", - b"judge.tasks.judge_task|3": b"3", - b"judge.tasks.judge_task|+Inf": b"3", - }, - celery_observability.CELERY_TASK_RUNTIME_COUNT_KEY: { - b"judge.tasks.judge_task": b"3", - }, - celery_observability.CELERY_TASK_RUNTIME_SUM_KEY: { - b"judge.tasks.judge_task": b"4.5", - }, - celery_observability.CELERY_TASK_RUNTIME_LAST_KEY: { - b"judge.tasks.judge_task|success": b"1.5", - }, - celery_observability.CELERY_TASK_LAST_SEEN_AT_KEY: { - b"judge.tasks.judge_task|success": str(timezone.now().timestamp() - 30).encode(), - b"judge.tasks.judge_task|failure": str(timezone.now().timestamp() - 10).encode(), - }, - } - - with patch("utils.observability_metrics.cache.hgetall", side_effect=lambda key: hashes.get(key, {})): - samples = self._samples_by_metric(list(CodePlaceCollector()._celery_task_metrics())) - - total_samples = samples["codeplace_celery_task"] - self.assertEqual(total_samples[0].name, "codeplace_celery_task_total") - self.assertEqual(total_samples[0].labels, { - "task_name": "judge.tasks.judge_task", - "status": "success", - }) - self.assertEqual(total_samples[0].value, 3) - runtime_samples = samples["codeplace_celery_task_runtime_seconds"] - bucket_samples = [sample for sample in runtime_samples if sample.name.endswith("_bucket")] - self.assertTrue(any(sample.labels["le"] == "+Inf" and sample.value == 3 for sample in bucket_samples)) - self.assertTrue(any(sample.name.endswith("_sum") and sample.value == 4.5 for sample in runtime_samples)) - last_success_samples = samples["codeplace_celery_task_last_success_age_seconds"] - self.assertEqual(last_success_samples[0].labels, {"task_name": "judge.tasks.judge_task"}) - self.assertGreaterEqual(last_success_samples[0].value, 29) - last_runtime = samples["codeplace_celery_task_last_runtime_seconds"][0] - self.assertEqual(last_runtime.value, 1.5) - - def test_celery_task_metrics_tolerate_malformed_redis_values(self): - hashes = { - celery_observability.CELERY_TASK_TOTAL_KEY: { - b"judge.tasks.judge_task|success": b"bad-count", - }, - celery_observability.CELERY_TASK_RUNTIME_BUCKET_KEY: { - b"judge.tasks.judge_task|1": b"bad-bucket", - }, - celery_observability.CELERY_TASK_RUNTIME_COUNT_KEY: { - b"judge.tasks.judge_task": b"bad-count", - }, - celery_observability.CELERY_TASK_RUNTIME_SUM_KEY: { - b"judge.tasks.judge_task": b"bad-sum", - }, - celery_observability.CELERY_TASK_RUNTIME_LAST_KEY: { - b"judge.tasks.judge_task|success": b"bad-runtime", - }, - celery_observability.CELERY_TASK_LAST_SEEN_AT_KEY: { - b"judge.tasks.judge_task|success": b"bad-timestamp", - }, - } - - with patch("utils.observability_metrics.cache.hgetall", side_effect=lambda key: hashes.get(key, {})): - samples = self._samples_by_metric(list(CodePlaceCollector()._celery_task_metrics())) - - total_sample = samples["codeplace_celery_task"][0] - self.assertEqual(total_sample.name, "codeplace_celery_task_total") - self.assertEqual(total_sample.value, 0) - last_runtime_sample = samples["codeplace_celery_task_last_runtime_seconds"][0] - self.assertEqual(last_runtime_sample.value, 0) - self.assertEqual(samples["codeplace_celery_task_last_success_age_seconds"], []) - def test_redis_health_metrics_are_collected(self): redis_client = MagicMock() redis_client.info.return_value = { @@ -178,63 +97,6 @@ def test_redis_collector_failure_is_reported_when_maxclients_is_unavailable(self self.assertEqual(redis_success.value, 0) -class CeleryObservabilityTest(SimpleTestCase): - - def tearDown(self): - celery_observability._task_start_times.clear() - celery_observability._request_id_tokens.clear() - - def test_record_task_writes_total_and_runtime_hashes(self): - redis_client = MagicMock() - with patch("utils.celery_observability.cache.hincrby") as hincrby, \ - patch("utils.celery_observability.cache.client.get_client", return_value=redis_client): - celery_observability._record_task("judge.tasks.judge_task", "success", 2.5) - - hincrby.assert_any_call( - celery_observability.CELERY_TASK_TOTAL_KEY, - "judge.tasks.judge_task|success", - 1, - ) - hincrby.assert_any_call( - celery_observability.CELERY_TASK_RUNTIME_COUNT_KEY, - "judge.tasks.judge_task", - 1, - ) - redis_client.hincrbyfloat.assert_any_call( - celery_observability.CELERY_TASK_RUNTIME_SUM_KEY, - "judge.tasks.judge_task", - 2.5, - ) - redis_client.hset.assert_any_call( - celery_observability.CELERY_TASK_RUNTIME_LAST_KEY, - "judge.tasks.judge_task|success", - 2.5, - ) - - def test_before_task_publish_propagates_current_request_id(self): - token = set_request_id("request-123") - headers = {} - try: - celery_observability._on_before_task_publish(headers=headers) - finally: - reset_request_id(token) - - self.assertEqual(headers["x-request-id"], "request-123") - - def test_task_prerun_and_postrun_restore_request_id_context(self): - task = MagicMock() - task.name = "judge.tasks.judge_task" - task.request.headers = {"x-request-id": "request-456"} - - with patch("utils.celery_observability._record_task"), \ - patch("utils.celery_observability.logger"): - celery_observability._on_task_prerun(task_id="task-1", task=task) - self.assertEqual(get_request_id(), "request-456") - celery_observability._on_task_postrun(task_id="task-1", task=task, state="SUCCESS") - - self.assertIsNone(get_request_id()) - - class CodePlaceMetricsEndpointTest(SimpleTestCase): def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): @@ -257,7 +119,6 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): with patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ patch.object(CodePlaceCollector, "_celery_broker_queue_length", return_value=broker_queue_metric), \ patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric), \ - patch.object(CodePlaceCollector, "_celery_task_metrics", return_value=[]), \ patch.object(CodePlaceCollector, "_redis_health_metrics", return_value=[]): response = self.client.get("/metrics") diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 739cc7bc..221f00f8 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -35,8 +35,6 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_judge_task_outcome_total{status,scope}` - `codeplace_waiting_queue_length` - `codeplace_celery_broker_queue_length` -- `codeplace_celery_task_total{task_name,status}` -- `codeplace_celery_task_runtime_seconds{task_name}` - `codeplace_frontend_error_total{surface,error_type}` `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index e08fd5ce..f91f3666 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -198,43 +198,11 @@ data: }, { "type": "row", - "title": "Async Jobs", + "title": "Judge / Async Outcomes", "gridPos": {"x": 0, "y": 12, "w": 24, "h": 1}, "collapsed": false, "panels": [] }, - { - "type": "timeseries", - "title": "Celery Task Throughput", - "gridPos": {"x": 0, "y": 13, "w": 8, "h": 7}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, - "targets": [ - {"expr": "sum by (namespace, task_name, status) (rate(codeplace_celery_task_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{task_name}} {{status}}"} - ] - }, - { - "type": "timeseries", - "title": "Celery Task Runtime / Freshness", - "gridPos": {"x": 8, "y": 13, "w": 8, "h": 7}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "s"}}, - "targets": [ - {"expr": "histogram_quantile(0.95, sum by (namespace, task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} {{task_name}} p95"}, - {"expr": "codeplace_celery_task_last_success_age_seconds{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{task_name}} last success age"} - ] - }, - { - "type": "bargauge", - "title": "Celery Last Runtime", - "gridPos": {"x": 16, "y": 13, "w": 8, "h": 7}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "s"}}, - "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "codeplace_celery_task_last_runtime_seconds{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{task_name}} {{status}}"} - ] - }, { "type": "timeseries", "title": "Judge Duration p95", diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index f441a531..57332b24 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -291,60 +291,6 @@ spec: annotations: summary: "Celery beat is not ready" description: "The celery-beat Deployment has no available replica for 2 minutes, or its availability metric is missing." - - alert: CeleryTaskFailures - expr: sum by (namespace, task_name) (increase(codeplace_celery_task_total{namespace=~"code-place-(dev|prod)", status="failure"}[10m])) > 0 - for: 2m - labels: - severity: warning - priority: P1 - annotations: - summary: "Celery task failures detected" - description: "A Celery task has failed in the last 10 minutes." - - alert: CeleryTaskRetries - expr: sum by (namespace, task_name) (increase(codeplace_celery_task_total{namespace=~"code-place-(dev|prod)", status="retry"}[10m])) > 3 - for: 5m - labels: - severity: warning - priority: P1 - annotations: - summary: "Celery task retries are elevated" - description: "A Celery task retried more than 3 times in 10 minutes." - - alert: CeleryTaskRuntimeHigh - expr: histogram_quantile(0.95, sum by (namespace, task_name, le) (rate(codeplace_celery_task_runtime_seconds_bucket{namespace=~"code-place-(dev|prod)"}[10m]))) > 120 - for: 10m - labels: - severity: warning - priority: P1 - annotations: - summary: "Celery task runtime is high" - description: "Celery task p95 runtime has been above 120 seconds for 10 minutes." - - alert: CeleryMinuteScheduledTaskStale - expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_fluctuation"} > 600 - for: 5m - labels: - severity: warning - priority: P1 - annotations: - summary: "Minute scheduled Celery task is stale" - description: "calculate_user_score_fluctuation has not completed successfully for more than 10 minutes, or no success metric has ever been observed. Check celery-beat, celery-worker, Redis broker, and task logs." - - alert: CeleryDailyScheduledTaskStale - expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name="account.tasks.calculate_user_score_basis"} > 93600 - for: 30m - labels: - severity: warning - priority: P1 - annotations: - summary: "Daily scheduled Celery task is stale" - description: "calculate_user_score_basis has not completed successfully for more than 26 hours, or no success metric has ever been observed. Check celery-beat, celery-worker, Redis broker, and task logs." - - alert: CeleryWeeklyScheduledTaskStale - expr: codeplace_celery_task_last_success_age_seconds{namespace=~"code-place-(dev|prod)", task_name=~"problem.tasks.update_weekly_stats|problem.tasks.update_bonus_problem"} > 691200 - for: 1h - labels: - severity: warning - priority: P1 - annotations: - summary: "Weekly scheduled Celery task is stale" - description: "A weekly problem maintenance task has not completed successfully for more than 8 days, or no success metric has ever been observed. Check celery-beat, celery-worker, Redis broker, and task logs." - alert: PodCrashLooping expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|vllm-.*"}[5m])) > 0 for: 5m From bc8f42ec2755b79144fda5afa13673aff3aaea04 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 03:35:47 +0900 Subject: [PATCH 140/147] =?UTF-8?q?Sentry=20=EB=A6=B4=EB=A6=AC=EC=A6=88=20?= =?UTF-8?q?=EA=B0=92=EC=9D=84=20=EB=B0=B0=ED=8F=AC=20SHA=EB=A1=9C=20?= =?UTF-8?q?=EA=B0=B1=EC=8B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci2develop.yml | 23 ++++++++++++++++++- .../workflows/ci2production-and-release.yml | 23 ++++++++++++++++++- .../dev/backend-deployment-patch.yaml | 2 ++ .../dev/celery-beat-deployment-patch.yaml | 2 ++ .../dev/celery-worker-deployment-patch.yaml | 2 ++ .../prod/backend-deployment-patch.yaml | 2 ++ .../prod/celery-beat-deployment-patch.yaml | 2 ++ .../prod/celery-worker-deployment-patch.yaml | 2 ++ 8 files changed, 56 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index b6fe14fd..57755c8d 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -136,6 +136,24 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml + - name: Update Backend Sentry Release + if: needs.ci-backend-dev.result == 'success' + uses: mikefarah/yq@master + with: + cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/backend-deployment-patch.yaml + + - name: Update Celery Worker Sentry Release + if: needs.ci-backend-dev.result == 'success' + uses: mikefarah/yq@master + with: + cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-worker-deployment-patch.yaml + + - name: Update Celery Beat Sentry Release + if: needs.ci-backend-dev.result == 'success' + uses: mikefarah/yq@master + with: + cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-beat-deployment-patch.yaml + - name: Update Frontend Image Tag if: needs.ci-frontend-dev.result == 'success' uses: mikefarah/yq@master @@ -153,6 +171,9 @@ jobs: git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' git add kubernetes/overlays/dev/kustomization.yaml + git add kubernetes/overlays/dev/backend-deployment-patch.yaml + git add kubernetes/overlays/dev/celery-worker-deployment-patch.yaml + git add kubernetes/overlays/dev/celery-beat-deployment-patch.yaml # 변경사항이 없으면 종료 if git diff-index --quiet HEAD; then @@ -162,6 +183,6 @@ jobs: # 충돌 방지를 위해 최신 develop 브랜치로 rebase 후 push # [skip ci] 태그를 커밋 메시지에 추가하여 추가 CI 실행을 방지 - git commit -m "ci: Update dev image tags to ${{ github.sha }} [skip ci]" + git commit -m "ci: Update dev image tags and Sentry release to ${{ github.sha }} [skip ci]" git pull --rebase origin ${{ github.ref_name }} git push origin ${{ github.ref_name }} diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index 28e3db59..72b1ff78 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -165,6 +165,24 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml + - name: Update Backend Sentry Release + if: needs.ci-backend.result == 'success' + uses: mikefarah/yq@master + with: + cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/backend-deployment-patch.yaml + + - name: Update Celery Worker Sentry Release + if: needs.ci-backend.result == 'success' + uses: mikefarah/yq@master + with: + cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-worker-deployment-patch.yaml + + - name: Update Celery Beat Sentry Release + if: needs.ci-backend.result == 'success' + uses: mikefarah/yq@master + with: + cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-beat-deployment-patch.yaml + - name: Update Frontend Image Tag if: needs.ci-frontend.result == 'success' uses: mikefarah/yq@master @@ -182,6 +200,9 @@ jobs: git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' git add kubernetes/overlays/prod/kustomization.yaml + git add kubernetes/overlays/prod/backend-deployment-patch.yaml + git add kubernetes/overlays/prod/celery-worker-deployment-patch.yaml + git add kubernetes/overlays/prod/celery-beat-deployment-patch.yaml # 변경사항이 없으면 종료 if git diff-index --quiet HEAD; then @@ -190,7 +211,7 @@ jobs: fi # [skip ci] 태그를 커밋 메시지에 추가하여 추가 CI 실행을 방지 - git commit -m "ci: Update prod image tags to ${{ github.sha }} [skip ci]" + git commit -m "ci: Update prod image tags and Sentry release to ${{ github.sha }} [skip ci]" git pull --rebase origin main git push origin main diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index 78acaaa2..7a98516d 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -17,5 +17,7 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: SENTRY_RELEASE + value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index 8bbe2697..3712eff3 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -20,5 +20,7 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: SENTRY_RELEASE + value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index 82089303..0a8b217a 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -23,5 +23,7 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: SENTRY_RELEASE + value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/prod/backend-deployment-patch.yaml b/kubernetes/overlays/prod/backend-deployment-patch.yaml index de88b416..e4468851 100644 --- a/kubernetes/overlays/prod/backend-deployment-patch.yaml +++ b/kubernetes/overlays/prod/backend-deployment-patch.yaml @@ -24,3 +24,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" + - name: SENTRY_RELEASE + value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml index db60527a..bb4e4f49 100644 --- a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml @@ -20,3 +20,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" + - name: SENTRY_RELEASE + value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml index f98ee708..cb654565 100644 --- a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml @@ -23,3 +23,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" + - name: SENTRY_RELEASE + value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" From 66bf6adf1537c63348876cb320a8d13b288b7588 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 03:38:07 +0900 Subject: [PATCH 141/147] =?UTF-8?q?Sentry=20=ED=94=84=EB=A1=A0=ED=8A=B8?= =?UTF-8?q?=EC=97=94=EB=93=9C=20=EB=B9=8C=EB=93=9C=20=EC=84=A4=EC=A0=95=20?= =?UTF-8?q?=EC=97=B0=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci2develop.yml | 2 ++ .github/workflows/ci2production-and-release.yml | 4 +++- backend/oj/settings.py | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index 57755c8d..9abd20a6 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -87,6 +87,8 @@ jobs: SERVER_NAME=${{ secrets.DEV_SERVER_NAME }} APP_VERSION=${{ github.sha }}-dev SENTRY_ENVIRONMENT=development + SENTRY_DSN=${{ secrets.SENTRY_DSN }} + USE_SENTRY=1 ci-hub-auth-dev: needs: [detect-changes-by-component] diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index 72b1ff78..d4b3e5b2 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -113,8 +113,10 @@ jobs: provenance: false build-args: | SERVER_NAME=${{ secrets.PROD_SERVER_NAME }} - APP_VERSION=${{ github.sha }} + APP_VERSION=${{ github.sha }}-prod SENTRY_ENVIRONMENT=production + SENTRY_DSN=${{ secrets.SENTRY_DSN }} + USE_SENTRY=1 ci-hub-auth: needs: [detect-changes-by-component] diff --git a/backend/oj/settings.py b/backend/oj/settings.py index 26c624f3..1861ccd9 100644 --- a/backend/oj/settings.py +++ b/backend/oj/settings.py @@ -15,6 +15,8 @@ import celery.schedules import sentry_sdk +from sentry_sdk.integrations.celery import CeleryIntegration +from sentry_sdk.integrations.django import DjangoIntegration production_env = get_env("OJ_ENV", "dev") == "production" if production_env: @@ -82,6 +84,10 @@ def _before_send_sentry(event, hint): "send_default_pii": False, "environment": get_env("SENTRY_ENVIRONMENT", get_env("OJ_ENV", "dev")), "before_send": _before_send_sentry, + "integrations": [ + DjangoIntegration(), + CeleryIntegration(), + ], } SENTRY_RELEASE = get_env("SENTRY_RELEASE", "") if SENTRY_RELEASE: From 4c102ecccc6da598981470ab0885c4afbdb508b5 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 03:42:39 +0900 Subject: [PATCH 142/147] =?UTF-8?q?Sentry=20=EB=A6=B4=EB=A6=AC=EC=A6=88=20?= =?UTF-8?q?=EC=84=A4=EC=A0=95=20=EB=8B=A8=EC=9D=BC=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci2develop.yml | 17 +---------------- .github/workflows/ci2production-and-release.yml | 17 +---------------- .../overlays/dev/backend-deployment-patch.yaml | 5 +++-- .../dev/celery-beat-deployment-patch.yaml | 5 +++-- .../dev/celery-worker-deployment-patch.yaml | 5 +++-- kubernetes/overlays/dev/kustomization.yaml | 6 ++++++ .../overlays/prod/backend-deployment-patch.yaml | 5 +++-- .../prod/celery-beat-deployment-patch.yaml | 5 +++-- .../prod/celery-worker-deployment-patch.yaml | 5 +++-- kubernetes/overlays/prod/kustomization.yaml | 7 +++++++ 10 files changed, 33 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index 9abd20a6..565130b9 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -142,19 +142,7 @@ jobs: if: needs.ci-backend-dev.result == 'success' uses: mikefarah/yq@master with: - cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/backend-deployment-patch.yaml - - - name: Update Celery Worker Sentry Release - if: needs.ci-backend-dev.result == 'success' - uses: mikefarah/yq@master - with: - cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-worker-deployment-patch.yaml - - - name: Update Celery Beat Sentry Release - if: needs.ci-backend-dev.result == 'success' - uses: mikefarah/yq@master - with: - cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-beat-deployment-patch.yaml + cmd: yq -i '(.configMapGenerator[] | select(.name == "sentry-release").literals[] | select(test("^SENTRY_RELEASE="))) = "SENTRY_RELEASE=${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml - name: Update Frontend Image Tag if: needs.ci-frontend-dev.result == 'success' @@ -173,9 +161,6 @@ jobs: git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' git add kubernetes/overlays/dev/kustomization.yaml - git add kubernetes/overlays/dev/backend-deployment-patch.yaml - git add kubernetes/overlays/dev/celery-worker-deployment-patch.yaml - git add kubernetes/overlays/dev/celery-beat-deployment-patch.yaml # 변경사항이 없으면 종료 if git diff-index --quiet HEAD; then diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index d4b3e5b2..175be2b7 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -171,19 +171,7 @@ jobs: if: needs.ci-backend.result == 'success' uses: mikefarah/yq@master with: - cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/backend-deployment-patch.yaml - - - name: Update Celery Worker Sentry Release - if: needs.ci-backend.result == 'success' - uses: mikefarah/yq@master - with: - cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-worker-deployment-patch.yaml - - - name: Update Celery Beat Sentry Release - if: needs.ci-backend.result == 'success' - uses: mikefarah/yq@master - with: - cmd: yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-beat-deployment-patch.yaml + cmd: yq -i '(.configMapGenerator[] | select(.name == "sentry-release").literals[] | select(test("^SENTRY_RELEASE="))) = "SENTRY_RELEASE=${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml - name: Update Frontend Image Tag if: needs.ci-frontend.result == 'success' @@ -202,9 +190,6 @@ jobs: git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' git add kubernetes/overlays/prod/kustomization.yaml - git add kubernetes/overlays/prod/backend-deployment-patch.yaml - git add kubernetes/overlays/prod/celery-worker-deployment-patch.yaml - git add kubernetes/overlays/prod/celery-beat-deployment-patch.yaml # 변경사항이 없으면 종료 if git diff-index --quiet HEAD; then diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index 7a98516d..583b1e70 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -7,6 +7,9 @@ spec: spec: containers: - name: backend + envFrom: + - configMapRef: + name: sentry-release env: - name: CSRF_TRUSTED_ORIGINS value: "https://k3s.code-place-dev.site" @@ -17,7 +20,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" - - name: SENTRY_RELEASE - value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index 3712eff3..be25bd66 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -12,6 +12,9 @@ spec: spec: containers: - name: celery-beat + envFrom: + - configMapRef: + name: sentry-release env: - name: SENTRY_DSN valueFrom: @@ -20,7 +23,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" - - name: SENTRY_RELEASE - value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index 0a8b217a..65f536d2 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -15,6 +15,9 @@ spec: spec: containers: - name: celery-worker + envFrom: + - configMapRef: + name: sentry-release env: - name: SENTRY_DSN valueFrom: @@ -23,7 +26,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" - - name: SENTRY_RELEASE - value: "82d1bcd221199cd9ad9beb930239a58b38340cba-dev" - name: OTEL_ENABLED value: "1" diff --git a/kubernetes/overlays/dev/kustomization.yaml b/kubernetes/overlays/dev/kustomization.yaml index 60142883..573126b5 100644 --- a/kubernetes/overlays/dev/kustomization.yaml +++ b/kubernetes/overlays/dev/kustomization.yaml @@ -7,6 +7,12 @@ resources: - ./secrets/common-credentials.yaml - ./secrets/sentry-secret.yaml namespace: code-place-dev +configMapGenerator: + - name: sentry-release + literals: + - SENTRY_RELEASE=82d1bcd221199cd9ad9beb930239a58b38340cba-dev +generatorOptions: + disableNameSuffixHash: true patches: - path: backend-deployment-patch.yaml - path: frontend-deployment-patch.yaml diff --git a/kubernetes/overlays/prod/backend-deployment-patch.yaml b/kubernetes/overlays/prod/backend-deployment-patch.yaml index e4468851..af94ca92 100644 --- a/kubernetes/overlays/prod/backend-deployment-patch.yaml +++ b/kubernetes/overlays/prod/backend-deployment-patch.yaml @@ -13,6 +13,9 @@ spec: spec: containers: - name: backend + envFrom: + - configMapRef: + name: sentry-release imagePullPolicy: IfNotPresent env: - name: CSRF_TRUSTED_ORIGINS @@ -24,5 +27,3 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" - - name: SENTRY_RELEASE - value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml index bb4e4f49..7b8128bb 100644 --- a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml @@ -12,6 +12,9 @@ spec: spec: containers: - name: celery-beat + envFrom: + - configMapRef: + name: sentry-release env: - name: SENTRY_DSN valueFrom: @@ -20,5 +23,3 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" - - name: SENTRY_RELEASE - value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml index cb654565..a81dbfbd 100644 --- a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml @@ -15,6 +15,9 @@ spec: spec: containers: - name: celery-worker + envFrom: + - configMapRef: + name: sentry-release env: - name: SENTRY_DSN valueFrom: @@ -23,5 +26,3 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "prod" - - name: SENTRY_RELEASE - value: "3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod" diff --git a/kubernetes/overlays/prod/kustomization.yaml b/kubernetes/overlays/prod/kustomization.yaml index ab415ea4..3cbd1e6e 100644 --- a/kubernetes/overlays/prod/kustomization.yaml +++ b/kubernetes/overlays/prod/kustomization.yaml @@ -10,6 +10,13 @@ resources: namespace: code-place-prod +configMapGenerator: + - name: sentry-release + literals: + - SENTRY_RELEASE=3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod +generatorOptions: + disableNameSuffixHash: true + patches: - path: backend-deployment-patch.yaml - path: frontend-deployment-patch.yaml From def0ed6c7a6952f81360d7426960088f0208278d Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 03:49:19 +0900 Subject: [PATCH 143/147] =?UTF-8?q?Sentry=20=EB=A6=B4=EB=A6=AC=EC=A6=88?= =?UTF-8?q?=EB=A5=BC=20APP=5FVERSION=EC=9C=BC=EB=A1=9C=20=ED=86=B5?= =?UTF-8?q?=EC=9D=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci2develop.yml | 6 +++--- .github/workflows/ci2production-and-release.yml | 6 +++--- backend/oj/settings.py | 6 +++--- frontend/Dockerfile | 1 + frontend/config/dev.env.js | 2 +- kubernetes/overlays/dev/backend-deployment-patch.yaml | 2 +- kubernetes/overlays/dev/celery-beat-deployment-patch.yaml | 2 +- kubernetes/overlays/dev/celery-worker-deployment-patch.yaml | 2 +- kubernetes/overlays/dev/kustomization.yaml | 4 ++-- kubernetes/overlays/prod/backend-deployment-patch.yaml | 2 +- kubernetes/overlays/prod/celery-beat-deployment-patch.yaml | 2 +- .../overlays/prod/celery-worker-deployment-patch.yaml | 2 +- kubernetes/overlays/prod/kustomization.yaml | 4 ++-- 13 files changed, 21 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index 565130b9..45dbad4e 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -138,11 +138,11 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml - - name: Update Backend Sentry Release + - name: Update Backend App Version if: needs.ci-backend-dev.result == 'success' uses: mikefarah/yq@master with: - cmd: yq -i '(.configMapGenerator[] | select(.name == "sentry-release").literals[] | select(test("^SENTRY_RELEASE="))) = "SENTRY_RELEASE=${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml + cmd: yq -i '(.configMapGenerator[] | select(.name == "app-version").literals[] | select(test("^APP_VERSION="))) = "APP_VERSION=${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml - name: Update Frontend Image Tag if: needs.ci-frontend-dev.result == 'success' @@ -170,6 +170,6 @@ jobs: # 충돌 방지를 위해 최신 develop 브랜치로 rebase 후 push # [skip ci] 태그를 커밋 메시지에 추가하여 추가 CI 실행을 방지 - git commit -m "ci: Update dev image tags and Sentry release to ${{ github.sha }} [skip ci]" + git commit -m "ci: Update dev image tags and app version to ${{ github.sha }} [skip ci]" git pull --rebase origin ${{ github.ref_name }} git push origin ${{ github.ref_name }} diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index 175be2b7..3be44aac 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -167,11 +167,11 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml - - name: Update Backend Sentry Release + - name: Update Backend App Version if: needs.ci-backend.result == 'success' uses: mikefarah/yq@master with: - cmd: yq -i '(.configMapGenerator[] | select(.name == "sentry-release").literals[] | select(test("^SENTRY_RELEASE="))) = "SENTRY_RELEASE=${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml + cmd: yq -i '(.configMapGenerator[] | select(.name == "app-version").literals[] | select(test("^APP_VERSION="))) = "APP_VERSION=${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml - name: Update Frontend Image Tag if: needs.ci-frontend.result == 'success' @@ -198,7 +198,7 @@ jobs: fi # [skip ci] 태그를 커밋 메시지에 추가하여 추가 CI 실행을 방지 - git commit -m "ci: Update prod image tags and Sentry release to ${{ github.sha }} [skip ci]" + git commit -m "ci: Update prod image tags and app version to ${{ github.sha }} [skip ci]" git pull --rebase origin main git push origin main diff --git a/backend/oj/settings.py b/backend/oj/settings.py index 1861ccd9..2f5d3762 100644 --- a/backend/oj/settings.py +++ b/backend/oj/settings.py @@ -89,9 +89,9 @@ def _before_send_sentry(event, hint): CeleryIntegration(), ], } - SENTRY_RELEASE = get_env("SENTRY_RELEASE", "") - if SENTRY_RELEASE: - SENTRY_OPTIONS["release"] = SENTRY_RELEASE + APP_VERSION = get_env("APP_VERSION", "") + if APP_VERSION: + SENTRY_OPTIONS["release"] = APP_VERSION sentry_sdk.init(**SENTRY_OPTIONS) with open(os.path.join(DATA_DIR, "config", "secret.key"), "r") as f: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index d1e53e1e..74296d59 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -12,6 +12,7 @@ ARG APP_VERSION ARG SENTRY_ENVIRONMENT ARG SENTRY_DSN ARG USE_SENTRY +ENV APP_VERSION=$APP_VERSION ENV VUE_APP_VERSION=$APP_VERSION ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT ENV SENTRY_DSN=$SENTRY_DSN diff --git a/frontend/config/dev.env.js b/frontend/config/dev.env.js index 4f3abc1f..88b5d1cc 100644 --- a/frontend/config/dev.env.js +++ b/frontend/config/dev.env.js @@ -3,7 +3,7 @@ // let version = `"${date}-${commit}"` const sentry = require("./sentry") -let version = JSON.stringify(process.env.VUE_APP_VERSION || "dev") +let version = JSON.stringify(process.env.APP_VERSION || process.env.VUE_APP_VERSION || "dev") let useSentry = JSON.stringify(sentry.isSentryEnabled() ? "1" : "0") let sentryDsn = JSON.stringify(sentry.getSentryDsn()) let sentryEnvironment = JSON.stringify(sentry.getSentryEnvironment()) diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index 583b1e70..a9e37cf1 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -9,7 +9,7 @@ spec: - name: backend envFrom: - configMapRef: - name: sentry-release + name: app-version env: - name: CSRF_TRUSTED_ORIGINS value: "https://k3s.code-place-dev.site" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index be25bd66..cecd1dd7 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -14,7 +14,7 @@ spec: - name: celery-beat envFrom: - configMapRef: - name: sentry-release + name: app-version env: - name: SENTRY_DSN valueFrom: diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index 65f536d2..8641d783 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -17,7 +17,7 @@ spec: - name: celery-worker envFrom: - configMapRef: - name: sentry-release + name: app-version env: - name: SENTRY_DSN valueFrom: diff --git a/kubernetes/overlays/dev/kustomization.yaml b/kubernetes/overlays/dev/kustomization.yaml index 573126b5..675c54d8 100644 --- a/kubernetes/overlays/dev/kustomization.yaml +++ b/kubernetes/overlays/dev/kustomization.yaml @@ -8,9 +8,9 @@ resources: - ./secrets/sentry-secret.yaml namespace: code-place-dev configMapGenerator: - - name: sentry-release + - name: app-version literals: - - SENTRY_RELEASE=82d1bcd221199cd9ad9beb930239a58b38340cba-dev + - APP_VERSION=82d1bcd221199cd9ad9beb930239a58b38340cba-dev generatorOptions: disableNameSuffixHash: true patches: diff --git a/kubernetes/overlays/prod/backend-deployment-patch.yaml b/kubernetes/overlays/prod/backend-deployment-patch.yaml index af94ca92..e22e72eb 100644 --- a/kubernetes/overlays/prod/backend-deployment-patch.yaml +++ b/kubernetes/overlays/prod/backend-deployment-patch.yaml @@ -15,7 +15,7 @@ spec: - name: backend envFrom: - configMapRef: - name: sentry-release + name: app-version imagePullPolicy: IfNotPresent env: - name: CSRF_TRUSTED_ORIGINS diff --git a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml index 7b8128bb..421d7e8a 100644 --- a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml @@ -14,7 +14,7 @@ spec: - name: celery-beat envFrom: - configMapRef: - name: sentry-release + name: app-version env: - name: SENTRY_DSN valueFrom: diff --git a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml index a81dbfbd..95507282 100644 --- a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml @@ -17,7 +17,7 @@ spec: - name: celery-worker envFrom: - configMapRef: - name: sentry-release + name: app-version env: - name: SENTRY_DSN valueFrom: diff --git a/kubernetes/overlays/prod/kustomization.yaml b/kubernetes/overlays/prod/kustomization.yaml index 3cbd1e6e..fbf53074 100644 --- a/kubernetes/overlays/prod/kustomization.yaml +++ b/kubernetes/overlays/prod/kustomization.yaml @@ -11,9 +11,9 @@ resources: namespace: code-place-prod configMapGenerator: - - name: sentry-release + - name: app-version literals: - - SENTRY_RELEASE=3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod + - APP_VERSION=3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod generatorOptions: disableNameSuffixHash: true From e3f6556858699bb6f77611b290d6341e84e0bce4 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 03:52:44 +0900 Subject: [PATCH 144/147] =?UTF-8?q?APP=5FVERSION=20=EC=A3=BC=EC=9E=85=20?= =?UTF-8?q?=EB=B0=A9=EC=8B=9D=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci2develop.yml | 10 +++------- .github/workflows/ci2production-and-release.yml | 10 +++------- backend/Dockerfile | 2 ++ kubernetes/overlays/dev/backend-deployment-patch.yaml | 3 --- .../overlays/dev/celery-beat-deployment-patch.yaml | 3 --- .../overlays/dev/celery-worker-deployment-patch.yaml | 3 --- kubernetes/overlays/dev/kustomization.yaml | 6 ------ kubernetes/overlays/prod/backend-deployment-patch.yaml | 3 --- .../overlays/prod/celery-beat-deployment-patch.yaml | 3 --- .../overlays/prod/celery-worker-deployment-patch.yaml | 3 --- kubernetes/overlays/prod/kustomization.yaml | 7 ------- 11 files changed, 8 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index 45dbad4e..7aa16fe6 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -56,6 +56,8 @@ jobs: push: true tags: ${{ secrets.HARBOR_REGISTRY }}/code-place-dev/backend:${{ github.sha }}-dev provenance: false + build-args: | + APP_VERSION=${{ github.sha }}-dev ci-frontend-dev: needs: [detect-changes-by-component] @@ -138,12 +140,6 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml - - name: Update Backend App Version - if: needs.ci-backend-dev.result == 'success' - uses: mikefarah/yq@master - with: - cmd: yq -i '(.configMapGenerator[] | select(.name == "app-version").literals[] | select(test("^APP_VERSION="))) = "APP_VERSION=${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml - - name: Update Frontend Image Tag if: needs.ci-frontend-dev.result == 'success' uses: mikefarah/yq@master @@ -170,6 +166,6 @@ jobs: # 충돌 방지를 위해 최신 develop 브랜치로 rebase 후 push # [skip ci] 태그를 커밋 메시지에 추가하여 추가 CI 실행을 방지 - git commit -m "ci: Update dev image tags and app version to ${{ github.sha }} [skip ci]" + git commit -m "ci: Update dev image tags to ${{ github.sha }} [skip ci]" git pull --rebase origin ${{ github.ref_name }} git push origin ${{ github.ref_name }} diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index 3be44aac..165b2d42 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -84,6 +84,8 @@ jobs: push: true tags: ${{ secrets.HARBOR_REGISTRY }}/code-place-prod/backend:${{ github.sha }}-prod provenance: false + build-args: | + APP_VERSION=${{ github.sha }}-prod ci-frontend: needs: [detect-changes-by-component] @@ -167,12 +169,6 @@ jobs: with: cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml - - name: Update Backend App Version - if: needs.ci-backend.result == 'success' - uses: mikefarah/yq@master - with: - cmd: yq -i '(.configMapGenerator[] | select(.name == "app-version").literals[] | select(test("^APP_VERSION="))) = "APP_VERSION=${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml - - name: Update Frontend Image Tag if: needs.ci-frontend.result == 'success' uses: mikefarah/yq@master @@ -198,7 +194,7 @@ jobs: fi # [skip ci] 태그를 커밋 메시지에 추가하여 추가 CI 실행을 방지 - git commit -m "ci: Update prod image tags and app version to ${{ github.sha }} [skip ci]" + git commit -m "ci: Update prod image tags to ${{ github.sha }} [skip ci]" git pull --rebase origin main git push origin main diff --git a/backend/Dockerfile b/backend/Dockerfile index bbbd543f..fdaebc7a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,8 +1,10 @@ FROM python:3.11-alpine ARG TARGETARCH ARG TARGETVARIANT +ARG APP_VERSION ENV OJ_ENV production +ENV APP_VERSION=$APP_VERSION WORKDIR /app COPY ./deploy/requirements.txt /app/deploy/ diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index a9e37cf1..78acaaa2 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -7,9 +7,6 @@ spec: spec: containers: - name: backend - envFrom: - - configMapRef: - name: app-version env: - name: CSRF_TRUSTED_ORIGINS value: "https://k3s.code-place-dev.site" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index cecd1dd7..8bbe2697 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -12,9 +12,6 @@ spec: spec: containers: - name: celery-beat - envFrom: - - configMapRef: - name: app-version env: - name: SENTRY_DSN valueFrom: diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index 8641d783..82089303 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -15,9 +15,6 @@ spec: spec: containers: - name: celery-worker - envFrom: - - configMapRef: - name: app-version env: - name: SENTRY_DSN valueFrom: diff --git a/kubernetes/overlays/dev/kustomization.yaml b/kubernetes/overlays/dev/kustomization.yaml index 675c54d8..60142883 100644 --- a/kubernetes/overlays/dev/kustomization.yaml +++ b/kubernetes/overlays/dev/kustomization.yaml @@ -7,12 +7,6 @@ resources: - ./secrets/common-credentials.yaml - ./secrets/sentry-secret.yaml namespace: code-place-dev -configMapGenerator: - - name: app-version - literals: - - APP_VERSION=82d1bcd221199cd9ad9beb930239a58b38340cba-dev -generatorOptions: - disableNameSuffixHash: true patches: - path: backend-deployment-patch.yaml - path: frontend-deployment-patch.yaml diff --git a/kubernetes/overlays/prod/backend-deployment-patch.yaml b/kubernetes/overlays/prod/backend-deployment-patch.yaml index e22e72eb..de88b416 100644 --- a/kubernetes/overlays/prod/backend-deployment-patch.yaml +++ b/kubernetes/overlays/prod/backend-deployment-patch.yaml @@ -13,9 +13,6 @@ spec: spec: containers: - name: backend - envFrom: - - configMapRef: - name: app-version imagePullPolicy: IfNotPresent env: - name: CSRF_TRUSTED_ORIGINS diff --git a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml index 421d7e8a..db60527a 100644 --- a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml @@ -12,9 +12,6 @@ spec: spec: containers: - name: celery-beat - envFrom: - - configMapRef: - name: app-version env: - name: SENTRY_DSN valueFrom: diff --git a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml index 95507282..f98ee708 100644 --- a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml @@ -15,9 +15,6 @@ spec: spec: containers: - name: celery-worker - envFrom: - - configMapRef: - name: app-version env: - name: SENTRY_DSN valueFrom: diff --git a/kubernetes/overlays/prod/kustomization.yaml b/kubernetes/overlays/prod/kustomization.yaml index fbf53074..ab415ea4 100644 --- a/kubernetes/overlays/prod/kustomization.yaml +++ b/kubernetes/overlays/prod/kustomization.yaml @@ -10,13 +10,6 @@ resources: namespace: code-place-prod -configMapGenerator: - - name: app-version - literals: - - APP_VERSION=3a65dc653fc6fb3050a04e373bbf58e7d391c762-prod -generatorOptions: - disableNameSuffixHash: true - patches: - path: backend-deployment-patch.yaml - path: frontend-deployment-patch.yaml From 83e525e87601d1ec2e7d74c632214f97459ba35a Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 03:59:41 +0900 Subject: [PATCH 145/147] =?UTF-8?q?=EB=82=B4=EB=B6=80=20frontend=20error?= =?UTF-8?q?=20=EC=88=98=EC=A7=91=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/oj/urls.py | 2 - backend/utils/json_logging.py | 7 -- backend/utils/observability_metrics.py | 5 - backend/utils/tests.py | 53 --------- backend/utils/views.py | 68 ------------ .../observability-improvement-plan/_index.md | 1 - frontend/src/pages/admin/index.js | 2 - frontend/src/pages/oj/index.js | 2 - frontend/src/utils/client_error_reporter.js | 104 ------------------ .../grafana-dashboard-codeplace.yaml | 22 ---- kubernetes/monitoring/prometheus-rules.yaml | 9 -- 11 files changed, 275 deletions(-) delete mode 100644 frontend/src/utils/client_error_reporter.js diff --git a/backend/oj/urls.py b/backend/oj/urls.py index 1808967a..7f25e9c2 100644 --- a/backend/oj/urls.py +++ b/backend/oj/urls.py @@ -1,13 +1,11 @@ from django.urls import include, re_path from utils.observability_metrics import register_codeplace_metrics -from utils.views import ClientErrorReportAPIView register_codeplace_metrics() urlpatterns = [ re_path(r"^", include("django_prometheus.urls")), - re_path(r"^api/client_error/?$", ClientErrorReportAPIView.as_view(), name="client_error_report_api"), re_path(r"^api/", include("account.urls.oj")), re_path(r"^api/admin/", include("account.urls.admin")), re_path(r"^api/", include("announcement.urls.oj")), diff --git a/backend/utils/json_logging.py b/backend/utils/json_logging.py index ffa4f5b9..62b213ad 100644 --- a/backend/utils/json_logging.py +++ b/backend/utils/json_logging.py @@ -51,13 +51,6 @@ def format(self, record): "task_status", "submission_id", "retry_reason", - "frontend_surface", - "frontend_error_type", - "frontend_message", - "frontend_route", - "frontend_release", - "frontend_component", - "frontend_info", ): value = getattr(record, key, None) if value is not None: diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 723a5860..9ce713c9 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -25,11 +25,6 @@ ["method", "endpoint"], buckets=(0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10), ) -FRONTEND_ERROR_TOTAL = Counter( - "codeplace_frontend_error_total", - "Total frontend runtime errors reported by browser clients.", - ["surface", "error_type"], -) AI_HINT_REQUESTS_TOTAL = Counter( "codeplace_ai_hint_requests_total", "Total backend AI hint streaming requests to the LLM service.", diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 26aebe07..cf43a4c2 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -129,59 +129,6 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): self.assertIn("codeplace_celery_broker_queue_length", body) -class ClientErrorReportAPITest(SimpleTestCase): - - def test_client_error_report_records_metric_and_log(self): - labels = MagicMock() - with patch("utils.views.FRONTEND_ERROR_TOTAL.labels", return_value=labels) as metric_labels, \ - patch("utils.views.frontend_error_logger") as frontend_error_logger: - response = self.client.post( - "/api/client_error", - data=json.dumps({ - "surface": "oj", - "error_type": "vue", - "message": "boom\nwith newline", - "route": "/problem/1", - "release": "20260619-test", - "component": "ProblemView", - "info": "render", - }), - content_type="application/json", - HTTP_X_REQUEST_ID="request-frontend-1", - ) - - self.assertEqual(response.status_code, 200) - metric_labels.assert_called_once_with(surface="oj", error_type="vue") - labels.inc.assert_called_once() - extra = frontend_error_logger.warning.call_args.kwargs["extra"] - self.assertEqual(extra["request_id"], "request-frontend-1") - self.assertEqual(extra["frontend_surface"], "oj") - self.assertEqual(extra["frontend_error_type"], "vue") - self.assertEqual(extra["frontend_message"], "boom with newline") - self.assertEqual(extra["frontend_route"], "/problem/1") - self.assertEqual(extra["frontend_release"], "20260619-test") - - def test_client_error_report_bounds_metric_labels(self): - labels = MagicMock() - with patch("utils.views.FRONTEND_ERROR_TOTAL.labels", return_value=labels) as metric_labels, \ - patch("utils.views.frontend_error_logger") as frontend_error_logger: - response = self.client.post( - "/api/client_error", - data=json.dumps({ - "surface": "attacker-controlled", - "error_type": "TypeError: high cardinality", - "message": "token=abc123 " + ("x" * 500), - }), - content_type="application/json", - ) - - self.assertEqual(response.status_code, 200) - metric_labels.assert_called_once_with(surface="unknown", error_type="unknown") - extra = frontend_error_logger.warning.call_args.kwargs["extra"] - self.assertTrue(extra["frontend_message"].startswith("token=[REDACTED]")) - self.assertLessEqual(len(extra["frontend_message"]), 300) - - class ObservabilityTracingTest(SimpleTestCase): def tearDown(self): diff --git a/backend/utils/views.py b/backend/utils/views.py index ca7c5c25..18621079 100644 --- a/backend/utils/views.py +++ b/backend/utils/views.py @@ -1,47 +1,12 @@ import os -import re from django.conf import settings from account.serializers import FileUploadForm from profile.serializers import ImageUploadForm from utils.shortcuts import rand_str from utils.api import CSRFExemptAPIView -from utils.observability_metrics import FRONTEND_ERROR_TOTAL import logging logger = logging.getLogger(__name__) -frontend_error_logger = logging.getLogger("frontend.error") - -FRONTEND_ERROR_TYPES = { - "vue", - "window_error", - "unhandled_rejection", -} -FRONTEND_SURFACES = { - "oj", - "admin", - "unknown", -} -MAX_FRONTEND_FIELD_LENGTH = 300 -FRONTEND_SECRET_PATTERNS = ( - re.compile(r"(?i)(authorization|cookie)\s*[:=]\s*[^,\s]+"), - re.compile(r"(?i)(password|token|secret|code)=([^&\s]+)"), -) - - -def _clean_frontend_field(value, default=""): - if value is None: - return default - value = str(value).replace("\r", " ").replace("\n", " ").strip() - if not value: - return default - for pattern in FRONTEND_SECRET_PATTERNS: - value = pattern.sub(lambda match: f"{match.group(1)}=[REDACTED]", value) - return value[:MAX_FRONTEND_FIELD_LENGTH] - - -def _clean_choice(value, allowed, default): - value = _clean_frontend_field(value, default) - return value if value in allowed else default class SimditorImageUploadAPIView(CSRFExemptAPIView): @@ -93,36 +58,3 @@ def post(self, request): "file_path": f"{settings.UPLOAD_PREFIX}/{file_name}", "file_name": file.name }) - - -class ClientErrorReportAPIView(CSRFExemptAPIView): - - def post(self, request): - surface = _clean_choice(request.data.get("surface"), FRONTEND_SURFACES, "unknown") - error_type = _clean_choice(request.data.get("error_type"), FRONTEND_ERROR_TYPES, "unknown") - frontend_message = _clean_frontend_field(request.data.get("message"), "unknown") - frontend_route = _clean_frontend_field(request.data.get("route"), "") - frontend_release = _clean_frontend_field(request.data.get("release"), "") - frontend_component = _clean_frontend_field(request.data.get("component"), "") - frontend_info = _clean_frontend_field(request.data.get("info"), "") - request_id = _clean_frontend_field( - request.META.get("HTTP_X_REQUEST_ID") or request.data.get("request_id"), - "", - ) - - FRONTEND_ERROR_TOTAL.labels(surface=surface, error_type=error_type).inc() - frontend_error_logger.warning( - "Frontend runtime error reported", - extra={ - "request_id": request_id or None, - "frontend_surface": surface, - "frontend_error_type": error_type, - "frontend_message": frontend_message, - "frontend_route": frontend_route, - "frontend_release": frontend_release, - "frontend_component": frontend_component, - "frontend_info": frontend_info, - "remote_addr": request.META.get("REMOTE_ADDR"), - }, - ) - return self.success({"reported": True}) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 221f00f8..6825710c 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -35,7 +35,6 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_judge_task_outcome_total{status,scope}` - `codeplace_waiting_queue_length` - `codeplace_celery_broker_queue_length` -- `codeplace_frontend_error_total{surface,error_type}` `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. diff --git a/frontend/src/pages/admin/index.js b/frontend/src/pages/admin/index.js index 84d48e9a..5d2df0a5 100644 --- a/frontend/src/pages/admin/index.js +++ b/frontend/src/pages/admin/index.js @@ -25,7 +25,6 @@ import "./style.less" import ECharts from "vue-echarts" import "@/utils/echarts" import { initSentry } from "@/utils/sentry" -import { initClientErrorReporter } from "@/utils/client_error_reporter" const legacyPanelTag = "Panel" const legacySaveTag = "Save" @@ -69,6 +68,5 @@ Vue.prototype.$success = (msg) => { } initSentry(Vue) -initClientErrorReporter(Vue, { surface: "admin", router }) new Vue(Vue.util.extend({ router, store, i18n }, App)).$mount("#app") diff --git a/frontend/src/pages/oj/index.js b/frontend/src/pages/oj/index.js index 8d822638..76b96a2e 100644 --- a/frontend/src/pages/oj/index.js +++ b/frontend/src/pages/oj/index.js @@ -24,7 +24,6 @@ import filters from "@/utils/filters.js" import ECharts from "vue-echarts" import "@/utils/echarts" import { initSentry } from "@/utils/sentry" -import { initClientErrorReporter } from "@/utils/client_error_reporter" const legacyPanelTag = "Panel" @@ -60,6 +59,5 @@ Vue.prototype.$info = (s) => Vue.prototype.$Message.info(s) Vue.prototype.$success = (s) => Vue.prototype.$Message.success(s) initSentry(Vue) -initClientErrorReporter(Vue, { surface: "oj", router }) new Vue(Vue.util.extend({ router, store, i18n }, App)).$mount("#app") diff --git a/frontend/src/utils/client_error_reporter.js b/frontend/src/utils/client_error_reporter.js deleted file mode 100644 index 8993ca33..00000000 --- a/frontend/src/utils/client_error_reporter.js +++ /dev/null @@ -1,104 +0,0 @@ -const CLIENT_ERROR_ENDPOINT = "/api/client_error" -const MAX_MESSAGE_LENGTH = 300 - -function cleanField(value) { - if (value === undefined || value === null) { - return "" - } - return String(value).replace(/[\r\n]+/g, " ").trim().slice(0, MAX_MESSAGE_LENGTH) -} - -function currentRequestId() { - return cleanField(window.__CODEPLACE_LAST_REQUEST_ID__) -} - -function currentRoute(router) { - if (router && router.currentRoute && router.currentRoute.path) { - return router.currentRoute.path - } - return window.location.pathname -} - -function reportClientError(payload) { - const requestId = currentRequestId() - const body = JSON.stringify({ - ...payload, - request_id: requestId, - release: process.env.VERSION, - }) - const headers = { - "Content-Type": "application/json", - } - if (requestId) { - headers["X-Request-ID"] = requestId - } - - if (navigator.sendBeacon) { - const blob = new Blob([body], { type: "application/json" }) - if (navigator.sendBeacon(CLIENT_ERROR_ENDPOINT, blob)) { - return - } - } - - window.fetch(CLIENT_ERROR_ENDPOINT, { - method: "POST", - headers, - body, - credentials: "same-origin", - keepalive: true, - }).catch(() => {}) -} - -function errorMessage(error) { - if (!error) { - return "unknown" - } - if (error.message) { - return error.message - } - return error.reason || error.toString() -} - -export function initClientErrorReporter(Vue, { surface = "unknown", router } = {}) { - const previousErrorHandler = Vue.config.errorHandler - - Vue.config.errorHandler = (error, vm, info) => { - reportClientError({ - surface, - error_type: "vue", - message: cleanField(errorMessage(error)), - route: cleanField(currentRoute(router)), - component: cleanField(vm && vm.$options && (vm.$options.name || vm.$options._componentTag)), - info: cleanField(info), - }) - - if (previousErrorHandler) { - previousErrorHandler.call(Vue.config, error, vm, info) - } else { - console.error(error) - } - } - - if (window.__CODEPLACE_CLIENT_ERROR_REPORTER_READY__) { - return - } - window.__CODEPLACE_CLIENT_ERROR_REPORTER_READY__ = true - - window.addEventListener("error", (event) => { - reportClientError({ - surface, - error_type: "window_error", - message: cleanField(errorMessage(event.error) || event.message), - route: cleanField(currentRoute(router)), - }) - }) - - window.addEventListener("unhandledrejection", (event) => { - reportClientError({ - surface, - error_type: "unhandled_rejection", - message: cleanField(errorMessage(event.reason)), - route: cleanField(currentRoute(router)), - }) - }) -} diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index f91f3666..2a317ea1 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -527,28 +527,6 @@ data: {"expr": "codeplace_redis_max_clients{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} maxclients"}, {"expr": "increase(codeplace_redis_rejected_connections_total{namespace=~\"$namespace\"}[5m])", "legendFormat": "{{namespace}} rejected/5m"} ] - }, - { - "type": "timeseries", - "title": "Frontend Runtime Errors", - "gridPos": {"x": 0, "y": 71, "w": 12, "h": 6}, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "ops", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "green", "value": null}, - {"color": "orange", "value": 0.02}, - {"color": "red", "value": 0.1} - ] - } - } - }, - "targets": [ - {"expr": "sum by (namespace, surface, error_type) (rate(codeplace_frontend_error_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{surface}} {{error_type}}"} - ] } ] } diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 57332b24..234999ae 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -220,15 +220,6 @@ spec: annotations: summary: "Public frontend endpoint latency is high" description: "A synthetic HTTPS probe has taken more than 2 seconds for 5 minutes." - - alert: FrontendRuntimeErrorsSpike - expr: sum by (namespace, surface) (increase(codeplace_frontend_error_total{namespace=~"code-place-(dev|prod)"}[5m])) > 10 - for: 5m - labels: - severity: warning - priority: P1 - annotations: - summary: "Frontend runtime errors increased" - description: "Browser clients reported more than 10 frontend runtime errors in 5 minutes. Check recent deploys, Sentry events, and frontend.error logs." - alert: JudgeWaitingQueueBacklog expr: codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"} > 5 for: 3m From b3a80c2db5fa671e5240c9398e26d48b2dd16abb Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 04:10:34 +0900 Subject: [PATCH 146/147] =?UTF-8?q?=EA=B3=BC=ED=95=9C=20Redis=20=EA=B4=80?= =?UTF-8?q?=EC=B8=A1=20=EB=A9=94=ED=8A=B8=EB=A6=AD=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/judge/dispatcher.py | 2 - backend/utils/observability_metrics.py | 164 +----------------- backend/utils/tests.py | 85 +-------- .../observability-improvement-plan/_index.md | 1 - .../grafana-dashboard-codeplace.yaml | 50 +----- kubernetes/monitoring/prometheus-rules.yaml | 27 --- 6 files changed, 7 insertions(+), 322 deletions(-) diff --git a/backend/judge/dispatcher.py b/backend/judge/dispatcher.py index 9b995c88..e3b63fde 100644 --- a/backend/judge/dispatcher.py +++ b/backend/judge/dispatcher.py @@ -20,7 +20,6 @@ from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.constants import CacheKey, ProblemScore, ProblemField, Tier -from utils.observability_metrics import record_judge_duration from utils.observability_tracing import get_tracer logger = logging.getLogger(__name__) @@ -189,7 +188,6 @@ def _judge(self, span): self.submission.judge_start_time = timezone.now() resp = self._request(urljoin(server.service_url, "/judge"), data=data) self.submission.judge_end_time = timezone.now() - record_judge_duration((self.submission.judge_end_time - self.submission.judge_start_time).total_seconds()) if not resp: Submission.objects.filter(id=self.submission.id).update(result=JudgeStatus.SYSTEM_ERROR) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index 9ce713c9..d11d8bb2 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -1,18 +1,14 @@ import logging from prometheus_client import Counter, Histogram, REGISTRY -from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, HistogramMetricFamily +from prometheus_client.core import GaugeMetricFamily from utils.cache import cache from utils.constants import CacheKey logger = logging.getLogger(__name__) -JUDGE_DURATION_BUCKETS = (1, 3, 5, 10, 30, 60, 120, 300, 600, float("inf")) CELERY_BROKER_QUEUE_KEY = "celery" -JUDGE_DURATION_BUCKET_KEY = "codeplace:judge_duration_seconds_bucket" -JUDGE_DURATION_COUNT_KEY = "codeplace:judge_duration_seconds_count" -JUDGE_DURATION_SUM_KEY = "codeplace:judge_duration_seconds_sum" HTTP_REQUESTS_TOTAL = Counter( "codeplace_http_requests_total", @@ -64,35 +60,10 @@ def describe(self): "codeplace_celery_broker_queue_length", "Number of Celery tasks waiting in the Redis broker default queue.", ) - yield HistogramMetricFamily( - "codeplace_submission_judge_duration_seconds", - "Judge duration for submissions completed in the last 10 minutes.", - ) - yield GaugeMetricFamily( - "codeplace_redis_connected_clients", - "Current Redis connected clients from INFO clients.", - ) - yield GaugeMetricFamily( - "codeplace_redis_max_clients", - "Configured Redis maxclients from INFO clients or CONFIG GET maxclients.", - ) - yield CounterMetricFamily( - "codeplace_redis_rejected_connections_total", - "Total Redis rejected connections from INFO stats.", - ) - yield GaugeMetricFamily( - "codeplace_observability_collector_success", - "Whether each CodePlace custom metrics collector completed successfully.", - labels=["collector"], - ) def collect(self): - self._collector_success = {} yield self._waiting_queue_length() yield self._celery_broker_queue_length() - yield self._judge_duration_histogram() - yield from self._redis_health_metrics() - yield self._collector_success_metric() def _waiting_queue_length(self): metric = GaugeMetricFamily( @@ -104,9 +75,6 @@ def _waiting_queue_length(self): except Exception as e: logger.warning("Failed to collect waiting queue length: %s", e) metric.add_metric([], 0) - self._mark_collector_success("waiting_queue", False) - else: - self._mark_collector_success("waiting_queue", True) return metric def _celery_broker_queue_length(self): @@ -119,125 +87,8 @@ def _celery_broker_queue_length(self): except Exception as e: logger.warning("Failed to collect Celery broker queue length: %s", e) metric.add_metric([], 0) - self._mark_collector_success("celery_broker_queue", False) - else: - self._mark_collector_success("celery_broker_queue", True) return metric - def _judge_duration_histogram(self): - metric = HistogramMetricFamily( - "codeplace_submission_judge_duration_seconds", - "Judge duration recorded by celery workers.", - ) - try: - duration_buckets = self._redis_hash(JUDGE_DURATION_BUCKET_KEY) - duration_sum = self._float_or_zero(cache.get(JUDGE_DURATION_SUM_KEY, 0)) - except Exception as e: - logger.warning("Failed to collect judge duration metrics: %s", e) - duration_buckets = {} - duration_sum = 0 - self._mark_collector_success("judge_duration", False) - else: - self._mark_collector_success("judge_duration", True) - - cumulative_buckets = [] - for bucket in JUDGE_DURATION_BUCKETS: - label = "+Inf" if bucket == float("inf") else str(bucket) - cumulative_buckets.append((label, self._float_or_zero(duration_buckets.get(label, 0)))) - metric.add_metric([], cumulative_buckets, duration_sum) - return metric - - def _redis_health_metrics(self): - connected_clients_metric = GaugeMetricFamily( - "codeplace_redis_connected_clients", - "Current Redis connected clients from INFO clients.", - ) - max_clients_metric = GaugeMetricFamily( - "codeplace_redis_max_clients", - "Configured Redis maxclients from INFO clients or CONFIG GET maxclients.", - ) - rejected_connections_metric = CounterMetricFamily( - "codeplace_redis_rejected_connections_total", - "Total Redis rejected connections from INFO stats.", - ) - try: - client = cache.client.get_client(write=False) - info = client.info() - connected_clients = float(info.get("connected_clients") or 0) - rejected_connections = float(info.get("rejected_connections") or 0) - max_clients = self._float_or_zero(info.get("maxclients")) - if max_clients == 0: - try: - raw_max_clients = client.config_get("maxclients") - max_clients = self._float_or_zero(raw_max_clients.get("maxclients")) - except Exception as e: - logger.warning("Failed to collect Redis maxclients metric: %s", e) - redis_success = False - else: - redis_success = True - else: - redis_success = True - except Exception as e: - logger.warning("Failed to collect Redis health metrics: %s", e) - connected_clients = max_clients = rejected_connections = 0 - self._mark_collector_success("redis", False) - else: - self._mark_collector_success("redis", redis_success) - - connected_clients_metric.add_metric([], connected_clients) - max_clients_metric.add_metric([], max_clients) - rejected_connections_metric.add_metric([], rejected_connections) - yield connected_clients_metric - yield max_clients_metric - yield rejected_connections_metric - - def _collector_success_metric(self): - metric = GaugeMetricFamily( - "codeplace_observability_collector_success", - "Whether each CodePlace custom metrics collector completed successfully.", - labels=["collector"], - ) - collectors = ( - "waiting_queue", - "celery_broker_queue", - "judge_duration", - "redis", - ) - for collector in collectors: - metric.add_metric([collector], 1 if self._collector_success.get(collector, False) else 0) - return metric - - def _mark_collector_success(self, collector, success): - if not hasattr(self, "_collector_success"): - self._collector_success = {} - self._collector_success[collector] = bool(success) - - @staticmethod - def _redis_hash(key): - values = cache.hgetall(key) or {} - return { - CodePlaceCollector._decode(field): CodePlaceCollector._decode(value) - for field, value in values.items() - } - - @staticmethod - def _decode(value): - if isinstance(value, bytes): - return value.decode() - return str(value) - - @staticmethod - def _float_or_zero(value): - parsed = CodePlaceCollector._float_or_none(value) - return parsed if parsed is not None else 0.0 - - @staticmethod - def _float_or_none(value): - try: - return float(value) - except (TypeError, ValueError): - return None - def register_codeplace_metrics(): if getattr(register_codeplace_metrics, "_registered", False): @@ -247,16 +98,3 @@ def register_codeplace_metrics(): except ValueError: logger.debug("CodePlace metrics collector is already registered") register_codeplace_metrics._registered = True - - -def record_judge_duration(duration_seconds): - try: - duration = max(float(duration_seconds), 0) - for bucket in JUDGE_DURATION_BUCKETS: - if duration <= bucket: - label = "+Inf" if bucket == float("inf") else str(bucket) - cache.hincrby(JUDGE_DURATION_BUCKET_KEY, label, 1) - cache.redis_incr(JUDGE_DURATION_COUNT_KEY, 1) - cache.incrbyfloat(JUDGE_DURATION_SUM_KEY, duration) - except Exception: - logger.exception("Failed to record judge duration metric") diff --git a/backend/utils/tests.py b/backend/utils/tests.py index cf43a4c2..92bf79a1 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -1,10 +1,10 @@ import json import logging import os -from unittest.mock import patch, mock_open, MagicMock +from unittest.mock import patch, mock_open from django.test import SimpleTestCase, TestCase, override_settings from django.core.cache import cache -from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily +from prometheus_client.core import GaugeMetricFamily from utils.json_logging import CodePlaceJsonFormatter from utils.constants import CacheKey @@ -25,77 +25,6 @@ def test_celery_broker_queue_length_is_collected_from_redis(self): llen.assert_called_once_with("celery") self.assertEqual(samples["codeplace_celery_broker_queue_length"][0].value, 7) - def test_judge_duration_histogram_is_collected_from_redis(self): - with patch("utils.observability_metrics.cache.hgetall", return_value={ - b"1": b"0", - b"3": b"1", - b"+Inf": b"2", - }), patch("utils.observability_metrics.cache.get", return_value="4.5"): - samples = self._samples_by_metric([CodePlaceCollector()._judge_duration_histogram()]) - - bucket_samples = [ - sample for sample in samples["codeplace_submission_judge_duration_seconds"] - if sample.name.endswith("_bucket") - ] - self.assertTrue(any(sample.labels["le"] == "3" and sample.value == 1 for sample in bucket_samples)) - self.assertTrue(any(sample.labels["le"] == "+Inf" and sample.value == 2 for sample in bucket_samples)) - self.assertTrue(any( - sample.name.endswith("_sum") and sample.value == 4.5 - for sample in samples["codeplace_submission_judge_duration_seconds"] - )) - - def test_redis_health_metrics_are_collected(self): - redis_client = MagicMock() - redis_client.info.return_value = { - "connected_clients": 25, - "rejected_connections": 3, - } - redis_client.config_get.return_value = {"maxclients": "1000"} - - with patch("utils.observability_metrics.cache.client.get_client", return_value=redis_client): - samples = self._samples_by_metric(list(CodePlaceCollector()._redis_health_metrics())) - - self.assertEqual(samples["codeplace_redis_connected_clients"][0].value, 25) - self.assertEqual(samples["codeplace_redis_max_clients"][0].value, 1000) - rejected_sample = samples["codeplace_redis_rejected_connections"][0] - self.assertEqual(rejected_sample.name, "codeplace_redis_rejected_connections_total") - self.assertEqual(rejected_sample.value, 3) - - def test_redis_health_metrics_prefer_info_maxclients(self): - redis_client = MagicMock() - redis_client.info.return_value = { - "connected_clients": 25, - "rejected_connections": 3, - "maxclients": 2000, - } - - with patch("utils.observability_metrics.cache.client.get_client", return_value=redis_client): - samples = self._samples_by_metric(list(CodePlaceCollector()._redis_health_metrics())) - - redis_client.config_get.assert_not_called() - self.assertEqual(samples["codeplace_redis_max_clients"][0].value, 2000) - - def test_redis_collector_failure_is_reported_when_maxclients_is_unavailable(self): - redis_client = MagicMock() - redis_client.info.return_value = { - "connected_clients": 25, - "rejected_connections": 3, - } - redis_client.config_get.side_effect = RuntimeError("CONFIG disabled") - collector = CodePlaceCollector() - collector._collector_success = {} - - with patch("utils.observability_metrics.cache.client.get_client", return_value=redis_client), \ - patch("utils.observability_metrics.logger"): - list(collector._redis_health_metrics()) - samples = self._samples_by_metric([collector._collector_success_metric()]) - - redis_success = [ - sample for sample in samples["codeplace_observability_collector_success"] - if sample.labels["collector"] == "redis" - ][0] - self.assertEqual(redis_success.value, 0) - class CodePlaceMetricsEndpointTest(SimpleTestCase): @@ -110,16 +39,8 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): "Number of Celery tasks waiting in the Redis broker default queue.", ) broker_queue_metric.add_metric([], 0) - judge_duration_metric = HistogramMetricFamily( - "codeplace_submission_judge_duration_seconds", - "Judge duration for submissions completed in the last 10 minutes.", - ) - judge_duration_metric.add_metric([], [("+Inf", 0)], 0) - with patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ - patch.object(CodePlaceCollector, "_celery_broker_queue_length", return_value=broker_queue_metric), \ - patch.object(CodePlaceCollector, "_judge_duration_histogram", return_value=judge_duration_metric), \ - patch.object(CodePlaceCollector, "_redis_health_metrics", return_value=[]): + patch.object(CodePlaceCollector, "_celery_broker_queue_length", return_value=broker_queue_metric): response = self.client.get("/metrics") self.assertEqual(response.status_code, 200) diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 6825710c..4649ee4b 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -30,7 +30,6 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_http_requests_total{method,endpoint,status_code}` - `codeplace_http_request_duration_seconds{method,endpoint}` -- `codeplace_submission_judge_duration_seconds` - `codeplace_submission_create_outcome_total{status,scope}` - `codeplace_judge_task_outcome_total{status,scope}` - `codeplace_waiting_queue_length` diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 2a317ea1..8a55a6bc 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -165,7 +165,7 @@ data: { "type": "bargauge", "title": "Judge Queue Length", - "gridPos": {"x": 16, "y": 4, "w": 4, "h": 8}, + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, @@ -174,28 +174,6 @@ data: {"expr": "codeplace_celery_broker_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} celery"} ] }, - { - "type": "bargauge", - "title": "Custom Collector Success", - "gridPos": {"x": 20, "y": 4, "w": 4, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} - ] - } - } - }, - "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "codeplace_observability_collector_success{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{collector}}"} - ] - }, { "type": "row", "title": "Judge / Async Outcomes", @@ -203,20 +181,10 @@ data: "collapsed": false, "panels": [] }, - { - "type": "timeseries", - "title": "Judge Duration p95", - "gridPos": {"x": 0, "y": 20, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "s"}}, - "targets": [ - {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_submission_judge_duration_seconds_bucket{namespace=~\"$namespace\"}[10m])))", "legendFormat": "{{namespace}} judge p95"} - ] - }, { "type": "timeseries", "title": "Submission / Judge Outcomes", - "gridPos": {"x": 0, "y": 28, "w": 24, "h": 8}, + "gridPos": {"x": 0, "y": 20, "w": 24, "h": 8}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "ops"}}, "targets": [ @@ -509,24 +477,12 @@ data: { "type": "timeseries", "title": "PostgreSQL Nodes Used", - "gridPos": {"x": 8, "y": 65, "w": 8, "h": 6}, + "gridPos": {"x": 8, "y": 65, "w": 16, "h": 6}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "min by (namespace) (cnpg_collector_nodes_used{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} nodes"} ] - }, - { - "type": "timeseries", - "title": "Redis Clients and Rejections", - "gridPos": {"x": 16, "y": 65, "w": 8, "h": 6}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "codeplace_redis_connected_clients{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} clients"}, - {"expr": "codeplace_redis_max_clients{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} maxclients"}, - {"expr": "increase(codeplace_redis_rejected_connections_total{namespace=~\"$namespace\"}[5m])", "legendFormat": "{{namespace}} rejected/5m"} - ] } ] } diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 234999ae..42b44765 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -238,15 +238,6 @@ spec: annotations: summary: "Celery broker queue is backing up" description: "More than 20 Celery tasks have been waiting in the Redis broker default queue for 5 minutes. Check celery-worker capacity, Redis broker health, and task errors." - - alert: CodePlaceCollectorFailed - expr: codeplace_observability_collector_success{namespace=~"code-place-(dev|prod)"} == 0 - for: 5m - labels: - severity: warning - priority: P1 - annotations: - summary: "A CodePlace custom metrics collector is failing" - description: "A backend /metrics collector block has failed for 5 minutes. Metric values may be stale or zero even though the scrape target is up." - alert: SubmissionCreateSystemFailures expr: sum by (namespace, scope, status) (increase(codeplace_submission_create_outcome_total{namespace=~"code-place-(dev|prod)", status=~"db_error|enqueue_error"}[5m])) > 0 for: 2m @@ -427,24 +418,6 @@ spec: annotations: summary: "Redis memory usage is high" description: "Redis exporter reports memory usage above 85% of configured maxmemory for 10 minutes." - - alert: RedisConnectedClientsHigh - expr: (codeplace_redis_connected_clients{namespace=~"code-place-(dev|prod)"} / clamp_min(codeplace_redis_max_clients{namespace=~"code-place-(dev|prod)"}, 1) > 0.8) and codeplace_redis_max_clients{namespace=~"code-place-(dev|prod)"} > 0 - for: 10m - labels: - severity: warning - priority: P1 - annotations: - summary: "Redis connected clients usage is high" - description: "Backend reports Redis connected clients above 80% of maxclients for 10 minutes." - - alert: RedisRejectedConnections - expr: increase(codeplace_redis_rejected_connections_total{namespace=~"code-place-(dev|prod)"}[5m]) > 0 - for: 2m - labels: - severity: warning - priority: P1 - annotations: - summary: "Redis rejected connections detected" - description: "Backend reports Redis rejected connections in the last 5 minutes. Check maxclients, connection pooling, and Redis resource pressure." - alert: PVCAlmostFull expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 for: 10m From 87337442b08725f4bd83239edb66481b1d6511e9 Mon Sep 17 00:00:00 2001 From: Junyoung Yang Date: Sat, 20 Jun 2026 04:21:16 +0900 Subject: [PATCH 147/147] =?UTF-8?q?=EB=B6=88=ED=95=84=EC=9A=94=ED=95=9C=20?= =?UTF-8?q?=EA=B4=80=EC=B8=A1=EC=84=B1=20=EC=BD=94=EB=93=9C=20=EC=A0=95?= =?UTF-8?q?=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/tests.py | 1 - frontend/src/utils/request_id.js | 1 - 2 files changed, 2 deletions(-) diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 92bf79a1..cee8dcec 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -7,7 +7,6 @@ from prometheus_client.core import GaugeMetricFamily from utils.json_logging import CodePlaceJsonFormatter -from utils.constants import CacheKey from utils.observability_metrics import CodePlaceCollector from utils import observability_tracing from utils.testcase_cache import TestCaseCacheManager diff --git a/frontend/src/utils/request_id.js b/frontend/src/utils/request_id.js index b80246fe..064e79e7 100644 --- a/frontend/src/utils/request_id.js +++ b/frontend/src/utils/request_id.js @@ -25,7 +25,6 @@ function rememberRequestId(requestId) { return } - window.__CODEPLACE_LAST_REQUEST_ID__ = requestId setRequestIdContext(requestId) }