diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index b6fe14fda..7aa16fe6f 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -56,6 +56,8 @@ jobs: push: true tags: ${{ secrets.HARBOR_REGISTRY }}/code-place-dev/backend:${{ github.sha }}-dev provenance: false + build-args: | + APP_VERSION=${{ github.sha }}-dev ci-frontend-dev: needs: [detect-changes-by-component] @@ -87,6 +89,8 @@ jobs: SERVER_NAME=${{ secrets.DEV_SERVER_NAME }} APP_VERSION=${{ github.sha }}-dev SENTRY_ENVIRONMENT=development + SENTRY_DSN=${{ secrets.SENTRY_DSN }} + USE_SENTRY=1 ci-hub-auth-dev: needs: [detect-changes-by-component] diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index 28e3db599..165b2d422 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -84,6 +84,8 @@ jobs: push: true tags: ${{ secrets.HARBOR_REGISTRY }}/code-place-prod/backend:${{ github.sha }}-prod provenance: false + build-args: | + APP_VERSION=${{ github.sha }}-prod ci-frontend: needs: [detect-changes-by-component] @@ -113,8 +115,10 @@ jobs: provenance: false build-args: | SERVER_NAME=${{ secrets.PROD_SERVER_NAME }} - APP_VERSION=${{ github.sha }} + APP_VERSION=${{ github.sha }}-prod SENTRY_ENVIRONMENT=production + SENTRY_DSN=${{ secrets.SENTRY_DSN }} + USE_SENTRY=1 ci-hub-auth: needs: [detect-changes-by-component] diff --git a/backend/Dockerfile b/backend/Dockerfile index bbbd543f0..fdaebc7a1 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,8 +1,10 @@ FROM python:3.11-alpine ARG TARGETARCH ARG TARGETVARIANT +ARG APP_VERSION ENV OJ_ENV production +ENV APP_VERSION=$APP_VERSION WORKDIR /app COPY ./deploy/requirements.txt /app/deploy/ diff --git a/backend/account/middleware.py b/backend/account/middleware.py index 0cbac0e55..3208d973a 100644 --- a/backend/account/middleware.py +++ b/backend/account/middleware.py @@ -1,4 +1,5 @@ import logging +import re import time import uuid @@ -16,6 +17,7 @@ request_logger = logging.getLogger("codeplace.request") MAX_REQUEST_ID_LENGTH = 128 +REQUEST_ID_ALLOWED_CHARS = re.compile(r"[^A-Za-z0-9_.:-]+") class RequestIDMiddleware: @@ -37,6 +39,9 @@ def __call__(self, request): @staticmethod def _request_id(request): request_id = request.META.get("HTTP_X_REQUEST_ID") + if not request_id: + return uuid.uuid4().hex + request_id = REQUEST_ID_ALLOWED_CHARS.sub("-", request_id).strip("-") if not request_id: return uuid.uuid4().hex return request_id[:MAX_REQUEST_ID_LENGTH] diff --git a/backend/account/tests.py b/backend/account/tests.py index 653ca07cc..5dd80383f 100644 --- a/backend/account/tests.py +++ b/backend/account/tests.py @@ -8,7 +8,7 @@ from django.contrib import auth from django.db import DatabaseError from django.http.response import JsonResponse -from django.test import RequestFactory, TestCase +from django.test import RequestFactory, SimpleTestCase from django.utils import decorators from django.utils.timezone import now from utils.api.tests import APIClient, APITestCase @@ -18,9 +18,53 @@ from .models import AdminType, ProblemPermission, User from .decorators import scheduler_only +from .middleware import RequestIDMiddleware from .tasks import calculate_user_score_basis, calculate_user_score_fluctuation +class RequestIDMiddlewareTest(SimpleTestCase): + + def setUp(self): + self.factory = RequestFactory() + + def test_generates_request_id_when_header_is_missing(self): + middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True})) + request = self.factory.get("/") + + response = middleware(request) + + self.assertEqual(len(request.request_id), 32) + self.assertEqual(response["X-Request-ID"], request.request_id) + + def test_accepts_existing_request_id_and_truncates_it(self): + request_id = "a" * 150 + middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True})) + request = self.factory.get("/", HTTP_X_REQUEST_ID=request_id) + + response = middleware(request) + + self.assertEqual(request.request_id, "a" * 128) + self.assertEqual(response["X-Request-ID"], request.request_id) + + def test_sanitizes_request_id_before_returning_response_header(self): + middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True})) + request = self.factory.get("/", HTTP_X_REQUEST_ID="trace-1\nSet-Cookie:bad") + + response = middleware(request) + + self.assertEqual(request.request_id, "trace-1-Set-Cookie:bad") + self.assertEqual(response["X-Request-ID"], request.request_id) + + def test_generates_request_id_when_header_has_no_safe_characters(self): + middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True})) + request = self.factory.get("/", HTTP_X_REQUEST_ID="\n\t") + + response = middleware(request) + + self.assertEqual(len(request.request_id), 32) + self.assertEqual(response["X-Request-ID"], request.request_id) + + class PermissionDecoratorTest(APITestCase): """ 데코레이터 테스트 diff --git a/backend/judge/dispatcher.py b/backend/judge/dispatcher.py index 60e92bc27..e3b63fdee 100644 --- a/backend/judge/dispatcher.py +++ b/backend/judge/dispatcher.py @@ -10,7 +10,6 @@ from django.db import transaction, IntegrityError from django.db.models import F from django.http import HttpResponseNotFound -from opentelemetry import trace from account.models import User, UserScore, UserSolved from conf.models import JudgeServer @@ -21,9 +20,10 @@ from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.constants import CacheKey, ProblemScore, ProblemField, Tier +from utils.observability_tracing import get_tracer logger = logging.getLogger(__name__) -tracer = trace.get_tracer(__name__) +tracer = get_tracer(__name__) # 继续处理在队列中的问题 @@ -44,15 +44,20 @@ def __init__(self): self.server = None def __enter__(self) -> [JudgeServer, None]: - with transaction.atomic(): - servers = JudgeServer.objects.select_for_update().filter(is_disabled=False).order_by("task_number") - servers = [s for s in servers if s.status == "normal"] - for server in servers: - if server.task_number <= server.cpu_core * 2: - server.task_number = F("task_number") + 1 - server.save(update_fields=["task_number"]) - self.server = server - return server + with tracer.start_as_current_span("judge_server.select") as span: + with transaction.atomic(): + servers = JudgeServer.objects.select_for_update().filter(is_disabled=False).order_by("task_number") + servers = [s for s in servers if s.status == "normal"] + span.set_attribute("codeplace.judge_server.candidates", len(servers)) + for server in servers: + if server.task_number <= server.cpu_core * 2: + server.task_number = F("task_number") + 1 + server.save(update_fields=["task_number"]) + self.server = server + span.set_attribute("codeplace.judge_server.selected", True) + span.set_attribute("codeplace.judge_server.hostname", server.hostname) + return server + span.set_attribute("codeplace.judge_server.selected", False) return None def __exit__(self, exc_type, exc_val, exc_tb): @@ -211,7 +216,11 @@ def _judge(self, span): self.submission.result = error_test_case[0]["result"] else: self.submission.result = JudgeStatus.PARTIALLY_ACCEPTED - self.submission.save() + with tracer.start_as_current_span("submission.result_save") as save_span: + save_span.set_attribute("codeplace.submission_id", self.submission.id) + save_span.set_attribute("codeplace.problem_id", self.problem.id) + save_span.set_attribute("codeplace.judge.result", self.submission.result) + self.submission.save() span.set_attribute("codeplace.judge.result", self.submission.result) if self.contest_id: diff --git a/backend/judge/tasks.py b/backend/judge/tasks.py index 483be700d..e8ac3380e 100644 --- a/backend/judge/tasks.py +++ b/backend/judge/tasks.py @@ -1,12 +1,19 @@ import celery -from opentelemetry import trace +import logging from account.models import User from utils.shortcuts import CELERY_TASK_ARGS +from utils.observability_tracing import get_tracer +from utils.observability_metrics import JUDGE_TASK_OUTCOME_TOTAL from submission.models import Submission from judge.dispatcher import JudgeDispatcher -tracer = trace.get_tracer(__name__) +logger = logging.getLogger(__name__) +tracer = get_tracer(__name__) + + +def _record_judge_task_outcome(status, scope): + JUDGE_TASK_OUTCOME_TOTAL.labels(status=status, scope=scope).inc() @celery.shared_task(**CELERY_TASK_ARGS()) @@ -14,8 +21,30 @@ def judge_task(submission_id, problem_id): with tracer.start_as_current_span("judge_task") as span: span.set_attribute("codeplace.submission_id", submission_id) span.set_attribute("codeplace.problem_id", problem_id) - uid = Submission.objects.get(id=submission_id).user_id - if User.objects.get(id=uid).is_disabled: - span.set_attribute("codeplace.judge.skipped", "user_disabled") - return - JudgeDispatcher(submission_id, problem_id).judge() + scope = "unknown" + try: + submission = Submission.objects.get(id=submission_id) + scope = "contest" if submission.contest_id else "practice" + except Submission.DoesNotExist: + _record_judge_task_outcome("submission_missing", scope) + logger.exception("Judge task submission does not exist: %s", submission_id) + raise + + try: + user = User.objects.get(id=submission.user_id) + if user.is_disabled: + span.set_attribute("codeplace.judge.skipped", "user_disabled") + _record_judge_task_outcome("user_disabled", scope) + return + + JudgeDispatcher(submission_id, problem_id).judge() + except User.DoesNotExist: + _record_judge_task_outcome("user_missing", scope) + logger.exception("Judge task user does not exist for submission %s", submission_id) + raise + except Exception: + _record_judge_task_outcome("error", scope) + logger.exception("Judge task failed for submission %s", submission_id) + raise + else: + _record_judge_task_outcome("success", scope) diff --git a/backend/judge/tests.py b/backend/judge/tests.py new file mode 100644 index 000000000..a2ea46f00 --- /dev/null +++ b/backend/judge/tests.py @@ -0,0 +1,43 @@ +from unittest import mock + +from django.test import SimpleTestCase + +from judge.tasks import judge_task + + +class JudgeTaskObservabilityTest(SimpleTestCase): + + @mock.patch("judge.tasks.JUDGE_TASK_OUTCOME_TOTAL") + @mock.patch("judge.tasks.JudgeDispatcher") + @mock.patch("judge.tasks.User") + @mock.patch("judge.tasks.Submission") + def test_judge_task_records_success_outcome(self, submission_model, user_model, dispatcher, outcome_total): + submission = mock.Mock(user_id=1, contest_id=None) + submission_model.objects.get.return_value = submission + user_model.objects.get.return_value = mock.Mock(is_disabled=False) + labels = mock.Mock() + outcome_total.labels.return_value = labels + + judge_task.run(10, 20) + + dispatcher.assert_called_once_with(10, 20) + dispatcher.return_value.judge.assert_called_once() + outcome_total.labels.assert_called_once_with(status="success", scope="practice") + labels.inc.assert_called_once() + + @mock.patch("judge.tasks.JUDGE_TASK_OUTCOME_TOTAL") + @mock.patch("judge.tasks.JudgeDispatcher") + @mock.patch("judge.tasks.User") + @mock.patch("judge.tasks.Submission") + def test_judge_task_records_disabled_user_outcome(self, submission_model, user_model, dispatcher, outcome_total): + submission = mock.Mock(user_id=1, contest_id=7) + submission_model.objects.get.return_value = submission + user_model.objects.get.return_value = mock.Mock(is_disabled=True) + labels = mock.Mock() + outcome_total.labels.return_value = labels + + judge_task.run(10, 20) + + dispatcher.assert_not_called() + outcome_total.labels.assert_called_once_with(status="user_disabled", scope="contest") + labels.inc.assert_called_once() diff --git a/backend/oj/settings.py b/backend/oj/settings.py index 5bba42a04..2f5d37620 100644 --- a/backend/oj/settings.py +++ b/backend/oj/settings.py @@ -15,6 +15,8 @@ import celery.schedules import sentry_sdk +from sentry_sdk.integrations.celery import CeleryIntegration +from sentry_sdk.integrations.django import DjangoIntegration production_env = get_env("OJ_ENV", "dev") == "production" if production_env: @@ -22,16 +24,75 @@ else: from .dev_settings import * +SENTRY_SENSITIVE_EXACT_FIELDS = { + "authorization", + "cookie", + "password", + "token", + "secret", + "code", + "src", +} +SENTRY_SENSITIVE_SUBSTRINGS = { + "authorization", + "cookie", + "password", + "token", + "secret", + "source_code", + "sourcecode", + "spj_code", + "src", +} + + +def _is_sentry_sensitive_key(key): + normalized = str(key).lower() + return ( + normalized in SENTRY_SENSITIVE_EXACT_FIELDS + or any(field in normalized for field in SENTRY_SENSITIVE_SUBSTRINGS) + ) + + +def _redact_sentry_value(value): + if isinstance(value, dict): + return { + key: "[REDACTED]" if _is_sentry_sensitive_key(key) else _redact_sentry_value(item) + for key, item in value.items() + } + if isinstance(value, list): + return [_redact_sentry_value(item) for item in value] + return value + + +def _before_send_sentry(event, hint): + event = _redact_sentry_value(event) + request = event.get("request") or {} + request_id = (request.get("headers") or {}).get("X-Request-ID") + if request_id: + event.setdefault("tags", {})["request_id"] = request_id + return event + + SENTRY_DSN = get_env( "SENTRY_DSN", "", ) if SENTRY_DSN: - sentry_sdk.init( - dsn=SENTRY_DSN, - send_default_pii=True, - environment=get_env("SENTRY_ENVIRONMENT", get_env("OJ_ENV", "dev")), - ) + SENTRY_OPTIONS = { + "dsn": SENTRY_DSN, + "send_default_pii": False, + "environment": get_env("SENTRY_ENVIRONMENT", get_env("OJ_ENV", "dev")), + "before_send": _before_send_sentry, + "integrations": [ + DjangoIntegration(), + CeleryIntegration(), + ], + } + APP_VERSION = get_env("APP_VERSION", "") + if APP_VERSION: + SENTRY_OPTIONS["release"] = APP_VERSION + sentry_sdk.init(**SENTRY_OPTIONS) with open(os.path.join(DATA_DIR, "config", "secret.key"), "r") as f: SECRET_KEY = f.read() diff --git a/backend/problem/llm_hint.py b/backend/problem/llm_hint.py index 588d45a5e..70e6b6e11 100644 --- a/backend/problem/llm_hint.py +++ b/backend/problem/llm_hint.py @@ -7,6 +7,8 @@ from django.utils.html import escape, strip_tags import time +from utils.observability_metrics import AI_HINT_DURATION_SECONDS, AI_HINT_REQUESTS_TOTAL + LOCAL_VLLM_CHAT_COMPLETIONS_URL = "http://localhost:8000/v1/chat/completions" CLUSTER_VLLM_CHAT_COMPLETIONS_URL = "http://vllm.code-place-prod:8000/v1/chat/completions" VLLM_MODEL = "Qwen/Qwen3.5-9B" @@ -316,6 +318,8 @@ def stream_problem_hint(problem, previous_hints=None, user_code=None): return response = None + started_at = time.monotonic() + status = "success" try: response = requests.post( get_vllm_chat_completions_url(), @@ -338,13 +342,24 @@ def stream_problem_hint(problem, previous_hints=None, user_code=None): try: chunk = json.loads(payload) except ValueError as exc: + status = "stream_parse_error" raise LLMHintError("Failed to parse vLLM stream") from exc text = _extract_stream_delta(chunk) if text: yield text except requests.RequestException as exc: + status = "request_error" raise LLMHintError("Failed to call vLLM") from exc + except GeneratorExit: + status = "client_closed" + raise + except Exception: + if status == "success": + status = "error" + raise finally: + AI_HINT_REQUESTS_TOTAL.labels(status=status).inc() + AI_HINT_DURATION_SECONDS.labels(status=status).observe(max(time.monotonic() - started_at, 0)) if response is not None: response.close() diff --git a/backend/problem/tests.py b/backend/problem/tests.py index 28a949719..25d4f4895 100644 --- a/backend/problem/tests.py +++ b/backend/problem/tests.py @@ -23,7 +23,7 @@ from utils.constants import CONTEST_PASSWORD_SESSION_KEY from .llm_hint import (CLUSTER_VLLM_CHAT_COMPLETIONS_URL, LOCAL_VLLM_CHAT_COMPLETIONS_URL, VLLM_CONNECT_TIMEOUT_SEC, VLLM_MODEL, VLLM_STREAM_READ_TIMEOUT_SEC, - get_vllm_chat_completions_url) + LLMHintError, get_vllm_chat_completions_url, stream_problem_hint) from .views.admin import TestCaseAPI from .utils import parse_problem_template @@ -320,6 +320,74 @@ def test_stream_llm_hint(self, mocked_post): (VLLM_CONNECT_TIMEOUT_SEC, VLLM_STREAM_READ_TIMEOUT_SEC), ) + @mock.patch("problem.views.oj.AI_HINT_API_OUTCOME_TOTAL") + @mock.patch("problem.llm_hint.requests.post") + def test_stream_llm_hint_records_api_success_outcome(self, mocked_post, outcome_total): + mocked_post.return_value = self._mock_streaming_response([ + 'data: {"choices":[{"delta":{"content":"힌트"}}]}', + "data: [DONE]", + ]) + labels = mock.Mock() + outcome_total.labels.return_value = labels + + resp = self.client.get(f"{self.url}?problem_id={self.problem._id}") + self._streaming_body(resp) + + outcome_total.labels.assert_called_once_with(status="success", scope="practice") + labels.inc.assert_called_once() + + @mock.patch("problem.llm_hint.AI_HINT_DURATION_SECONDS") + @mock.patch("problem.llm_hint.AI_HINT_REQUESTS_TOTAL") + @mock.patch("problem.llm_hint.requests.post") + def test_stream_llm_hint_records_success_metric(self, mocked_post, requests_total, duration_seconds): + mocked_post.return_value = self._mock_streaming_response([ + 'data: {"choices":[{"delta":{"content":"힌트"}}]}', + "data: [DONE]", + ]) + request_labels = mock.Mock() + duration_labels = mock.Mock() + requests_total.labels.return_value = request_labels + duration_seconds.labels.return_value = duration_labels + + self.assertEqual("".join(stream_problem_hint(self.problem)), "힌트") + + requests_total.labels.assert_called_once_with(status="success") + request_labels.inc.assert_called_once() + duration_seconds.labels.assert_called_once_with(status="success") + duration_labels.observe.assert_called_once() + + @mock.patch("problem.llm_hint.AI_HINT_DURATION_SECONDS") + @mock.patch("problem.llm_hint.AI_HINT_REQUESTS_TOTAL") + @mock.patch("problem.llm_hint.requests.post", side_effect=requests.Timeout) + def test_stream_llm_hint_records_request_error_metric(self, mocked_post, requests_total, duration_seconds): + request_labels = mock.Mock() + duration_labels = mock.Mock() + requests_total.labels.return_value = request_labels + duration_seconds.labels.return_value = duration_labels + + with self.assertRaises(LLMHintError): + list(stream_problem_hint(self.problem)) + + requests_total.labels.assert_called_once_with(status="request_error") + request_labels.inc.assert_called_once() + duration_seconds.labels.assert_called_once_with(status="request_error") + duration_labels.observe.assert_called_once() + + @mock.patch("problem.views.oj.AI_HINT_API_OUTCOME_TOTAL") + @mock.patch("problem.llm_hint.requests.post") + def test_stream_llm_hint_records_problem_limit_outcome(self, mocked_post, outcome_total): + for i in range(5): + ProblemAIHintLog.objects.create(user=self.user, problem=self.problem, hint_content=f"더미 {i}") + labels = mock.Mock() + outcome_total.labels.return_value = labels + + resp = self.client.get(f"{self.url}?problem_id={self.problem._id}") + self._streaming_body(resp) + + outcome_total.labels.assert_called_once_with(status="problem_limit_exceeded", scope="practice") + labels.inc.assert_called_once() + mocked_post.assert_not_called() + @mock.patch("problem.llm_hint.requests.post") def test_stream_llm_hint_problem_limit(self, mocked_post): """한 문제당 5회 제한에 걸리는지 테스트""" diff --git a/backend/problem/views/oj.py b/backend/problem/views/oj.py index af383f7c6..03eddf60c 100644 --- a/backend/problem/views/oj.py +++ b/backend/problem/views/oj.py @@ -14,6 +14,7 @@ from submission.models import JudgeStatus, Submission from utils.api import APIView from utils.constants import Difficulty, ProblemField, Tier +from utils.observability_metrics import AI_HINT_API_OUTCOME_TOTAL from ..llm_hint import LLMHintError, MAX_USER_CODE_LENGTH, stream_problem_hint from ..models import (Problem, ProblemRuleType, ProblemTag, get_default_week_info, ProblemAIHintLog) @@ -164,17 +165,41 @@ def _error_events(self, message, err="llm-hint-unavailable"): yield self._format_sse_event("app-error", {"error": err, "message": message}) yield self._format_sse_event("done", {"done": True}) - def _error_response(self, message, err="llm-hint-unavailable"): + def _record_outcome(self, status, scope): + AI_HINT_API_OUTCOME_TOTAL.labels(status=status, scope=scope).inc() + + @staticmethod + def _persist_hint_log(hint_log, full_text_chunks): + full_text = "".join(full_text_chunks) + if full_text.strip(): + if hint_log: + hint_log.hint_content = full_text + hint_log.save(update_fields=['hint_content']) + return True + + if hint_log: + hint_log.delete() + return False + + def _error_response(self, message, err="llm-hint-unavailable", metric_status=None, scope="practice"): + if metric_status: + self._record_outcome(metric_status, scope) return self._streaming_response(self._error_events(message, err=err)) def get(self, request): - if not request.user.is_authenticated: - return self._error_response("로그인이 필요합니다.", err="permission-denied") - problem_id = request.GET.get("problem_id") contest_id = request.GET.get("contest_id") + scope = "contest" if contest_id else "practice" + if not request.user.is_authenticated: + return self._error_response( + "로그인이 필요합니다.", + err="permission-denied", + metric_status="permission_denied", + scope=scope, + ) + if not problem_id: - return self._error_response("문제 번호가 필요합니다.") + return self._error_response("문제 번호가 필요합니다.", metric_status="bad_request", scope=scope) # 사용자가 현재 작성 중인 코드 (없으면 None) # 서버에서 한 번 더 길이를 제한해 과도하게 큰 요청을 차단 @@ -185,7 +210,12 @@ def get(self, request): try: contest = Contest.objects.select_related("created_by").get(id=contest_id, visible=True) except Contest.DoesNotExist: - return self._error_response("대회를 찾을 수 없습니다.", err="permission-denied") + return self._error_response( + "대회를 찾을 수 없습니다.", + err="permission-denied", + metric_status="permission_denied", + scope=scope, + ) if not request.user.is_contest_admin(contest): if contest.contest_type == ContestType.PASSWORD_PROTECTED_CONTEST: @@ -193,22 +223,36 @@ def get(self, request): request.session.get(CONTEST_PASSWORD_SESSION_KEY, {}).get(contest.id), contest.password, ): - return self._error_response("비밀번호가 올바르지 않거나 만료되었습니다.", err="permission-denied") + return self._error_response( + "비밀번호가 올바르지 않거나 만료되었습니다.", + err="permission-denied", + metric_status="permission_denied", + scope=scope, + ) if contest.status == ContestStatus.CONTEST_NOT_START: - return self._error_response("아직 시작하지 않은 대회입니다.", err="permission-denied") + return self._error_response( + "아직 시작하지 않은 대회입니다.", + err="permission-denied", + metric_status="permission_denied", + scope=scope, + ) if not contest.ai_assistant_enabled: - return self._error_response("이 대회에서는 AI 조교를 사용할 수 없습니다.") + return self._error_response( + "이 대회에서는 AI 조교를 사용할 수 없습니다.", + metric_status="disabled", + scope=scope, + ) try: problem = Problem.objects.get(_id=problem_id, contest_id=contest_id, visible=True) except Problem.DoesNotExist: - return self._error_response("문제를 찾을 수 없습니다.") + return self._error_response("문제를 찾을 수 없습니다.", metric_status="problem_not_found", scope=scope) else: try: problem = Problem.objects.get(_id=problem_id, contest_id__isnull=True, visible=True) except Problem.DoesNotExist: - return self._error_response("문제를 찾을 수 없습니다.") + return self._error_response("문제를 찾을 수 없습니다.", metric_status="problem_not_found", scope=scope) hint_log = None previous_hints = [] @@ -225,7 +269,11 @@ def get(self, request): if problem_hint_count >= 5: return self._error_response( - "이 문제에 대한 AI 조교 사용 횟수(5회)를 모두 소진했습니다. 이전 답변을 복습해 보세요.", err="problem-limit-exceeded") + "이 문제에 대한 AI 조교 사용 횟수(5회)를 모두 소진했습니다. 이전 답변을 복습해 보세요.", + err="problem-limit-exceeded", + metric_status="problem_limit_exceeded", + scope=scope, + ) # 이전 힌트 문자열 추출 previous_hints = [log.hint_content for log in previous_logs if log.hint_content.strip()] @@ -244,48 +292,32 @@ def generator(): for chunk in stream_problem_hint(problem, previous_hints=previous_hints, user_code=user_code): full_text_chunks.append(chunk) yield self._format_sse_event("chunk", {"text": chunk}) - full_text = "".join(full_text_chunks) - if full_text.strip(): - hint_log.hint_content = full_text - hint_log.save(update_fields=['hint_content']) - elif hint_log: - hint_log.delete() + if self._persist_hint_log(hint_log, full_text_chunks): + self._record_outcome("success", scope) + else: + self._record_outcome("empty_response", scope) yield self._format_sse_event("done", {"done": True}) except LLMHintError as exc: logger.warning("Failed to stream LLM hint for problem %s: %s", problem_id, exc) - full_text = "".join(full_text_chunks) - if full_text.strip(): - # (1) 일부라도 생성된 텍스트가 있다면 그 시점까지의 내용을 저장 - hint_log.hint_content = full_text - hint_log.save(update_fields=['hint_content']) - elif hint_log: - # (2) 생성된 텍스트가 아예 없다면 선제 생성한 로그 삭제 및 횟수 복구 - hint_log.delete() + self._persist_hint_log(hint_log, full_text_chunks) + self._record_outcome("llm_error", scope) yield from self._error_events("힌트를 생성하지 못했습니다.", err="llm-hint-unavailable") except (GeneratorExit, BrokenPipeError, ConnectionResetError): logger.info("LLM hint stream client disconnected for problem %s", problem_id) - full_text = "".join(full_text_chunks) - if full_text.strip(): - hint_log.hint_content = full_text - hint_log.save(update_fields=['hint_content']) - elif hint_log: - hint_log.delete() + self._persist_hint_log(hint_log, full_text_chunks) + self._record_outcome("client_disconnected", scope) return except Exception: logger.exception("Unexpected error while streaming LLM hint for problem %s", problem_id) - full_text = "".join(full_text_chunks) - if full_text.strip(): - hint_log.hint_content = full_text - hint_log.save(update_fields=['hint_content']) - elif hint_log: - hint_log.delete() + self._persist_hint_log(hint_log, full_text_chunks) + self._record_outcome("unexpected_error", scope) yield from self._error_events("힌트를 생성하지 못했습니다.", err="llm-hint-unavailable") return self._streaming_response(generator()) diff --git a/backend/submission/tests.py b/backend/submission/tests.py index e0695f7a4..d32cf37ba 100644 --- a/backend/submission/tests.py +++ b/backend/submission/tests.py @@ -57,6 +57,20 @@ def test_create_submission_success(self): self.assertSuccess(resp) self.assertIn("submission_id", resp.data["data"]) + @mock.patch("submission.views.oj.SUBMISSION_CREATE_OUTCOME_TOTAL") + def test_create_submission_records_success_outcome(self, outcome_total): + data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) + data["problem_id"] = self.problem.id + labels = mock.Mock() + outcome_total.labels.return_value = labels + + with mock.patch('judge.tasks.judge_task.apply_async'): + resp = self.client.post(self.url, data=data) + + self.assertSuccess(resp) + outcome_total.labels.assert_called_once_with(status="success", scope="practice") + labels.inc.assert_called_once() + def test_create_submission_nonexistent_problem(self): data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) data["problem_id"] = 99999 @@ -64,6 +78,19 @@ def test_create_submission_nonexistent_problem(self): resp = self.client.post(self.url, data=data) self.assertFailed(resp, "Problem not exist") + @mock.patch("submission.views.oj.SUBMISSION_CREATE_OUTCOME_TOTAL") + def test_create_submission_records_problem_not_found_outcome(self, outcome_total): + data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) + data["problem_id"] = 99999 + labels = mock.Mock() + outcome_total.labels.return_value = labels + + resp = self.client.post(self.url, data=data) + + self.assertFailed(resp, "Problem not exist") + outcome_total.labels.assert_called_once_with(status="problem_not_found", scope="practice") + labels.inc.assert_called_once() + def test_create_submission_invalid_language(self): data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) data["problem_id"] = self.problem.id diff --git a/backend/submission/views/oj.py b/backend/submission/views/oj.py index 6b1a9f415..a302bac0c 100644 --- a/backend/submission/views/oj.py +++ b/backend/submission/views/oj.py @@ -1,4 +1,5 @@ import ipaddress +import logging from django.db.models import Q, Count from django.utils.dateparse import parse_datetime @@ -12,14 +13,21 @@ from utils.api import APIView, validate_serializer from utils.cache import cache from utils.captcha import Captcha +from utils.observability_metrics import SUBMISSION_CREATE_OUTCOME_TOTAL from utils.throttling import TokenBucket from ..models import JudgeStatus, Submission from ..serializers import (CreateSubmissionSerializer, SubmissionModelSerializer, ShareSubmissionSerializer) from ..serializers import SubmissionSafeModelSerializer, SubmissionListSerializer +logger = logging.getLogger(__name__) + class SubmissionAPI(APIView): + @staticmethod + def _record_create_outcome(status, scope): + SUBMISSION_CREATE_OUTCOME_TOTAL.labels(status=status, scope=scope).inc() + def throttling(self, request): # 使用 open_api 的请求暂不做限制 auth_method = getattr(request, "auth_method", "") @@ -53,9 +61,11 @@ def check_contest_permission(self, request): def post(self, request): data = request.data hide_id = False + scope = "contest" if data.get("contest_id") else "practice" if data.get("contest_id"): error = self.check_contest_permission(request) if error: + self._record_create_outcome("contest_permission_denied", scope) return error contest = self.contest if not contest.problem_details_permission(request.user): @@ -63,30 +73,45 @@ def post(self, request): if data.get("captcha"): if not Captcha(request).check(data["captcha"]): + self._record_create_outcome("invalid_captcha", scope) return self.error("Invalid captcha") error = self.throttling(request) if error: + self._record_create_outcome("throttled", scope) return self.error(error) try: problem = Problem.objects.get(id=data["problem_id"], contest_id=data.get("contest_id"), visible=True) except Problem.DoesNotExist: + self._record_create_outcome("problem_not_found", scope) return self.error("Problem not exist") if data["language"] not in problem.languages: + self._record_create_outcome("language_not_allowed", scope) return self.error(f"{data['language']} is now allowed in the problem") - submission = Submission.objects.create( - user_id=request.user.id, - username=request.user.username, - language=data["language"], - code=data["code"], - problem_id=problem.id, - ip=request.session["ip"], - contest_id=data.get("contest_id")) + try: + submission = Submission.objects.create( + user_id=request.user.id, + username=request.user.username, + language=data["language"], + code=data["code"], + problem_id=problem.id, + ip=request.session["ip"], + contest_id=data.get("contest_id")) + except Exception: + self._record_create_outcome("db_error", scope) + logger.exception("Failed to create submission for problem %s", data["problem_id"]) + raise # use this for debug # JudgeDispatcher(submission.id, problem.id).judge() - judge_task.apply_async(args=(submission.id, problem.id)) + try: + judge_task.apply_async(args=(submission.id, problem.id)) + except Exception: + self._record_create_outcome("enqueue_error", scope) + logger.exception("Failed to enqueue judge task for submission %s", submission.id) + raise + self._record_create_outcome("success", scope) if hide_id: return self.success() else: diff --git a/backend/utils/json_logging.py b/backend/utils/json_logging.py index 6468c0c3a..62b213ad4 100644 --- a/backend/utils/json_logging.py +++ b/backend/utils/json_logging.py @@ -3,6 +3,7 @@ from datetime import datetime, timezone from utils.observability_context import get_request_id +from utils.observability_tracing import get_current_trace_context SENSITIVE_EXACT_FIELDS = { @@ -37,6 +38,7 @@ def format(self, record): "message": record.getMessage(), "request_id": getattr(record, "request_id", None) or get_request_id(), } + payload.update(get_current_trace_context()) for key in ( "method", "path", @@ -45,7 +47,10 @@ def format(self, record): "user_id", "remote_addr", "task_id", + "task_name", + "task_status", "submission_id", + "retry_reason", ): value = getattr(record, key, None) if value is not None: diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index e51144948..d11d8bb2b 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -1,31 +1,14 @@ import logging -from datetime import timedelta -from django.db.models import Count -from django.utils import timezone from prometheus_client import Counter, Histogram, REGISTRY -from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily +from prometheus_client.core import GaugeMetricFamily -from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.constants import CacheKey logger = logging.getLogger(__name__) -JUDGE_DURATION_BUCKETS = (1, 3, 5, 10, 30, 60, 120, 300, 600, float("inf")) -JUDGE_STATUS_LABELS = { - JudgeStatus.COMPILE_ERROR: "compile_error", - JudgeStatus.WRONG_ANSWER: "wrong_answer", - JudgeStatus.ACCEPTED: "accepted", - JudgeStatus.CPU_TIME_LIMIT_EXCEEDED: "cpu_time_limit_exceeded", - JudgeStatus.REAL_TIME_LIMIT_EXCEEDED: "real_time_limit_exceeded", - JudgeStatus.MEMORY_LIMIT_EXCEEDED: "memory_limit_exceeded", - JudgeStatus.RUNTIME_ERROR: "runtime_error", - JudgeStatus.SYSTEM_ERROR: "system_error", - JudgeStatus.PENDING: "pending", - JudgeStatus.JUDGING: "judging", - JudgeStatus.PARTIALLY_ACCEPTED: "partially_accepted", -} +CELERY_BROKER_QUEUE_KEY = "celery" HTTP_REQUESTS_TOTAL = Counter( "codeplace_http_requests_total", @@ -38,60 +21,49 @@ ["method", "endpoint"], buckets=(0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10), ) +AI_HINT_REQUESTS_TOTAL = Counter( + "codeplace_ai_hint_requests_total", + "Total backend AI hint streaming requests to the LLM service.", + ["status"], +) +AI_HINT_DURATION_SECONDS = Histogram( + "codeplace_ai_hint_duration_seconds", + "Backend AI hint streaming duration.", + ["status"], + buckets=(0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600), +) +AI_HINT_API_OUTCOME_TOTAL = Counter( + "codeplace_ai_hint_api_outcome_total", + "Total user-facing AI hint API outcomes.", + ["status", "scope"], +) +SUBMISSION_CREATE_OUTCOME_TOTAL = Counter( + "codeplace_submission_create_outcome_total", + "Total user-facing submission create API outcomes.", + ["status", "scope"], +) +JUDGE_TASK_OUTCOME_TOTAL = Counter( + "codeplace_judge_task_outcome_total", + "Total Celery judge task outcomes.", + ["status", "scope"], +) class CodePlaceCollector: def describe(self): - yield GaugeMetricFamily( - "codeplace_submission_status_count", - "Current number of submissions grouped by judge status.", - labels=["status"], - ) yield GaugeMetricFamily( "codeplace_waiting_queue_length", "Number of submissions waiting because no judge-server was available.", ) yield GaugeMetricFamily( - "codeplace_judge_server_available", - "Whether each judge-server is enabled and has a fresh heartbeat.", - labels=["hostname"], - ) - yield GaugeMetricFamily( - "codeplace_judge_server_last_heartbeat_age_seconds", - "Seconds since the latest judge-server heartbeat.", - labels=["hostname"], - ) - yield GaugeMetricFamily( - "codeplace_judge_server_task_number", - "Current task_number recorded for each judge-server.", - labels=["hostname"], - ) - yield HistogramMetricFamily( - "codeplace_submission_judge_duration_seconds", - "Judge duration for submissions completed in the last 10 minutes.", + "codeplace_celery_broker_queue_length", + "Number of Celery tasks waiting in the Redis broker default queue.", ) def collect(self): - yield from self._submission_status_count() yield self._waiting_queue_length() - yield from self._judge_server_metrics() - yield self._judge_duration_histogram() - - def _submission_status_count(self): - metric = GaugeMetricFamily( - "codeplace_submission_status_count", - "Current number of submissions grouped by judge status.", - labels=["status"], - ) - try: - counts = dict(Submission.objects.values_list("result").order_by("result").annotate(count=Count("id"))) - except Exception as e: - logger.warning("Failed to collect submission status counts: %s", e) - counts = {} - for status_value, status_label in JUDGE_STATUS_LABELS.items(): - metric.add_metric([status_label], counts.get(status_value, 0)) - yield metric + yield self._celery_broker_queue_length() def _waiting_queue_length(self): metric = GaugeMetricFamily( @@ -105,66 +77,16 @@ def _waiting_queue_length(self): metric.add_metric([], 0) return metric - def _judge_server_metrics(self): - from conf.models import JudgeServer - - available_metric = GaugeMetricFamily( - "codeplace_judge_server_available", - "Whether each judge-server is enabled and has a fresh heartbeat.", - labels=["hostname"], - ) - heartbeat_metric = GaugeMetricFamily( - "codeplace_judge_server_last_heartbeat_age_seconds", - "Seconds since the latest judge-server heartbeat.", - labels=["hostname"], - ) - task_metric = GaugeMetricFamily( - "codeplace_judge_server_task_number", - "Current task_number recorded for each judge-server.", - labels=["hostname"], - ) - now = timezone.now() - try: - servers = list(JudgeServer.objects.all()) - except Exception as e: - logger.warning("Failed to collect judge-server metrics: %s", e) - servers = [] - for server in servers: - if server.is_disabled: - continue - hostname = server.hostname or "unknown" - heartbeat_age = max((now - server.last_heartbeat).total_seconds(), 0) - available = 1 if server.status == "normal" else 0 - available_metric.add_metric([hostname], available) - heartbeat_metric.add_metric([hostname], heartbeat_age) - task_metric.add_metric([hostname], server.task_number) - yield available_metric - yield heartbeat_metric - yield task_metric - - def _judge_duration_histogram(self): - metric = HistogramMetricFamily( - "codeplace_submission_judge_duration_seconds", - "Judge duration for submissions completed in the last 10 minutes.", + def _celery_broker_queue_length(self): + metric = GaugeMetricFamily( + "codeplace_celery_broker_queue_length", + "Number of Celery tasks waiting in the Redis broker default queue.", ) - cutoff = timezone.now() - timedelta(minutes=10) - durations = [] try: - submissions = Submission.objects.filter( - judge_start_time__gte=cutoff, - judge_start_time__isnull=False, - judge_end_time__isnull=False, - ).only("judge_start_time", "judge_end_time") - for submission in submissions: - durations.append(max((submission.judge_end_time - submission.judge_start_time).total_seconds(), 0)) + metric.add_metric([], cache.llen(CELERY_BROKER_QUEUE_KEY) or 0) except Exception as e: - logger.warning("Failed to collect judge duration metrics: %s", e) - - cumulative_buckets = [] - for bucket in JUDGE_DURATION_BUCKETS: - label = "+Inf" if bucket == float("inf") else str(bucket) - cumulative_buckets.append((label, sum(1 for duration in durations if duration <= bucket))) - metric.add_metric([], cumulative_buckets, sum(durations)) + logger.warning("Failed to collect Celery broker queue length: %s", e) + metric.add_metric([], 0) return metric diff --git a/backend/utils/observability_tracing.py b/backend/utils/observability_tracing.py index 9702522f6..c3c768171 100644 --- a/backend/utils/observability_tracing.py +++ b/backend/utils/observability_tracing.py @@ -1,30 +1,48 @@ import logging +from opentelemetry import trace + from utils.shortcuts import get_env logger = logging.getLogger(__name__) +_OTEL_CONFIGURED = False -def configure_opentelemetry(service_name): - if get_env("OTEL_ENABLED", "0").lower() not in ("1", "true", "yes", "on"): - return +def get_tracer(name): + return trace.get_tracer(name) + +def get_current_trace_context(): try: - from opentelemetry import trace - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter - from opentelemetry.instrumentation.celery import CeleryInstrumentor - from opentelemetry.instrumentation.django import DjangoInstrumentor - from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor - from opentelemetry.instrumentation.redis import RedisInstrumentor - from opentelemetry.instrumentation.requests import RequestsInstrumentor - from opentelemetry.sdk.resources import Resource - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased + span_context = trace.get_current_span().get_span_context() except Exception: - logger.exception("OpenTelemetry packages are unavailable") + return {} + if not getattr(span_context, "is_valid", False): + return {} + return { + "trace_id": format(span_context.trace_id, "032x"), + "span_id": format(span_context.span_id, "016x"), + } + + +def configure_opentelemetry(service_name): + global _OTEL_CONFIGURED + if get_env("OTEL_ENABLED", "0").lower() not in ("1", "true", "yes", "on"): + return + if _OTEL_CONFIGURED: return + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + from opentelemetry.instrumentation.celery import CeleryInstrumentor + from opentelemetry.instrumentation.django import DjangoInstrumentor + from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor + from opentelemetry.instrumentation.redis import RedisInstrumentor + from opentelemetry.instrumentation.requests import RequestsInstrumentor + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased + try: sampler_ratio = float(get_env("OTEL_TRACES_SAMPLER_ARG", "0.05")) except ValueError: @@ -37,6 +55,12 @@ def configure_opentelemetry(service_name): }) provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(sampler_ratio))) endpoint = get_env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector.monitoring.svc.cluster.local:4317") + current_provider = trace.get_tracer_provider() + if type(current_provider).__name__ != "ProxyTracerProvider": + logger.info("OpenTelemetry tracer provider is already configured; skipping CodePlace setup") + _OTEL_CONFIGURED = True + return + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True))) trace.set_tracer_provider(provider) @@ -45,3 +69,4 @@ def configure_opentelemetry(service_name): Psycopg2Instrumentor().instrument() RedisInstrumentor().instrument() CeleryInstrumentor().instrument() + _OTEL_CONFIGURED = True diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 7e2f85efc..cee8dcec0 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -1,12 +1,138 @@ import json +import logging import os -from unittest.mock import patch, mock_open, MagicMock -from django.test import TestCase, override_settings +from unittest.mock import patch, mock_open +from django.test import SimpleTestCase, TestCase, override_settings from django.core.cache import cache +from prometheus_client.core import GaugeMetricFamily +from utils.json_logging import CodePlaceJsonFormatter +from utils.observability_metrics import CodePlaceCollector +from utils import observability_tracing from utils.testcase_cache import TestCaseCacheManager +class CodePlaceCollectorTest(SimpleTestCase): + + def _samples_by_metric(self, metrics): + return {metric.name: metric.samples for metric in metrics} + + def test_celery_broker_queue_length_is_collected_from_redis(self): + with patch("utils.observability_metrics.cache.llen", return_value=7) as llen: + samples = self._samples_by_metric([CodePlaceCollector()._celery_broker_queue_length()]) + + llen.assert_called_once_with("celery") + self.assertEqual(samples["codeplace_celery_broker_queue_length"][0].value, 7) + + +class CodePlaceMetricsEndpointTest(SimpleTestCase): + + def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): + waiting_queue_metric = GaugeMetricFamily( + "codeplace_waiting_queue_length", + "Number of submissions waiting because no judge-server was available.", + ) + waiting_queue_metric.add_metric([], 0) + broker_queue_metric = GaugeMetricFamily( + "codeplace_celery_broker_queue_length", + "Number of Celery tasks waiting in the Redis broker default queue.", + ) + broker_queue_metric.add_metric([], 0) + with patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ + patch.object(CodePlaceCollector, "_celery_broker_queue_length", return_value=broker_queue_metric): + response = self.client.get("/metrics") + + self.assertEqual(response.status_code, 200) + body = response.content.decode() + self.assertIn("django_http_requests_total_by_method_total", body) + self.assertIn("codeplace_waiting_queue_length", body) + self.assertIn("codeplace_celery_broker_queue_length", body) + + +class ObservabilityTracingTest(SimpleTestCase): + + def tearDown(self): + observability_tracing._OTEL_CONFIGURED = False + + def test_configure_opentelemetry_is_idempotent(self): + observability_tracing._OTEL_CONFIGURED = True + real_import = __import__ + + def fail_import(name, *args, **kwargs): + if name == "opentelemetry": + raise AssertionError("OpenTelemetry should not be imported again") + return real_import(name, *args, **kwargs) + + with patch("utils.observability_tracing.get_env", return_value="1"), \ + patch("builtins.__import__", side_effect=fail_import): + observability_tracing.configure_opentelemetry("codeplace-test") + + +class CodePlaceJsonFormatterTest(SimpleTestCase): + + def test_preserves_status_code_and_redacts_sensitive_fields(self): + record = logging.LogRecord( + name="codeplace.request", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="request completed", + args=(), + exc_info=None, + ) + record.status_code = 500 + record.request_id = "request-1" + record.token = "secret-token" + record.source_code = "print('secret')" + + payload = json.loads(CodePlaceJsonFormatter().format(record)) + + self.assertEqual(payload["status_code"], 500) + self.assertEqual(payload["request_id"], "request-1") + self.assertNotIn("token", payload) + self.assertNotIn("source_code", payload) + + def test_includes_trace_context_when_available(self): + record = logging.LogRecord( + name="codeplace.request", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="request completed", + args=(), + exc_info=None, + ) + + with patch("utils.json_logging.get_current_trace_context", return_value={ + "trace_id": "0" * 31 + "1", + "span_id": "0" * 15 + "2", + }): + payload = json.loads(CodePlaceJsonFormatter().format(record)) + + self.assertEqual(payload["trace_id"], "0" * 31 + "1") + self.assertEqual(payload["span_id"], "0" * 15 + "2") + + def test_redacts_nested_sensitive_fields(self): + formatter = CodePlaceJsonFormatter() + payload = { + "status_code": 200, + "data": { + "password": "secret", + "submission": { + "code": "print('secret')", + "language": "Python3", + }, + }, + } + + redacted = formatter._redact(payload) + + self.assertEqual(redacted["status_code"], 200) + self.assertEqual(redacted["data"]["password"], "[REDACTED]") + self.assertEqual(redacted["data"]["submission"]["code"], "[REDACTED]") + self.assertEqual(redacted["data"]["submission"]["language"], "Python3") + + class TestTestCaseCacheManager(TestCase): def setUp(self): diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index b8d0cf39e..4649ee4b3 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -13,11 +13,11 @@ CodePlace 관측성 1차 구현은 Kubernetes 운영 환경을 기준으로 합 관측 대상은 운영자가 장애 발생 후 1분 안에 알림을 받고, 5분 안에 원인 후보를 좁힐 수 있도록 다음 경로로 나눕니다. -- **User Path:** Traefik/Ingress, frontend nginx, backend Django API. +- **User Path:** public HTTPS probe, Traefik/Ingress, frontend nginx, backend Django API. - **Submission/Judge Path:** submission 생성, Celery `judge_task`, Redis `waiting_queue`, judge-server heartbeat, judge-server `/judge` 호출, 결과 저장. - **Async Jobs:** celery-worker, celery-beat, scheduled tasks. - **Data Stores:** PostgreSQL/CNPG, Redis Sentinel. -- **Platform:** backend/frontend/celery/judge Pod, node, PVC. +- **Platform:** backend/frontend/celery/judge Pod, node, PVC, monitoring stack. - **Client/Error:** frontend runtime error, backend exception, request ID 기반 로그. ## 2. 구현 내용 @@ -30,12 +30,10 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_http_requests_total{method,endpoint,status_code}` - `codeplace_http_request_duration_seconds{method,endpoint}` -- `codeplace_submission_status_count{status}` -- `codeplace_submission_judge_duration_seconds` +- `codeplace_submission_create_outcome_total{status,scope}` +- `codeplace_judge_task_outcome_total{status,scope}` - `codeplace_waiting_queue_length` -- `codeplace_judge_server_available{hostname}` -- `codeplace_judge_server_last_heartbeat_age_seconds{hostname}` -- `codeplace_judge_server_task_number{hostname}` +- `codeplace_celery_broker_queue_length` `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. @@ -63,6 +61,12 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. +Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 조건에서는 Loki SingleBinary + Longhorn PVC를 단기 보관 기준선으로 둡니다. + +- dev log retention: 3일. +- prod log retention: 7일. +- 수집 대상: `code-place-dev`, `code-place-prod`, `monitoring`, Traefik Pod. + ### Tracing OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OTEL_ENABLED=0`입니다. dev/staging에서 먼저 활성화하고 collector/Tempo/sampling 확인 후 prod 활성화를 결정합니다. @@ -80,13 +84,17 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT 신규 monitoring 리소스는 `kubernetes/monitoring` 아래에 두고, 애플리케이션 `dev/prod` overlay에 포함하지 않습니다. - `backend-service-monitor.yaml`: backend `/metrics` scrape, interval 15s. -- `prometheus-rules.yaml`: P0/P1 fast alert rules. +- `blackbox-exporter.yaml`, `public-endpoint-probes.yaml`: frontend/hub-auth/Grafana 공개 HTTPS probe. +- `datastore-pod-monitors.yaml`: CNPG/Redis exporter scrape. +- `logs/loki-values.yaml`, `logs/alloy-values.yaml`: Loki/Alloy Helm values. +- `prometheus-rules.yaml`: P0/P1 fast alert rules와 일부 recording rules. - `alertmanager-config.yaml`: P0/P1 Discord alert routing. - `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. +- `grafana-dashboard-logs.yaml`: Loki/Alloy 상태와 주요 앱 로그 조회 dashboard. - `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. -`alertmanager-contact-points` Secret은 repo에 저장하지 않습니다. backend의 `/data/config/secret.key`와 같은 비밀값이지만, 자동으로 파일이 생기는 구조는 아닙니다. 운영자가 `kubectl create secret`으로 만들거나 ExternalSecrets/SealedSecrets 같은 Secret 주입 도구로 생성해야 합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. +`alertmanager-contact-points` Secret은 repo에 평문으로 저장하지 않습니다. 운영자는 SealedSecret으로 `monitoring` namespace에 생성합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. ## 4. 빠른 알림 정책 @@ -97,10 +105,9 @@ P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합 - `BackendTargetDown`: backend scrape 실패 1분 지속. - `Api5xxSpike`: backend 5xx 비율 5% 초과 2분 지속. - `Ingress5xxSpike`: Traefik 5xx 비율 5% 초과 2분 지속. -- `JudgeAllUnavailable`: 사용 가능한 judge-server 0개 1분 지속. -- `JudgeHeartbeatCritical`: judge heartbeat age 180초 초과. -- `PostgresUnavailable`: PostgreSQL Pod not ready 1분 지속. -- `RedisUnavailable`: Redis Pod not ready 1분 지속. +- `PostgresUnavailable`: PostgreSQL Pod not ready 또는 readiness metric missing 1분 지속. +- `RedisUnavailable`: Redis/Sentinel Pod not ready 또는 readiness metric missing 1분 지속. +- `LokiUnavailable`: Loki Pod not ready 1분 지속. ### P1 @@ -112,6 +119,8 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `CeleryBeatDown`: beat Pod not ready 2분 지속. - `PodCrashLooping`: 주요 Pod restart 증가 5분 지속. - `PVCAlmostFull`: PVC 사용률 85% 초과 10분 지속. +- `CodePlaceCollectorFailed`: backend custom metric collector 실패 5분 지속. +- `FrontendRuntimeErrorsSpike`: frontend runtime error report 증가 5분 지속. ## 5. 운영 확인 절차 @@ -122,16 +131,17 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - prod: `kubectl apply -k kubernetes/overlays/prod` 4. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - `kubectl apply -k kubernetes/monitoring` -5. Prometheus target에서 `backend` ServiceMonitor가 healthy인지 확인합니다. -6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission status, waiting queue, judge heartbeat panel을 확인합니다. -7. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +5. Prometheus target에서 `backend`, `postgres`, `redis`, `blackbox-exporter`, `loki`, `alloy` target이 healthy인지 확인합니다. +6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission/judge outcome, waiting queue panel을 확인합니다. +7. Grafana의 `CodePlace Logs` dashboard 또는 Explore에서 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. +8. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. - `/metrics`는 cluster 내부 scrape 전용이며 외부 Ingress에 노출하지 않습니다. - Discord webhook URL은 Git에 저장하지 않습니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. -- OpenTelemetry는 `OTEL_ENABLED=0` 기본값을 유지하고 dev/staging에서 먼저 활성화합니다. +- OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지합니다. Collector/Tempo 리소스는 포함하지만 dev에서 먼저 확인한 뒤 prod 활성화를 결정합니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. ## 6. 검증 @@ -143,15 +153,18 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Metrics endpoint: Django test client 기준 `/metrics` 200 및 `codeplace_*` metric 포함 확인 - Kubernetes app render: `kubectl kustomize kubernetes/overlays/dev`, `kubectl kustomize kubernetes/overlays/prod` - Kubernetes monitoring render: `kubectl kustomize kubernetes/monitoring` +- Grafana dashboard JSON parse: `grafana-dashboard-codeplace.yaml`, `grafana-dashboard-logs.yaml` -`promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다. +`promtool`이 있는 환경에서는 `PrometheusRule.spec.groups`를 추출해서 `promtool check rules`로 확인합니다. ### Discord Webhook Secret -Discord webhook URL은 repo에 커밋하지 않습니다. 클러스터별로 다음 명령으로 생성합니다. +Discord webhook URL은 repo에 커밋하지 않습니다. SealedSecrets controller가 설치된 클러스터에서 생성합니다. ```sh kubectl -n monitoring create secret generic alertmanager-contact-points \ --from-literal=webhook-url="$ALERT_WEBHOOK_URL" \ - --dry-run=client -o yaml | kubectl apply -f - + --dry-run=client -o yaml \ + | kubeseal --controller-namespace kube-system --format yaml \ + > kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml ``` diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 1a5725eab..74296d598 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -10,8 +10,13 @@ COPY . . ARG APP_VERSION ARG SENTRY_ENVIRONMENT +ARG SENTRY_DSN +ARG USE_SENTRY +ENV APP_VERSION=$APP_VERSION ENV VUE_APP_VERSION=$APP_VERSION ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT +ENV SENTRY_DSN=$SENTRY_DSN +ENV USE_SENTRY=$USE_SENTRY RUN npm run build:dll RUN npm run build diff --git a/frontend/config/dev.env.js b/frontend/config/dev.env.js index 4f3abc1f6..88b5d1cc1 100644 --- a/frontend/config/dev.env.js +++ b/frontend/config/dev.env.js @@ -3,7 +3,7 @@ // let version = `"${date}-${commit}"` const sentry = require("./sentry") -let version = JSON.stringify(process.env.VUE_APP_VERSION || "dev") +let version = JSON.stringify(process.env.APP_VERSION || process.env.VUE_APP_VERSION || "dev") let useSentry = JSON.stringify(sentry.isSentryEnabled() ? "1" : "0") let sentryDsn = JSON.stringify(sentry.getSentryDsn()) let sentryEnvironment = JSON.stringify(sentry.getSentryEnvironment()) diff --git a/frontend/config/sentry.js b/frontend/config/sentry.js index b35191920..fedbab049 100644 --- a/frontend/config/sentry.js +++ b/frontend/config/sentry.js @@ -1,14 +1,12 @@ "use strict" function isSentryEnabled(nodeEnv = process.env.NODE_ENV) { - return (process.env.USE_SENTRY || (nodeEnv === "production" ? "1" : "0")) === "1" + const enabled = (process.env.USE_SENTRY || (nodeEnv === "production" ? "1" : "0")) === "1" + return enabled && Boolean(getSentryDsn()) } function getSentryDsn() { - return ( - process.env.SENTRY_DSN || - "https://143814aaa2d6e0b4550b2e5effefe90d@o4511586463776768.ingest.us.sentry.io/4511586483634176" - ) + return process.env.SENTRY_DSN || "" } function getSentryEnvironment(nodeEnv = process.env.NODE_ENV) { diff --git a/frontend/deploy/kube_nginx/README.md b/frontend/deploy/kube_nginx/README.md index 2bf65c870..0d58fbef9 100644 --- a/frontend/deploy/kube_nginx/README.md +++ b/frontend/deploy/kube_nginx/README.md @@ -8,4 +8,10 @@ K3s로 마이그레이션한 후, TLS (Transport Layer Security) 종료에 대 따라서 이 Nginx 인스턴스는 더 이상 TLS 인증 및 종료를 수행하지 않습니다. 대신, Traefik이 모든 인바운드 TLS 트래픽을 처리한 후 암호화되지 않은 HTTP 트래픽을 이 Nginx 서비스로 프록시합니다. +### 관측성 + +Kubernetes용 Nginx는 access log를 JSON 형식으로 `/dev/stdout`에 출력하고 error log를 `/dev/stderr`에 출력합니다. +들어온 `X-Request-ID`가 있으면 그대로 유지하고, 없으면 Nginx `$request_id`를 생성해 응답 header와 backend proxy header로 전달합니다. +이 값은 frontend nginx log, backend JSON log, Celery task log를 Loki에서 같은 `request_id`로 조회하는 데 사용합니다. + 참고로, TLS 처리를 포함했던 이전 Nginx 설정은 `frontend/deploy/nginx/` 디렉토리에서 확인할 수 있습니다. diff --git a/frontend/deploy/kube_nginx/api_proxy.conf b/frontend/deploy/kube_nginx/api_proxy.conf index 4310b9c87..edf00cd0b 100644 --- a/frontend/deploy/kube_nginx/api_proxy.conf +++ b/frontend/deploy/kube_nginx/api_proxy.conf @@ -1,6 +1,7 @@ proxy_pass http://backend; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $http_host; +proxy_set_header X-Request-ID $codeplace_request_id; client_max_body_size 200M; proxy_http_version 1.1; proxy_set_header Connection ''; diff --git a/frontend/deploy/kube_nginx/nginx.conf b/frontend/deploy/kube_nginx/nginx.conf index 5f21ba6c2..0f10c1fbb 100644 --- a/frontend/deploy/kube_nginx/nginx.conf +++ b/frontend/deploy/kube_nginx/nginx.conf @@ -21,9 +21,27 @@ http { gzip_vary on; gzip_types application/javascript text/css; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; + map $http_x_request_id $codeplace_request_id { + default $http_x_request_id; + "" $request_id; + } + + log_format main escape=json + '{' + '"timestamp":"$time_iso8601",' + '"level":"INFO",' + '"logger":"frontend.nginx",' + '"message":"request completed",' + '"request_id":"$codeplace_request_id",' + '"method":"$request_method",' + '"path":"$uri",' + '"status_code":$status,' + '"duration_seconds":$request_time,' + '"remote_addr":"$remote_addr",' + '"body_bytes_sent":$body_bytes_sent,' + '"user_agent":"$http_user_agent",' + '"x_forwarded_for":"$http_x_forwarded_for"' + '}'; access_log /dev/stdout main; @@ -35,6 +53,7 @@ http { add_header X-XSS-Protection "1; mode=block" always; add_header X-Frame-Options SAMEORIGIN always; add_header X-Content-Type-Options nosniff always; + add_header X-Request-ID $codeplace_request_id always; server { listen 8000 default_server; diff --git a/frontend/src/pages/admin/api.js b/frontend/src/pages/admin/api.js index 526ee5cb4..57b38d104 100644 --- a/frontend/src/pages/admin/api.js +++ b/frontend/src/pages/admin/api.js @@ -2,11 +2,13 @@ import Vue from "vue" import router from "./router" import axios from "axios" import utils from "@/utils/utils" +import { configureAxiosRequestId } from "@/utils/request_id" Vue.prototype.$http = axios axios.defaults.baseURL = "/api" axios.defaults.xsrfHeaderName = "X-CSRFToken" axios.defaults.xsrfCookieName = "csrftoken" +configureAxiosRequestId(axios) export default { // 登录 diff --git a/frontend/src/pages/oj/api.js b/frontend/src/pages/oj/api.js index 6a0ade9f2..2154ee684 100644 --- a/frontend/src/pages/oj/api.js +++ b/frontend/src/pages/oj/api.js @@ -1,11 +1,13 @@ import Vue from "vue" import store from "@/store" import axios from "axios" +import { configureAxiosRequestId } from "@/utils/request_id" Vue.prototype.$http = axios axios.defaults.baseURL = "/api" axios.defaults.xsrfHeaderName = "X-CSRFToken" axios.defaults.xsrfCookieName = "csrftoken" +configureAxiosRequestId(axios) export default { getPopup() { diff --git a/frontend/src/utils/request_id.js b/frontend/src/utils/request_id.js new file mode 100644 index 000000000..064e79e7a --- /dev/null +++ b/frontend/src/utils/request_id.js @@ -0,0 +1,75 @@ +import { setRequestIdContext } from "@/utils/sentry" + +const REQUEST_ID_HEADER = "X-Request-ID" +const RESPONSE_REQUEST_ID_HEADER = "x-request-id" + +function randomHex(length) { + if (window.crypto && window.crypto.getRandomValues) { + const values = new Uint8Array(Math.ceil(length / 2)) + window.crypto.getRandomValues(values) + return Array.from(values) + .map((value) => value.toString(16).padStart(2, "0")) + .join("") + .slice(0, length) + } + + return Math.random().toString(16).slice(2, 2 + length).padEnd(length, "0") +} + +function createRequestId() { + return `cp-${Date.now().toString(36)}-${randomHex(16)}` +} + +function rememberRequestId(requestId) { + if (!requestId) { + return + } + + setRequestIdContext(requestId) +} + +function responseRequestId(response) { + return ( + response && + response.headers && + (response.headers[RESPONSE_REQUEST_ID_HEADER] || response.headers[REQUEST_ID_HEADER]) + ) +} + +export function configureAxiosRequestId(axios) { + if (axios.__codeplaceRequestIdConfigured) { + return + } + axios.__codeplaceRequestIdConfigured = true + + axios.interceptors.request.use((config) => { + const requestId = createRequestId() + config.headers = config.headers || {} + config.headers[REQUEST_ID_HEADER] = requestId + config.metadata = { + ...(config.metadata || {}), + requestId, + } + rememberRequestId(requestId) + return config + }) + + axios.interceptors.response.use( + (response) => { + const requestId = + responseRequestId(response) || + (response.config && response.config.metadata && response.config.metadata.requestId) + response.requestId = requestId + rememberRequestId(requestId) + return response + }, + (error) => { + const requestId = + responseRequestId(error.response) || + (error.config && error.config.metadata && error.config.metadata.requestId) + error.requestId = requestId + rememberRequestId(requestId) + return Promise.reject(error) + }, + ) +} diff --git a/frontend/src/utils/sentry.js b/frontend/src/utils/sentry.js index f69876060..a38ff528c 100644 --- a/frontend/src/utils/sentry.js +++ b/frontend/src/utils/sentry.js @@ -1,11 +1,56 @@ import { init, setContext } from "@sentry/vue" -const SENTRY_DSN = - "https://143814aaa2d6e0b4550b2e5effefe90d@o4511586463776768.ingest.us.sentry.io/4511586483634176" +const SENSITIVE_EXACT_FIELDS = [ + "authorization", + "cookie", + "password", + "token", + "secret", + "code", + "src", +] + +const SENSITIVE_SUBSTRINGS = [ + "authorization", + "cookie", + "password", + "token", + "secret", + "source_code", + "sourcecode", + "spj_code", + "src", +] + +function isSensitiveKey(key) { + const normalized = String(key).toLowerCase() + return ( + SENSITIVE_EXACT_FIELDS.indexOf(normalized) !== -1 || + SENSITIVE_SUBSTRINGS.some((field) => normalized.indexOf(field) !== -1) + ) +} + +function redact(value) { + if (Array.isArray(value)) { + return value.map((item) => redact(item)) + } + + if (value && typeof value === "object") { + return Object.keys(value).reduce((result, key) => { + result[key] = isSensitiveKey(key) ? "[REDACTED]" : redact(value[key]) + return result + }, {}) + } + + return value +} const options = { release: process.env.VERSION, environment: process.env.SENTRY_ENVIRONMENT, + beforeSend(event) { + return redact(event) + }, denyUrls: [ // Chrome extensions /extensions\//i, @@ -22,13 +67,13 @@ const options = { } export function initSentry(Vue) { - if (process.env.USE_SENTRY !== "1") { + if (process.env.USE_SENTRY !== "1" || !process.env.SENTRY_DSN) { return } init({ Vue, - dsn: process.env.SENTRY_DSN || SENTRY_DSN, + dsn: process.env.SENTRY_DSN, ...options, }) @@ -37,3 +82,9 @@ export function initSentry(Vue) { location: window.location.href, }) } + +export function setRequestIdContext(requestId) { + setContext("request", { + request_id: requestId, + }) +} diff --git a/kubernetes/base/backend/pdb.yaml b/kubernetes/base/backend/pdb.yaml new file mode 100644 index 000000000..383578069 --- /dev/null +++ b/kubernetes/base/backend/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: backend +spec: + minAvailable: 1 + selector: + matchLabels: + app: backend diff --git a/kubernetes/base/frontend/deployment.yaml b/kubernetes/base/frontend/deployment.yaml index d60ffe24e..e95b0a1fa 100644 --- a/kubernetes/base/frontend/deployment.yaml +++ b/kubernetes/base/frontend/deployment.yaml @@ -42,6 +42,29 @@ spec: value: "0" - name: SERVER_NAME value: "localhost" + startupProbe: + httpGet: + path: / + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 24 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 volumeMounts: - name: vol-public mountPath: /data/public diff --git a/kubernetes/base/frontend/pdb.yaml b/kubernetes/base/frontend/pdb.yaml new file mode 100644 index 000000000..86e6d7804 --- /dev/null +++ b/kubernetes/base/frontend/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: frontend +spec: + minAvailable: 1 + selector: + matchLabels: + app: frontend diff --git a/kubernetes/base/hub-auth/deployment.yaml b/kubernetes/base/hub-auth/deployment.yaml index ea9725d41..62516cc11 100644 --- a/kubernetes/base/hub-auth/deployment.yaml +++ b/kubernetes/base/hub-auth/deployment.yaml @@ -27,6 +27,23 @@ spec: imagePullPolicy: Always ports: - containerPort: 80 + name: http + startupProbe: + tcpSocket: + port: http + periodSeconds: 5 + failureThreshold: 12 + readinessProbe: + tcpSocket: + port: http + periodSeconds: 10 + failureThreshold: 3 + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 3 resources: requests: memory: "128Mi" diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml index 3c1362d65..f23110433 100644 --- a/kubernetes/base/kustomization.yaml +++ b/kubernetes/base/kustomization.yaml @@ -19,6 +19,10 @@ resources: - ./judge-server/service.yaml - ./hub-auth/service.yaml + # Application Disruption Budgets + - ./backend/pdb.yaml + - ./frontend/pdb.yaml + # Application Deployments - ./backend/deployment.yaml - ./frontend/deployment.yaml diff --git a/kubernetes/base/postgres/README.md b/kubernetes/base/postgres/README.md index 358ecb11c..454b2e74e 100644 --- a/kubernetes/base/postgres/README.md +++ b/kubernetes/base/postgres/README.md @@ -5,6 +5,7 @@ ### 핵심 파일 - `postgres.yaml`: CNPG Operator를 통해 PostgreSQL 클러스터를 생성하기 위한 Custom Resource(CR) 정의 파일입니다. +- CNPG instance manager는 각 PostgreSQL Pod의 `metrics` 포트에서 Prometheus metric을 제공합니다. CodePlace monitoring은 별도 PodMonitor로 이 endpoint를 scrape합니다. --- @@ -60,3 +61,14 @@ kustomize build kubernetes/overlays/dev | kubectl apply -f - ``` > **관련 문서:** 더 자세한 정보는 [CloudNativePG 공식 문서](https://cloudnative-pg.io/documentation/current/)를 참고하세요. + +## 3. Monitoring + +PostgreSQL exporter scrape는 `kubernetes/monitoring/datastore-pod-monitors.yaml`에서 관리합니다. kube-prometheus-stack 설치 후 Prometheus Targets에서 `postgres` target이 healthy인지 확인합니다. + +주요 지표: + +- `cnpg_collector_up` +- `cnpg_collector_last_collection_error` +- `cnpg_collector_nodes_used` +- `cnpg_collector_wal_bytes` diff --git a/kubernetes/base/redis/README.md b/kubernetes/base/redis/README.md index cecacc15c..aec37cdf5 100644 --- a/kubernetes/base/redis/README.md +++ b/kubernetes/base/redis/README.md @@ -6,6 +6,7 @@ - `RedisReplication`: Redis 3개 Pod로 master/replica 구성을 만듭니다. - `RedisSentinel`: Sentinel 3개 Pod로 master 감시와 failover를 수행합니다. +- `redisExporter`: Redis/Sentinel Pod에 exporter sidecar를 붙여 Prometheus metric을 노출합니다. - 애플리케이션은 `redis-sentinel:26379`로 접속하고, master group 이름은 `mymaster`를 사용합니다. - `RedisSentinel` CR 이름은 `redis`입니다. Opstree Operator가 `-sentinel` Service를 생성하므로 실제 Sentinel Service 이름은 `redis-sentinel`입니다. @@ -57,3 +58,5 @@ REDIS_SENTINEL_HOSTS=redis-sentinel:26379 Redis와 Sentinel Pod는 `kubernetes.io/hostname` 기준으로 서로 다른 노드에 분산되도록 설정되어 있습니다. 이 구성은 최소 3개 이상의 스케줄 가능한 Kubernetes 노드를 전제로 합니다. 노드가 3개 미만이면 HA를 만족할 수 없으므로 Pod가 pending 상태로 남을 수 있습니다. Sentinel failover는 Redis master 장애 시 자동 승격을 수행하지만, failover 중에는 짧은 연결 재시도 구간이 생길 수 있습니다. 완전한 무중단에 가깝게 운영하려면 Longhorn replica와 Kubernetes 노드 장애 조건이 맞는지, 애플리케이션 retry 정책이 충분한지 함께 점검해야 합니다. + +Redis exporter는 `kubernetes/monitoring/datastore-pod-monitors.yaml`의 PodMonitor가 scrape합니다. kube-prometheus-stack 설치 후 Prometheus Targets에서 `redis` target이 healthy인지 확인합니다. diff --git a/kubernetes/base/redis/redis.yaml b/kubernetes/base/redis/redis.yaml index 0cb0f11b4..3a562186e 100644 --- a/kubernetes/base/redis/redis.yaml +++ b/kubernetes/base/redis/redis.yaml @@ -4,6 +4,18 @@ metadata: name: redis-replication spec: clusterSize: 3 + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:1.0 + imagePullPolicy: IfNotPresent + port: 9121 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi kubernetesConfig: image: quay.io/opstree/redis:v7.0.15 imagePullPolicy: IfNotPresent @@ -42,6 +54,18 @@ metadata: name: redis spec: clusterSize: 3 + redisExporter: + enabled: true + image: quay.io/opstree/redis-exporter:1.0 + imagePullPolicy: IfNotPresent + port: 9121 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi pdb: enabled: true minAvailable: 2 diff --git a/kubernetes/base/vllm/README.md b/kubernetes/base/vllm/README.md index 6b066ed0d..4d0fdf2f5 100644 --- a/kubernetes/base/vllm/README.md +++ b/kubernetes/base/vllm/README.md @@ -52,3 +52,27 @@ kubectl describe pod -n code-place-prod -l app=vllm - Longhorn replica 수는 YAML 에서 고정하지 않고, 필요하면 Longhorn UI 에서 직접 조정하는 전제를 둡니다. - `max-num-seqs=60` 은 처리량 위주 값이라, 메모리 압박이나 OOM 이 보이면 가장 먼저 낮춰야 합니다. - 더 보수적으로 운영하려면 이미지 tag 대신 digest 로 pin 하는 것이 가장 안전합니다. + +## Monitoring + +vLLM OpenAI-compatible server는 `/metrics`에서 Prometheus metrics를 제공합니다. +CodePlace monitoring kustomization은 prod vLLM Service를 `vllm-service-monitor.yaml`로 scrape합니다. +같은 monitoring kustomization은 `dcgm-exporter.yaml`도 배포해 `workload.code-place.ai/vllm=true` node의 NVIDIA GPU metric을 수집합니다. + +```bash +kubectl apply -k kubernetes/monitoring +kubectl -n monitoring get servicemonitor vllm +kubectl -n monitoring get daemonset,servicemonitor dcgm-exporter +``` + +Grafana에서는 `CodePlace AI Inference` dashboard에서 다음 상태를 확인합니다. + +- vLLM scrape up / Pod ready. +- running/waiting request count. +- KV cache usage. +- p95 e2e latency, queue latency, time-to-first-token. +- prompt/generation token throughput. +- `vllm-hf-cache` PVC usage. +- GPU utilization, framebuffer memory usage, temperature, XID error. + +알림은 vLLM scrape 실패와 GPU XID error를 P0로 처리합니다. waiting queue 증가, KV cache 90% 초과, p95 latency 60초 초과, DCGM exporter unavailable, GPU utilization 95% 초과, GPU framebuffer memory 90% 초과, GPU temperature 85C 초과는 P1로 처리합니다. diff --git a/kubernetes/monitoring/alertmanager-config.yaml b/kubernetes/monitoring/alertmanager-config.yaml index c26d85c4d..09a099516 100644 --- a/kubernetes/monitoring/alertmanager-config.yaml +++ b/kubernetes/monitoring/alertmanager-config.yaml @@ -13,7 +13,7 @@ spec: groupWait: 30s groupInterval: 5m repeatInterval: 1h - receiver: p1-discord + receiver: unmatched-muted routes: - matchers: - name: priority @@ -22,6 +22,15 @@ spec: groupInterval: 1m repeatInterval: 15m receiver: p0-discord + - matchers: + - name: priority + value: P1 + - name: namespace + value: code-place-dev + groupWait: 30s + groupInterval: 5m + repeatInterval: 1h + receiver: dev-p1-muted - matchers: - name: priority value: P1 @@ -30,6 +39,7 @@ spec: repeatInterval: 1h receiver: p1-discord receivers: + - name: unmatched-muted - name: p0-discord discordConfigs: - sendResolved: true @@ -38,8 +48,17 @@ spec: key: webhook-url title: '[{{ .Status }}][P0] {{ .CommonLabels.alertname }}' content: '{{ .CommonAnnotations.summary }}' - message: '{{ .CommonAnnotations.description }}' + message: |- + {{ .CommonAnnotations.description }} + + Priority: {{ .CommonLabels.priority }} + Namespace: {{ .CommonLabels.namespace }} + Overview: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-namespace={{ .CommonLabels.namespace }} + Logs: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-namespace={{ .CommonLabels.namespace }} + Monitoring: https://monitoring.code-place-dev.site/d/codeplace-monitoring-stack/codeplace-monitoring-stack?orgId=1 + Source: {{ range .Alerts }}{{ .GeneratorURL }}{{ end }} username: CodePlace Alertmanager + - name: dev-p1-muted - name: p1-discord discordConfigs: - sendResolved: true @@ -48,5 +67,13 @@ spec: key: webhook-url title: '[{{ .Status }}][P1] {{ .CommonLabels.alertname }}' content: '{{ .CommonAnnotations.summary }}' - message: '{{ .CommonAnnotations.description }}' + message: |- + {{ .CommonAnnotations.description }} + + Priority: {{ .CommonLabels.priority }} + Namespace: {{ .CommonLabels.namespace }} + Overview: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-namespace={{ .CommonLabels.namespace }} + Logs: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-namespace={{ .CommonLabels.namespace }} + Monitoring: https://monitoring.code-place-dev.site/d/codeplace-monitoring-stack/codeplace-monitoring-stack?orgId=1 + Source: {{ range .Alerts }}{{ .GeneratorURL }}{{ end }} username: CodePlace Alertmanager diff --git a/kubernetes/monitoring/backend-service-monitor.yaml b/kubernetes/monitoring/backend-service-monitor.yaml index 20966309b..928150fc2 100644 --- a/kubernetes/monitoring/backend-service-monitor.yaml +++ b/kubernetes/monitoring/backend-service-monitor.yaml @@ -6,7 +6,9 @@ metadata: release: kube-prometheus-stack spec: namespaceSelector: - any: true + matchNames: + - code-place-dev + - code-place-prod selector: matchLabels: app: backend diff --git a/kubernetes/monitoring/blackbox-exporter.yaml b/kubernetes/monitoring/blackbox-exporter.yaml new file mode 100644 index 000000000..8b7d773f0 --- /dev/null +++ b/kubernetes/monitoring/blackbox-exporter.yaml @@ -0,0 +1,120 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: blackbox-exporter-config +data: + blackbox.yml: | + modules: + http_2xx: + prober: http + timeout: 10s + http: + method: GET + preferred_ip_protocol: ip4 + valid_http_versions: + - HTTP/1.1 + - HTTP/2.0 + valid_status_codes: + - 200 + fail_if_not_ssl: true + tls_config: + insecure_skip_verify: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: blackbox-exporter + labels: + app: blackbox-exporter +spec: + replicas: 2 + selector: + matchLabels: + app: blackbox-exporter + template: + metadata: + labels: + app: blackbox-exporter + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: blackbox-exporter + containers: + - name: blackbox-exporter + image: quay.io/prometheus/blackbox-exporter:v0.27.0 + args: + - --config.file=/etc/blackbox-exporter/blackbox.yml + ports: + - name: http + containerPort: 9115 + readinessProbe: + httpGet: + path: /-/ready + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /-/healthy + port: http + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: config + mountPath: /etc/blackbox-exporter + readOnly: true + volumes: + - name: config + configMap: + name: blackbox-exporter-config +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: blackbox-exporter +spec: + minAvailable: 1 + selector: + matchLabels: + app: blackbox-exporter +--- +apiVersion: v1 +kind: Service +metadata: + name: blackbox-exporter + labels: + app: blackbox-exporter +spec: + selector: + app: blackbox-exporter + ports: + - name: http + port: 9115 + targetPort: http +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: blackbox-exporter + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: blackbox-exporter + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/datastore-pod-monitors.yaml b/kubernetes/monitoring/datastore-pod-monitors.yaml new file mode 100644 index 000000000..911b5f9ba --- /dev/null +++ b/kubernetes/monitoring/datastore-pod-monitors.yaml @@ -0,0 +1,41 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: postgres + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - code-place-dev + - code-place-prod + selector: + matchLabels: + cnpg.io/cluster: postgres + podMetricsEndpoints: + - port: metrics + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: redis + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - code-place-dev + - code-place-prod + selector: + matchExpressions: + - key: app + operator: In + values: + - redis-replication + - redis-sentinel + podMetricsEndpoints: + - portNumber: 9121 + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/dcgm-exporter.yaml b/kubernetes/monitoring/dcgm-exporter.yaml new file mode 100644 index 000000000..a574bd9a1 --- /dev/null +++ b/kubernetes/monitoring/dcgm-exporter.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dcgm-exporter + labels: + app.kubernetes.io/name: dcgm-exporter + app.kubernetes.io/version: "4.8.2" +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: dcgm-exporter + template: + metadata: + labels: + app.kubernetes.io/name: dcgm-exporter + app.kubernetes.io/version: "4.8.2" + spec: + automountServiceAccountToken: false + runtimeClassName: nvidia + nodeSelector: + workload.code-place.ai/vllm: "true" + containers: + - name: dcgm-exporter + image: nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless + env: + - name: DCGM_EXPORTER_LISTEN + value: ":9400" + - name: DCGM_EXPORTER_KUBERNETES + value: "true" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: metrics + containerPort: 9400 + securityContext: + runAsNonRoot: false + runAsUser: 0 + allowPrivilegeEscalation: false + capabilities: + add: + - SYS_ADMIN + drop: + - ALL + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + volumeMounts: + - name: pod-gpu-resources + mountPath: /var/lib/kubelet/pod-resources + readOnly: true + volumes: + - name: pod-gpu-resources + hostPath: + path: /var/lib/kubelet/pod-resources +--- +apiVersion: v1 +kind: Service +metadata: + name: dcgm-exporter + labels: + app.kubernetes.io/name: dcgm-exporter + app.kubernetes.io/version: "4.8.2" +spec: + selector: + app.kubernetes.io/name: dcgm-exporter + ports: + - name: metrics + port: 9400 + targetPort: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: dcgm-exporter + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: dcgm-exporter + jobLabel: app.kubernetes.io/name + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml new file mode 100644 index 000000000..f0ccc861c --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml @@ -0,0 +1,352 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-ai-inference + labels: + grafana_dashboard: "1" +data: + codeplace-ai-inference.json: | + { + "uid": "codeplace-ai-inference", + "title": "CodePlace AI Inference", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "ai", "vllm"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "vLLM Scrape Up", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "up{job=\"vllm\", namespace=\"code-place-prod\"}", "legendFormat": "vllm"} + ] + }, + { + "type": "stat", + "title": "vLLM Pod Ready", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"code-place-prod\", condition=\"true\", pod=~\"vllm-.*\"})", "legendFormat": "ready"} + ] + }, + { + "type": "stat", + "title": "Running Requests", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max({__name__=\"vllm:num_requests_running\", namespace=\"code-place-prod\"})", "legendFormat": "running"} + ] + }, + { + "type": "stat", + "title": "Waiting Requests", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max({__name__=\"vllm:num_requests_waiting\", namespace=\"code-place-prod\"})", "legendFormat": "waiting"} + ] + }, + { + "type": "stat", + "title": "KV Cache Usage", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.9} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max({__name__=\"vllm:kv_cache_usage_perc\", namespace=\"code-place-prod\"})", "legendFormat": "kv-cache"} + ] + }, + { + "type": "stat", + "title": "HF Cache PVC", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} + ] + }, + { + "type": "timeseries", + "title": "vLLM Requests", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "{__name__=\"vllm:num_requests_running\", namespace=\"code-place-prod\"}", "legendFormat": "running"}, + {"expr": "{__name__=\"vllm:num_requests_waiting\", namespace=\"code-place-prod\"}", "legendFormat": "waiting"} + ] + }, + { + "type": "timeseries", + "title": "vLLM Request Success Rate", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (finished_reason) (rate({__name__=\"vllm:request_success_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "{{finished_reason}}"} + ] + }, + { + "type": "timeseries", + "title": "vLLM p95 Latency", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"vllm:e2e_request_latency_seconds_bucket\", namespace=\"code-place-prod\"}[10m])))", "legendFormat": "e2e p95"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"vllm:request_queue_time_seconds_bucket\", namespace=\"code-place-prod\"}[10m])))", "legendFormat": "queue p95"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"vllm:time_to_first_token_seconds_bucket\", namespace=\"code-place-prod\"}[10m])))", "legendFormat": "ttft p95"} + ] + }, + { + "type": "timeseries", + "title": "vLLM Token Throughput", + "gridPos": {"x": 0, "y": 12, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum(rate({__name__=\"vllm:prompt_tokens_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "prompt tokens/s"}, + {"expr": "sum(rate({__name__=\"vllm:generation_tokens_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "generation tokens/s"} + ] + }, + { + "type": "timeseries", + "title": "vLLM Pod Resources", + "gridPos": {"x": 8, "y": 12, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"code-place-prod\", pod=~\"vllm-.*\", container=\"vllm\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, + {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"code-place-prod\", pod=~\"vllm-.*\", container=\"vllm\"})", "legendFormat": "{{pod}} memory bytes"} + ] + }, + { + "type": "timeseries", + "title": "vLLM HF Cache PVC", + "gridPos": {"x": 16, "y": 12, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} + ] + }, + { + "type": "timeseries", + "title": "AI Hint API Outcomes", + "gridPos": {"x": 0, "y": 20, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (namespace, scope, status) (rate(codeplace_ai_hint_api_outcome_total{namespace=~\"code-place-(dev|prod)\"}[5m]))", "legendFormat": "{{namespace}} {{scope}} {{status}}/s"} + ] + }, + { + "type": "timeseries", + "title": "Backend AI Hint Streams", + "gridPos": {"x": 8, "y": 20, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (namespace, status) (rate(codeplace_ai_hint_requests_total{namespace=~\"code-place-(dev|prod)\"}[5m]))", "legendFormat": "{{namespace}} {{status}} streams/s"} + ] + }, + { + "type": "timeseries", + "title": "Backend AI Hint Duration", + "gridPos": {"x": 16, "y": 20, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=~\"code-place-(dev|prod)\", status=\"success\"}[10m])))", "legendFormat": "{{namespace}} success p95"}, + {"expr": "histogram_quantile(0.95, sum by (namespace, status, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=~\"code-place-(dev|prod)\", status!=\"success\"}[10m])))", "legendFormat": "{{namespace}} {{status}} p95"} + ] + }, + { + "type": "timeseries", + "title": "GPU Utilization", + "gridPos": {"x": 0, "y": 28, "w": 6, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 80}, + {"color": "red", "value": 95} + ] + } + } + }, + "targets": [ + {"expr": "DCGM_FI_DEV_GPU_UTIL{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}"} + ] + }, + { + "type": "timeseries", + "title": "GPU FB Memory Usage", + "gridPos": {"x": 6, "y": 28, "w": 6, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.8}, + {"color": "red", "value": 0.9} + ] + } + } + }, + "targets": [ + {"expr": "DCGM_FI_DEV_FB_USED{namespace=\"monitoring\"} / clamp_min(DCGM_FI_DEV_FB_USED{namespace=\"monitoring\"} + DCGM_FI_DEV_FB_FREE{namespace=\"monitoring\"}, 1)", "legendFormat": "{{Hostname}} gpu{{gpu}}"} + ] + }, + { + "type": "timeseries", + "title": "GPU Temperature", + "gridPos": {"x": 12, "y": 28, "w": 6, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 75}, + {"color": "red", "value": 85} + ] + } + } + }, + "targets": [ + {"expr": "DCGM_FI_DEV_GPU_TEMP{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}"} + ] + }, + { + "type": "timeseries", + "title": "GPU XID Errors", + "gridPos": {"x": 18, "y": 28, "w": 6, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "targets": [ + {"expr": "DCGM_FI_DEV_XID_ERRORS{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}}"} + ] + }, + { + "type": "timeseries", + "title": "GPU Tensor / DRAM Activity", + "gridPos": {"x": 0, "y": 36, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "targets": [ + {"expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}} tensor"}, + {"expr": "DCGM_FI_PROF_DRAM_ACTIVE{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}} dram"} + ] + }, + { + "type": "timeseries", + "title": "GPU FB Memory Bytes", + "gridPos": {"x": 12, "y": 36, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "DCGM_FI_DEV_FB_USED{namespace=\"monitoring\"} * 1024 * 1024", "legendFormat": "{{Hostname}} gpu{{gpu}} used"}, + {"expr": "DCGM_FI_DEV_FB_FREE{namespace=\"monitoring\"} * 1024 * 1024", "legendFormat": "{{Hostname}} gpu{{gpu}} free"} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 43e73b13d..8a55a6bc4 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -13,6 +13,7 @@ data: "version": 1, "refresh": "30s", "tags": ["codeplace"], + "editable": true, "time": { "from": "now-1h", "to": "now" @@ -36,52 +37,451 @@ data: ] }, "panels": [ + { + "type": "stat", + "title": "Backend Up", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (up{job=\"backend\", namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "API / Ingress Request Rate", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "reqps"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace:api_request_rate5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} api"}, + {"expr": "codeplace:ingress_request_rate2m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} ingress"} + ] + }, + { + "type": "stat", + "title": "API / Ingress 5xx Ratio", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.01}, + {"color": "red", "value": 0.05} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace:api_5xx_ratio5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} api"}, + {"expr": "codeplace:ingress_5xx_rate2m{namespace=~\"$namespace\"} / clamp_min(codeplace:ingress_request_rate2m{namespace=~\"$namespace\"}, 1)", "legendFormat": "{{namespace}} ingress"} + ] + }, + { + "type": "stat", + "title": "p95 Latency", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 2} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace:api_p95_latency_seconds5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "Judge Waiting Queue", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 3}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace:judge_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} + ] + }, { "type": "timeseries", - "title": "Backend Request Rate", - "gridPos": {"x": 0, "y": 0, "w": 8, "h": 8}, + "title": "Backend Requests by Status", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "reqps"}}, "targets": [ - {"expr": "sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} requests/s"} + {"expr": "sum by (namespace, status_code) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{status_code}}"} ] }, { "type": "timeseries", - "title": "Backend 5xx Rate", - "gridPos": {"x": 8, "y": 0, "w": 8, "h": 8}, + "title": "Backend Latency", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, "targets": [ - {"expr": "sum by (namespace) (rate(codeplace_http_requests_total{namespace=~\"$namespace\", status_code=~\"5..\"}[5m]))", "legendFormat": "{{namespace}} 5xx/s"} + {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p95"}, + {"expr": "histogram_quantile(0.99, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p99"} ] }, + { + "type": "bargauge", + "title": "Judge Queue Length", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} waiting_queue"}, + {"expr": "codeplace_celery_broker_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} celery"} + ] + }, + { + "type": "row", + "title": "Judge / Async Outcomes", + "gridPos": {"x": 0, "y": 12, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, { "type": "timeseries", - "title": "Backend p95 Latency", - "gridPos": {"x": 16, "y": 0, "w": 8, "h": 8}, + "title": "Submission / Judge Outcomes", + "gridPos": {"x": 0, "y": 20, "w": 24, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (namespace, scope, status) (rate(codeplace_submission_create_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} create {{scope}} {{status}}/s"}, + {"expr": "sum by (namespace, scope, status) (rate(codeplace_judge_task_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} judge {{scope}} {{status}}/s"} + ] + }, + { + "type": "row", + "title": "Platform", + "gridPos": {"x": 0, "y": 36, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "Pod Ready", + "gridPos": {"x": 0, "y": 37, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p95"} + {"expr": "min by (namespace, pod) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}}"} ] }, { "type": "timeseries", - "title": "Submission Status", - "gridPos": {"x": 0, "y": 8, "w": 8, "h": 8}, + "title": "Pod Restarts", + "gridPos": {"x": 6, "y": 37, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "codeplace_submission_status_count{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{status}}"} + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} ] }, { "type": "timeseries", - "title": "Judge Waiting Queue", - "gridPos": {"x": 8, "y": 8, "w": 8, "h": 8}, + "title": "Container CPU Usage", + "gridPos": {"x": 12, "y": 37, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "cores"}}, + "targets": [ + {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + ] + }, + { + "type": "timeseries", + "title": "Container Memory Usage", + "gridPos": {"x": 18, "y": 37, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + ] + }, + { + "type": "timeseries", + "title": "PVC Usage", + "gridPos": {"x": 0, "y": 42, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}), 1)", "legendFormat": "{{namespace}} {{persistentvolumeclaim}}"} + ] + }, + { + "type": "timeseries", + "title": "Node Pressure", + "gridPos": {"x": 8, "y": 42, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "kube_node_status_condition{condition=~\"MemoryPressure|DiskPressure|PIDPressure\", status=\"true\"}", "legendFormat": "{{node}} {{condition}}"} + ] + }, + { + "type": "timeseries", + "title": "Deployment Unavailable Replicas", + "gridPos": {"x": 16, "y": 42, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "max by (namespace, deployment) (kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} unavailable"}, + {"expr": "max by (namespace, deployment) (kube_deployment_status_observed_generation{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"} != on (namespace, deployment) kube_deployment_metadata_generation{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} generation mismatch"} + ] + }, + { + "type": "timeseries", + "title": "Service Ready Endpoints", + "gridPos": {"x": 0, "y": 48, "w": 12, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "targets": [ + {"expr": "sum by (namespace, endpoint) (kube_endpoint_address{namespace=~\"$namespace\", endpoint=~\"backend|frontend|hub-auth|judge-server|vllm\", ready=\"true\"})", "legendFormat": "{{namespace}} {{endpoint}} ready endpoints"} + ] + }, + { + "type": "timeseries", + "title": "PDB Healthy Pods", + "gridPos": {"x": 12, "y": 48, "w": 12, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "kube_poddisruptionbudget_status_current_healthy{namespace=~\"$namespace\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} current"}, + {"expr": "kube_poddisruptionbudget_status_desired_healthy{namespace=~\"$namespace\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} desired"} + ] + }, + { + "type": "row", + "title": "Data Stores", + "gridPos": {"x": 0, "y": 54, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "PostgreSQL Ready Replicas", + "gridPos": {"x": 0, "y": 55, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 3} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"postgres-[0-9]+\"}) or vector(0)", "legendFormat": "{{namespace}} ready"} + ] + }, + { + "type": "stat", + "title": "Redis/Sentinel Ready Replicas", + "gridPos": {"x": 6, "y": 55, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 3}, + {"color": "green", "value": 6} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"redis-.*|redis-replication-.*\"}) or vector(0)", "legendFormat": "{{namespace}} ready"} + ] + }, + { + "type": "stat", + "title": "PostgreSQL Exporter Up", + "gridPos": {"x": 12, "y": 55, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (cnpg_collector_up{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "stat", + "title": "Redis Exporter Up", + "gridPos": {"x": 18, "y": 55, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (redis_up{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL Restarts", + "gridPos": {"x": 0, "y": 60, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Redis Restarts", + "gridPos": {"x": 6, "y": 60, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL WAL Bytes", + "gridPos": {"x": 12, "y": 60, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "Bps"}}, + "targets": [ + {"expr": "sum by (namespace) (rate(cnpg_collector_wal_bytes{namespace=~\"$namespace\", cluster=\"postgres\"}[5m]))", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "timeseries", + "title": "Redis Memory Usage", + "gridPos": {"x": 18, "y": 60, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} used"}, + {"expr": "max by (namespace, pod) (redis_memory_max_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} maxmemory"} + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL Collector Errors", + "gridPos": {"x": 0, "y": 65, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "codeplace_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} waiting"} + {"expr": "max by (namespace, pod) (cnpg_collector_last_collection_error{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} {{pod}}"} ] }, { "type": "timeseries", - "title": "Judge Heartbeat Age", - "gridPos": {"x": 16, "y": 8, "w": 8, "h": 8}, + "title": "PostgreSQL Nodes Used", + "gridPos": {"x": 8, "y": 65, "w": 16, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "codeplace_judge_server_last_heartbeat_age_seconds{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} {{hostname}}"} + {"expr": "min by (namespace) (cnpg_collector_nodes_used{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} nodes"} ] } ] diff --git a/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml b/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml new file mode 100644 index 000000000..8dec31960 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml @@ -0,0 +1,221 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-kubernetes-events + labels: + grafana_dashboard: "1" +data: + codeplace-kubernetes-events.json: | + { + "uid": "codeplace-kubernetes-events", + "title": "CodePlace Kubernetes Events", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "kubernetes", "events"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "namespace", + "type": "custom", + "query": "code-place-dev,code-place-prod,monitoring,longhorn-system,kube-system", + "multi": true, + "includeAll": true, + "allValue": "code-place-dev|code-place-prod|monitoring|longhorn-system|kube-system", + "current": { + "text": "code-place-dev", + "value": "code-place-dev" + } + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Event Exporter Ready", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kubernetes-event-exporter-.*\"}) or vector(0)", "legendFormat": "ready"} + ] + }, + { + "type": "stat", + "title": "Image Pull BackOff", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=~\"ImagePullBackOff|ErrImagePull\"}) or vector(0)", "legendFormat": "pods"} + ] + }, + { + "type": "stat", + "title": "CrashLoop BackOff", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=\"CrashLoopBackOff\"}) or vector(0)", "legendFormat": "pods"} + ] + }, + { + "type": "stat", + "title": "OOMKilled", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\", reason=\"OOMKilled\"}[10m])) or vector(0)", "legendFormat": "containers"} + ] + }, + { + "type": "stat", + "title": "Pending Pods", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)", "legendFormat": "pods"} + ] + }, + { + "type": "stat", + "title": "Unschedulable Pods", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_unschedulable{namespace=~\"$namespace\"}) or vector(0)", "legendFormat": "pods"} + ] + }, + { + "type": "timeseries", + "title": "Container Waiting Reasons", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (namespace, pod, container, reason) (kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=~\"CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError\"})", "legendFormat": "{{namespace}} {{pod}} {{container}} {{reason}}"} + ] + }, + { + "type": "timeseries", + "title": "Recent OOMKilled Containers", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "max by (namespace, pod, container) (max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\", reason=\"OOMKilled\"}[10m]))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + ] + }, + { + "type": "logs", + "title": "Kubernetes Warning Events", + "gridPos": {"x": 0, "y": 12, "w": 24, "h": 9}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=\"monitoring\", app_kubernetes_io_name=\"kubernetes-event-exporter\"}"} + ] + }, + { + "type": "logs", + "title": "Scheduling, Pull, Restart, and Probe Events", + "gridPos": {"x": 0, "y": 21, "w": 24, "h": 9}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=\"monitoring\", app_kubernetes_io_name=\"kubernetes-event-exporter\"} |~ \"FailedScheduling|ImagePullBackOff|ErrImagePull|BackOff|OOMKilling|Unhealthy|FailedMount|FailedAttachVolume\""} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml new file mode 100644 index 000000000..48e62d9a0 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -0,0 +1,347 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-logs + labels: + grafana_dashboard: "1" +data: + codeplace-logs.json: | + { + "uid": "codeplace-logs", + "title": "CodePlace Logs", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "logs"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "namespace", + "type": "custom", + "query": "code-place-dev,code-place-prod,monitoring", + "multi": true, + "includeAll": true, + "allValue": "code-place-dev|code-place-prod|monitoring", + "current": { + "text": "code-place-dev", + "value": "code-place-dev" + } + }, + { + "name": "request_id", + "type": "textbox", + "query": ".*", + "current": { + "text": ".*", + "value": ".*" + } + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Loki Ready Replicas", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-[0-9]+\"}) or vector(0)", "legendFormat": "single-binary"}, + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-gateway-.*\"}) or vector(0)", "legendFormat": "gateway"} + ] + }, + { + "type": "stat", + "title": "Alloy Nodes", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_daemonset_status_number_available{namespace=\"monitoring\", daemonset=\"alloy\"} or vector(0)", "legendFormat": "available"}, + {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"monitoring\", daemonset=\"alloy\"} or vector(0)", "legendFormat": "desired"} + ] + }, + { + "type": "stat", + "title": "Loki PVC Usage", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}), 1) or vector(0)", "legendFormat": "usage"} + ] + }, + { + "type": "stat", + "title": "Log Lines", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(count_over_time({namespace=~\"$namespace\"}[5m]))", "legendFormat": "lines"} + ] + }, + { + "type": "stat", + "title": "Backend Errors", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Loki", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"backend\"} | json | level=\"ERROR\" [5m]))", "legendFormat": "errors"} + ] + }, + { + "type": "stat", + "title": "Frontend 5xx", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Loki", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 500 [5m]))", "legendFormat": "5xx"} + ] + }, + { + "type": "timeseries", + "title": "Log Lines by Namespace", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 7}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum by (namespace) (rate({namespace=~\"code-place-dev|code-place-prod|monitoring|kube-system\"}[5m]))", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "timeseries", + "title": "Application Log Lines by App", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 7}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum by (namespace, app) (rate({namespace=~\"code-place-dev|code-place-prod\", app=~\"backend|frontend\"}[5m]))", "legendFormat": "{{namespace}} {{app}}"}, + {"expr": "sum by (namespace, container) (rate({namespace=~\"code-place-dev|code-place-prod\", container=~\"celery-worker|celery-beat|judge-server|hub-auth|vllm\"}[5m]))", "legendFormat": "{{namespace}} {{container}}"} + ] + }, + { + "type": "timeseries", + "title": "Structured Error Logs by Component", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 7}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum by (namespace, app) (rate({namespace=~\"code-place-dev|code-place-prod\", app=~\"backend|frontend\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{app}}"}, + {"expr": "sum by (namespace, container) (rate({namespace=~\"code-place-dev|code-place-prod\", container=~\"celery-worker|celery-beat|judge-server|hub-auth|vllm\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{container}}"} + ] + }, + { + "type": "stat", + "title": "Ingress 5xx", + "gridPos": {"x": 0, "y": 11, "w": 4, "h": 4}, + "datasource": "Loki", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(count_over_time({namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | DownstreamStatus >= 500 [5m]))", "legendFormat": "5xx"} + ] + }, + { + "type": "timeseries", + "title": "Log Lines by Node", + "gridPos": {"x": 0, "y": 15, "w": 4, "h": 6}, + "datasource": "Loki", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum by (node) (rate({namespace=~\"code-place-dev|code-place-prod|monitoring|kube-system\"}[5m]))", "legendFormat": "{{node}}"} + ] + }, + { + "type": "logs", + "title": "Recent Backend Errors", + "gridPos": {"x": 4, "y": 11, "w": 10, "h": 10}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\", app=\"backend\"} | json | level=\"ERROR\""} + ] + }, + { + "type": "logs", + "title": "Auth, Judge, Celery, and vLLM Logs", + "gridPos": {"x": 14, "y": 11, "w": 10, "h": 10}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\", container=~\"hub-auth|celery-worker|celery-beat|judge-server|vllm\"}"} + ] + }, + { + "type": "logs", + "title": "Ingress and Frontend 4xx/5xx Logs", + "gridPos": {"x": 0, "y": 21, "w": 24, "h": 8}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | DownstreamStatus >= 400", "legendFormat": "traefik"}, + {"expr": "{namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 400", "legendFormat": "frontend"} + ] + }, + { + "type": "logs", + "title": "Request ID Lookup", + "gridPos": {"x": 0, "y": 29, "w": 24, "h": 9}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\"} | json | request_id=~\"$request_id\""} + ] + }, + { + "type": "logs", + "title": "Frontend Runtime Error Reports", + "gridPos": {"x": 0, "y": 38, "w": 24, "h": 8}, + "datasource": "Loki", + "options": { + "showLabels": false, + "showTime": true, + "wrapLogMessage": true, + "dedupStrategy": "none" + }, + "targets": [ + {"expr": "{namespace=~\"$namespace\", app=\"backend\"} | json | logger=\"frontend.error\""} + ] + }, + { + "type": "timeseries", + "title": "Loki Ingestion Rate", + "gridPos": {"x": 0, "y": 46, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "targets": [ + {"expr": "sum(rate(loki_distributor_lines_received_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "lines received"}, + {"expr": "sum(rate(loki_distributor_bytes_received_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "bytes received"} + ] + }, + { + "type": "timeseries", + "title": "Loki Request Errors", + "gridPos": {"x": 8, "y": 46, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "reqps"}}, + "targets": [ + {"expr": "sum by (route, status_code) (rate(loki_request_duration_seconds_count{namespace=\"monitoring\", status_code=~\"5..\"}[5m])) or vector(0)", "legendFormat": "{{route}} {{status_code}}"}, + {"expr": "sum by (route, status_code) (rate(loki_request_duration_seconds_count{namespace=\"monitoring\", status_code=~\"4..\"}[5m])) or vector(0)", "legendFormat": "{{route}} {{status_code}}"} + ] + }, + { + "type": "timeseries", + "title": "Alloy Loki Write Backpressure", + "gridPos": {"x": 16, "y": 46, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum(rate(loki_write_batch_retries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "batch retries/s"}, + {"expr": "sum(rate(loki_write_dropped_entries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "dropped entries/s"} + ] + }, + { + "type": "timeseries", + "title": "Loki Canary Missing Entries", + "gridPos": {"x": 0, "y": 53, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum(increase(loki_canary_missing_entries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "missing entries"} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml new file mode 100644 index 000000000..7ced1fec9 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -0,0 +1,319 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-monitoring-stack + labels: + grafana_dashboard: "1" +data: + codeplace-monitoring-stack.json: | + { + "uid": "codeplace-monitoring-stack", + "title": "CodePlace Monitoring Stack", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "monitoring"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "Prometheus Ready Replicas", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\"}) or vector(0)", "legendFormat": "prometheus ready"} + ] + }, + { + "type": "stat", + "title": "Alertmanager Ready Replicas", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"alertmanager-kube-prometheus-stack-alertmanager-.*\"}) or vector(0)", "legendFormat": "alertmanager ready"} + ] + }, + { + "type": "stat", + "title": "Grafana Ready", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kube-prometheus-stack-grafana-.*\"}) or vector(0)", "legendFormat": "grafana"} + ] + }, + { + "type": "stat", + "title": "Prometheus Operator Ready", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kube-prometheus-stack-operator-.*\"}) or vector(0)", "legendFormat": "operator"} + ] + }, + { + "type": "stat", + "title": "Alertmanager Discovered", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(prometheus_notifications_alertmanagers_discovered) or vector(0)", "legendFormat": "targets"} + ] + }, + { + "type": "stat", + "title": "Rule Fail", + "gridPos": {"x": 20, "y": 0, "w": 2, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "failures"} + ] + }, + { + "type": "stat", + "title": "Grafana Metrics", + "gridPos": {"x": 0, "y": 36, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(up{namespace=\"monitoring\", service=\"kube-prometheus-stack-grafana\"}) or vector(0)", "legendFormat": "up"} + ] + }, + { + "type": "stat", + "title": "Notify Fail", + "gridPos": {"x": 22, "y": 0, "w": 2, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(increase(alertmanager_notifications_failed_total[5m])) or vector(0)", "legendFormat": "failures"} + ] + }, + { + "type": "timeseries", + "title": "Monitoring Pod Restarts", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=\"monitoring\", pod=~\"prometheus-.*|alertmanager-.*|kube-prometheus-stack-grafana-.*|kube-prometheus-stack-operator-.*\"}[1h]))", "legendFormat": "{{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Prometheus Target Scrape Failures", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (job, namespace) (up == 0) or vector(0)", "legendFormat": "{{namespace}} {{job}}"} + ] + }, + { + "type": "timeseries", + "title": "Monitoring Service Target Up", + "gridPos": {"x": 8, "y": 36, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "targets": [ + {"expr": "max by (service) (up{namespace=\"monitoring\", service=~\"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy\"})", "legendFormat": "{{service}}"} + ] + }, + { + "type": "timeseries", + "title": "Prometheus TSDB Head Series", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "prometheus_tsdb_head_series", "legendFormat": "{{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Prometheus Rule and Config Health", + "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "{{rule_group}} failures"}, + {"expr": "max(prometheus_config_last_reload_successful{namespace=\"monitoring\"}) or vector(0)", "legendFormat": "config reload successful"} + ] + }, + { + "type": "timeseries", + "title": "Alertmanager Notifications and Config", + "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (integration) (rate(alertmanager_notifications_total[5m]))", "legendFormat": "{{integration}} sent"}, + {"expr": "sum by (integration) (rate(alertmanager_notifications_failed_total[5m])) or vector(0)", "legendFormat": "{{integration}} failed"}, + {"expr": "max(alertmanager_config_last_reload_successful{namespace=\"monitoring\"}) or vector(0)", "legendFormat": "config reload successful"} + ] + }, + { + "type": "timeseries", + "title": "Prometheus Resource Usage", + "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\", container=\"prometheus\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, + {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\", container=\"prometheus\"})", "legendFormat": "{{pod}} memory bytes"} + ] + }, + { + "type": "timeseries", + "title": "Grafana Resource Usage", + "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "targets": [ + {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*\", container!=\"\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, + {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*\", container!=\"\"})", "legendFormat": "{{pod}} memory bytes"} + ] + }, + { + "type": "timeseries", + "title": "Monitoring PVC Usage", + "gridPos": {"x": 0, "y": 28, "w": 24, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "max by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim!~\".*loki.*|tempo-data\"}) / clamp_min(max by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim!~\".*loki.*|tempo-data\"}), 1) or vector(0)", "legendFormat": "{{persistentvolumeclaim}}"} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml new file mode 100644 index 000000000..6ab3c8d3f --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml @@ -0,0 +1,211 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-public-endpoints + labels: + grafana_dashboard: "1" +data: + codeplace-public-endpoints.json: | + { + "uid": "codeplace-public-endpoints", + "title": "CodePlace Public Endpoints", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "synthetic", "blackbox"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "Prod Endpoints Up", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace:public_endpoint_availability5m{environment=\"prod\"}", "legendFormat": "{{service}} {{instance}}"} + ] + }, + { + "type": "stat", + "title": "Dev Endpoints Up", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "codeplace:public_endpoint_availability5m{environment=\"dev\"}", "legendFormat": "{{service}} {{instance}}"} + ] + }, + { + "type": "stat", + "title": "HTTP Status", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 200} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "probe_http_status_code{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "stat", + "title": "TLS Days Left", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "d", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 3}, + {"color": "green", "value": 14} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "(probe_ssl_earliest_cert_expiry{probe_type=\"public-http\"} - time()) / 86400", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "stat", + "title": "Probe Duration", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 2} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "probe_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "stat", + "title": "Blackbox Exporter Replicas", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"blackbox-exporter\"} or vector(0)", "legendFormat": "available"}, + {"expr": "kube_deployment_spec_replicas{namespace=\"monitoring\", deployment=\"blackbox-exporter\"} or vector(0)", "legendFormat": "desired"} + ] + }, + { + "type": "timeseries", + "title": "Endpoint Availability", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}}, + "targets": [ + {"expr": "codeplace:public_endpoint_availability5m", "legendFormat": "{{environment}} {{service}} {{instance}}"} + ] + }, + { + "type": "timeseries", + "title": "Endpoint Duration", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "probe_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "timeseries", + "title": "TLS Days Left", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "d", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 14}, + {"color": "green", "value": 30} + ] + } + } + }, + "targets": [ + {"expr": "(probe_ssl_earliest_cert_expiry{probe_type=\"public-http\"} - time()) / 86400", "legendFormat": "{{environment}} {{instance}}"} + ] + }, + { + "type": "timeseries", + "title": "HTTP Phase Duration", + "gridPos": {"x": 0, "y": 12, "w": 24, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "s"}}, + "targets": [ + {"expr": "probe_http_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{phase}}"} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-storage.yaml b/kubernetes/monitoring/grafana-dashboard-storage.yaml new file mode 100644 index 000000000..43269d399 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-storage.yaml @@ -0,0 +1,245 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-storage + labels: + grafana_dashboard: "1" +data: + codeplace-storage.json: | + { + "uid": "codeplace-storage", + "title": "CodePlace Storage", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "storage", "longhorn"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "Longhorn Manager Replicas", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 3} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_daemonset_status_number_available{namespace=\"longhorn-system\", daemonset=\"longhorn-manager\"} or vector(0)", "legendFormat": "available"}, + {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"longhorn-system\", daemonset=\"longhorn-manager\"} or vector(0)", "legendFormat": "desired"} + ] + }, + { + "type": "stat", + "title": "Faulted Volumes", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "count(longhorn_volume_robustness == 3) or vector(0)", "legendFormat": "faulted"} + ] + }, + { + "type": "stat", + "title": "Degraded Volumes", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "count(longhorn_volume_robustness == 2) or vector(0)", "legendFormat": "degraded"} + ] + }, + { + "type": "stat", + "title": "Read-only Volumes", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(longhorn_volume_file_system_read_only) or vector(0)", "legendFormat": "read-only"} + ] + }, + { + "type": "stat", + "title": "Max Node Storage", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max((longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1))", "legendFormat": "usage"} + ] + }, + { + "type": "stat", + "title": "Max Disk Usage", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max((longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1))", "legendFormat": "usage"} + ] + }, + { + "type": "timeseries", + "title": "Volume Robustness", + "description": "1 healthy, 2 degraded, 3 faulted", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}}, + "targets": [ + {"expr": "longhorn_volume_robustness", "legendFormat": "{{pvc_namespace}}/{{pvc}}"} + ] + }, + { + "type": "timeseries", + "title": "Volume Actual Size", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "longhorn_volume_actual_size_bytes", "legendFormat": "{{pvc_namespace}}/{{pvc}} actual"}, + {"expr": "longhorn_volume_capacity_bytes", "legendFormat": "{{pvc_namespace}}/{{pvc}} capacity"} + ] + }, + { + "type": "timeseries", + "title": "Node Storage Usage", + "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "(longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1)", "legendFormat": "{{node}}"} + ] + }, + { + "type": "timeseries", + "title": "Disk Usage", + "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "(longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1)", "legendFormat": "{{node}} {{disk}}"} + ] + }, + { + "type": "timeseries", + "title": "Longhorn Manager CPU", + "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "m"}}, + "targets": [ + {"expr": "longhorn_manager_cpu_usage_millicpu", "legendFormat": "{{node}}"} + ] + }, + { + "type": "timeseries", + "title": "Longhorn Manager Memory", + "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes"}}, + "targets": [ + {"expr": "longhorn_manager_memory_usage_bytes", "legendFormat": "{{node}}"} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-traces.yaml b/kubernetes/monitoring/grafana-dashboard-traces.yaml new file mode 100644 index 000000000..3831d174c --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-traces.yaml @@ -0,0 +1,176 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-traces + labels: + grafana_dashboard: "1" +data: + codeplace-traces.json: | + { + "uid": "codeplace-traces", + "title": "CodePlace Traces", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "traces"], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "type": "stat", + "title": "Tempo Ready", + "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"tempo-.*\"}) or vector(0)", "legendFormat": "tempo"} + ] + }, + { + "type": "stat", + "title": "OTel Collector Replicas", + "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"otel-collector\"} or vector(0)", "legendFormat": "available"}, + {"expr": "kube_deployment_spec_replicas{namespace=\"monitoring\", deployment=\"otel-collector\"} or vector(0)", "legendFormat": "desired"} + ] + }, + { + "type": "stat", + "title": "Trace Receive Rate", + "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "spans/s"} + ] + }, + { + "type": "stat", + "title": "Trace Export Failures", + "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "sum(increase(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "failed"} + ] + }, + { + "type": "stat", + "title": "Tempo PVC Usage", + "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}), 1) or vector(0)", "legendFormat": "usage"} + ] + }, + { + "type": "timeseries", + "title": "Collector Spans", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum by (receiver, transport) (rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "{{receiver}} {{transport}} accepted"}, + {"expr": "sum by (exporter) (rate(otelcol_exporter_sent_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "{{exporter}} sent"} + ] + }, + { + "type": "timeseries", + "title": "Collector Refused or Failed Spans", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops"}}, + "targets": [ + {"expr": "sum(rate(otelcol_receiver_refused_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "refused"}, + {"expr": "sum(rate(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "failed"} + ] + }, + { + "type": "table", + "title": "Recent Backend/Celery Traces", + "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, + "datasource": "Tempo", + "targets": [ + { + "refId": "A", + "queryType": "traceql", + "query": "{ resource.service.name =~ \"codeplace-(backend|celery)\" }", + "limit": 20 + } + ] + }, + { + "type": "table", + "title": "Submission/Judge Trace Search", + "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, + "datasource": "Tempo", + "targets": [ + { + "refId": "A", + "queryType": "traceql", + "query": "{ name =~ \"judge_task|submission.judge|judge_server.select|judge_server.request|submission.result_save\" }", + "limit": 20 + } + ] + } + ] + } diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index e63df2eea..c13ec923d 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -1,20 +1,85 @@ prometheus: prometheusSpec: + replicas: 2 + podAntiAffinity: soft + podAntiAffinityTopologyKey: kubernetes.io/hostname + replicaExternalLabelNameClear: true evaluationInterval: 15s scrapeInterval: 15s + retention: 15d + retentionSize: 45GB serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false ruleSelectorNilUsesHelmValues: false + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi alertmanager: alertmanagerSpec: + replicas: 2 + podAntiAffinity: soft + podAntiAffinityTopologyKey: kubernetes.io/hostname + alertmanagerConfigMatcherStrategy: + type: None alertmanagerConfigSelector: matchLabels: alertmanagerConfig: codeplace alertmanagerConfigNamespaceSelector: matchLabels: kubernetes.io/metadata.name: monitoring + storage: + volumeClaimTemplate: + spec: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi grafana: + grafana.ini: + metrics: + enabled: true + persistence: + enabled: true + type: pvc + storageClassName: longhorn + accessModes: + - ReadWriteOnce + size: 10Gi + additionalDataSources: + - name: Loki + uid: Loki + type: loki + access: proxy + url: http://loki-gateway.monitoring.svc.cluster.local + isDefault: false + jsonData: + timeout: 60 + - name: Tempo + uid: Tempo + type: tempo + access: proxy + url: http://tempo.monitoring.svc.cluster.local:3200 + isDefault: false + jsonData: + tracesToLogsV2: + datasourceUid: Loki + spanStartTimeShift: "-5m" + spanEndTimeShift: "5m" + filterByTraceID: false + filterBySpanID: false + customQuery: true + query: "{namespace=~\"code-place-dev|code-place-prod\"} |= \"${__span.traceId}\"" ingress: enabled: true ingressClassName: traefik diff --git a/kubernetes/monitoring/kubernetes-event-exporter.yaml b/kubernetes/monitoring/kubernetes-event-exporter.yaml new file mode 100644 index 000000000..17addff29 --- /dev/null +++ b/kubernetes/monitoring/kubernetes-event-exporter.yaml @@ -0,0 +1,140 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-event-exporter +subjects: + - kind: ServiceAccount + name: kubernetes-event-exporter + namespace: monitoring +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubernetes-event-exporter-config + labels: + app.kubernetes.io/name: kubernetes-event-exporter +data: + config.yaml: | + logLevel: warn + logFormat: json + maxEventAgeSeconds: 60 + metricsNamePrefix: event_exporter_ + route: + routes: + - drop: + - type: "Normal" + match: + - namespace: "code-place-dev|code-place-prod|monitoring|longhorn-system|kube-system" + receiver: "stdout" + receivers: + - name: "stdout" + stdout: + deDot: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubernetes-event-exporter + template: + metadata: + labels: + app.kubernetes.io/name: kubernetes-event-exporter + spec: + serviceAccountName: kubernetes-event-exporter + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: event-exporter + image: ghcr.io/resmoio/kubernetes-event-exporter:v1.7 + imagePullPolicy: IfNotPresent + args: + - -conf=/data/config.yaml + ports: + - name: metrics + containerPort: 2112 + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: config + mountPath: /data + readOnly: true + volumes: + - name: config + configMap: + name: kubernetes-event-exporter-config +--- +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/name: kubernetes-event-exporter +spec: + selector: + app.kubernetes.io/name: kubernetes-event-exporter + ports: + - name: metrics + port: 2112 + targetPort: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kubernetes-event-exporter + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: kubernetes-event-exporter + jobLabel: app.kubernetes.io/name + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index 02e452345..b140e15ab 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -5,5 +5,22 @@ namespace: monitoring resources: - ./alertmanager-config.yaml - ./backend-service-monitor.yaml + - ./blackbox-exporter.yaml + - ./dcgm-exporter.yaml + - ./datastore-pod-monitors.yaml - ./grafana-dashboard-codeplace.yaml + - ./grafana-dashboard-ai-inference.yaml + - ./grafana-dashboard-logs.yaml + - ./grafana-dashboard-kubernetes-events.yaml + - ./grafana-dashboard-monitoring-stack.yaml + - ./grafana-dashboard-public-endpoints.yaml + - ./grafana-dashboard-storage.yaml + - ./grafana-dashboard-traces.yaml + - ./kubernetes-event-exporter.yaml + - ./longhorn-service-monitor.yaml + - ./otel-collector.yaml - ./prometheus-rules.yaml + - ./public-endpoint-probes.yaml + - ./tempo.yaml + - ./traefik-service-monitor.yaml + - ./vllm-service-monitor.yaml diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md new file mode 100644 index 000000000..60c42d41d --- /dev/null +++ b/kubernetes/monitoring/logs/README.md @@ -0,0 +1,92 @@ +# CodePlace Logs Stack + +This directory contains Helm values for the Kubernetes logs stack. + +- Loki: SingleBinary mode, filesystem storage, Longhorn PVC. +- Loki gateway: 2 replicas for the stateless write/query entrypoint. +- Alloy: DaemonSet log collector for every pod in `code-place-dev`, `code-place-prod`, and `monitoring`, plus Traefik pods in `kube-system`. It mounts host `/var/log` so `/var/log/pods` is readable from the Alloy container. It tolerates the standard `control-plane` and `master` NoSchedule taints so the current three-node cluster does not silently miss logs from a tainted control-plane node. +- Kubernetes Event Exporter: Warning events are written to stdout as JSON and collected by Alloy from the `monitoring` namespace. +- Retention: `code-place-dev` 3 days, `code-place-prod` 7 days. +- Monitoring: Loki and Alloy ServiceMonitors are enabled for kube-prometheus-stack. + +This is the selected baseline while cloud object storage is unavailable. Longhorn PVC is better than local hostPath or emptyDir because it gives Kubernetes-managed persistence and operationally visible volume health without introducing a new storage service. It is still not a durable log archive and should be treated as short-retention troubleshooting storage. + +Pinned chart versions: + +- `prometheus-community/kube-prometheus-stack`: `86.3.1` +- `grafana/loki`: `6.55.0` +- `grafana/alloy`: `1.10.0` + +## Install or Upgrade + +Install or upgrade kube-prometheus-stack first so the Prometheus Operator CRDs are present and the Grafana `Loki` datasource is provisioned. Then install Loki and Alloy. +The kube-prometheus-stack values set `alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy.type=None` so the `monitoring` namespace AlertmanagerConfig can route CodePlace alerts whose `namespace` labels are `code-place-dev` or `code-place-prod`. + +```sh +helm repo add grafana https://grafana.github.io/helm-charts +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update + +helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ + --namespace monitoring \ + --create-namespace \ + --version 86.3.1 \ + --values kubernetes/monitoring/kube-prometheus-stack-values.yaml + +helm upgrade --install loki grafana/loki \ + --namespace monitoring \ + --create-namespace \ + --version 6.55.0 \ + --values kubernetes/monitoring/logs/loki-values.yaml + +helm upgrade --install alloy grafana/alloy \ + --namespace monitoring \ + --create-namespace \ + --version 1.10.0 \ + --values kubernetes/monitoring/logs/alloy-values.yaml +``` + +## Verify + +After applying the monitoring resources, check the live resources directly: + +```sh +kubectl -n monitoring get pod | grep -E 'loki|alloy|grafana' +kubectl -n monitoring get pvc | grep loki +kubectl -n monitoring get servicemonitor loki alloy +``` + +Grafana should show a `Loki` datasource. Useful Explore queries: + +```logql +{namespace="code-place-dev"} +{namespace="code-place-prod", app="backend"} | json +{namespace="code-place-dev", container=~"backend|celery-worker|judge-server"} +{namespace="kube-system", app_kubernetes_io_name="traefik"} | json | DownstreamStatus >= 400 +{namespace="monitoring", app_kubernetes_io_name="kubernetes-event-exporter"} +``` + +Grafana should also show: + +- `CodePlace Overview` +- `CodePlace Logs` + +## Notes + +This is not a horizontally scalable Loki design. It is the selected cloud-free baseline for short retention. + +Keep these invariants unless the storage design changes intentionally: + +- Loki stays in `SingleBinary` deployment mode. +- Loki is installed from `grafana/loki` chart `6.55.0`. Do not upgrade the chart without revalidating rendered workloads and values compatibility. +- Loki uses filesystem storage on a Longhorn PVC. +- In the current three-node cluster, Alloy runs on every node. Loki and Tempo remain single-writer services on Longhorn PVCs, while stateless collectors, Loki gateway, and probe exporters can be replicated across nodes. +- If a node uses a custom NoSchedule taint, add the matching Alloy toleration before relying on the `AlloyDaemonSetUnavailable` alert for full-node log coverage. +- kube-prometheus-stack Prometheus and Alertmanager run with 2 replicas. Prometheus clears the replica external label so HA replicas do not create replica-labeled duplicate Alertmanager notifications. Prometheus, Alertmanager, and Grafana use Longhorn PVCs so metric data, silences, and UI state survive pod rescheduling. +- Alloy keeps namespace-based collection for `code-place-dev`, `code-place-prod`, and `monitoring`; do not depend on `app.kubernetes.io/name` for CodePlace app logs because the application manifests primarily use `app`. +- Alloy keeps `alloy.mounts.varlog=true`; otherwise the `/var/log/pods` targets are discovered but cannot be read. +- The PVC size is explicit at `50Gi`, and `LokiPVCAlmostFull` alerts at 85%. +- Retention stays short: dev 72h, prod 168h. +- The Loki chart's MinIO subchart stays disabled. If object storage becomes available, use a separately operated S3-compatible service and migrate Loki storage deliberately. + +If production log volume grows, first reduce noisy application logs or shorten retention. Increase the Longhorn PVC only after confirming available node storage and Longhorn replica health. If logs must survive a node/PVC failure domain, move prod to an internally operated S3-compatible object store such as a separately managed MinIO cluster. diff --git a/kubernetes/monitoring/logs/alloy-values.yaml b/kubernetes/monitoring/logs/alloy-values.yaml new file mode 100644 index 000000000..8903943a8 --- /dev/null +++ b/kubernetes/monitoring/logs/alloy-values.yaml @@ -0,0 +1,133 @@ +# Helm values for grafana/alloy. +# Alloy runs as a DaemonSet and collects CodePlace, monitoring, and Traefik logs. + +controller: + type: daemonset + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + +alloy: + mounts: + varlog: true + extraEnv: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + configMap: + create: true + content: | + discovery.kubernetes "pod" { + role = "pod" + + selectors { + role = "pod" + field = "spec.nodeName=" + sys.env("NODE_NAME") + } + } + + discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pod.targets + + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app", "__meta_kubernetes_pod_label_app_kubernetes_io_name"] + separator = "/" + regex = "(code-place-dev|code-place-prod|monitoring)/.*|kube-system/.*/traefik" + action = "keep" + } + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + action = "replace" + target_label = "app" + } + + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = "(.+)" + action = "replace" + target_label = "app_kubernetes_io_name" + } + + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + action = "replace" + separator = "/" + target_label = "job" + } + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "__path__" + separator = "/" + replacement = "/var/log/pods/*$1/*.log" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_id"] + action = "replace" + target_label = "container_runtime" + regex = `^(\S+):\/\/.+$` + replacement = "$1" + } + } + + loki.source.kubernetes "pod_logs" { + targets = discovery.relabel.pod_logs.output + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.static_labels { + values = { + cluster = "codeplace", + } + } + + forward_to = [loki.write.default.receiver] + } + + loki.write "default" { + endpoint { + url = "http://loki-gateway.monitoring.svc.cluster.local/loki/api/v1/push" + } + } + + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi + +serviceMonitor: + enabled: true + additionalLabels: + release: kube-prometheus-stack + interval: 30s diff --git a/kubernetes/monitoring/logs/loki-values.yaml b/kubernetes/monitoring/logs/loki-values.yaml new file mode 100644 index 000000000..e735cff69 --- /dev/null +++ b/kubernetes/monitoring/logs/loki-values.yaml @@ -0,0 +1,125 @@ +# Helm values for grafana/loki. +# This is an on-premises, cloud-free Loki deployment. It intentionally uses +# SingleBinary mode with a Longhorn-backed filesystem PVC instead of object storage. + +loki: + auth_enabled: false + commonConfig: + path_prefix: /var/loki + replication_factor: 1 + schemaConfig: + configs: + - from: "2026-06-19" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + storage: + type: filesystem + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + storage_config: + tsdb_shipper: + active_index_directory: /var/loki/tsdb-index + cache_location: /var/loki/tsdb-cache + filesystem: + directory: /var/loki/chunks + compactor: + working_directory: /var/loki/compactor + retention_enabled: true + delete_request_store: filesystem + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 168h + retention_stream: + - selector: '{namespace="code-place-dev"}' + priority: 1 + period: 72h + - selector: '{namespace="code-place-prod"}' + priority: 1 + period: 168h + ruler: + enable_api: true + +deploymentMode: SingleBinary + +singleBinary: + replicas: 1 + persistence: + enabled: true + storageClass: longhorn + accessModes: + - ReadWriteOnce + size: 50Gi + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + memory: 2Gi + +gateway: + enabled: true + replicas: 2 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/component: gateway + app.kubernetes.io/instance: loki + +lokiCanary: + enabled: true + +monitoring: + serviceMonitor: + enabled: true + labels: + release: kube-prometheus-stack + interval: 15s + scrapeTimeout: 10s + +test: + enabled: true + +chunksCache: + enabled: false + +resultsCache: + enabled: false + +minio: + enabled: false + +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomPlanner: + replicas: 0 +bloomBuilder: + replicas: 0 +bloomGateway: + replicas: 0 diff --git a/kubernetes/monitoring/longhorn-service-monitor.yaml b/kubernetes/monitoring/longhorn-service-monitor.yaml new file mode 100644 index 000000000..c667b68e4 --- /dev/null +++ b/kubernetes/monitoring/longhorn-service-monitor.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: longhorn + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - longhorn-system + selector: + matchLabels: + app: longhorn-manager + endpoints: + - port: manager + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/otel-collector.yaml b/kubernetes/monitoring/otel-collector.yaml new file mode 100644 index 000000000..116b95d5a --- /dev/null +++ b/kubernetes/monitoring/otel-collector.yaml @@ -0,0 +1,219 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-collector +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-collector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: otel-collector +subjects: + - kind: ServiceAccount + name: otel-collector + namespace: monitoring +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + processors: + memory_limiter: + check_interval: 5s + limit_mib: 384 + spike_limit_mib: 128 + k8sattributes: + auth_type: serviceAccount + extract: + metadata: + - k8s.namespace.name + - k8s.pod.name + - k8s.deployment.name + - k8s.node.name + batch: + timeout: 5s + send_batch_size: 512 + + exporters: + otlp/tempo: + endpoint: tempo.monitoring.svc.cluster.local:4317 + tls: + insecure: true + + extensions: + health_check: + endpoint: 0.0.0.0:13133 + + service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + extensions: + - health_check + pipelines: + traces: + receivers: + - otlp + processors: + - memory_limiter + - k8sattributes + - batch + exporters: + - otlp/tempo +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-collector + labels: + app: otel-collector +spec: + replicas: 2 + selector: + matchLabels: + app: otel-collector + template: + metadata: + labels: + app: otel-collector + spec: + serviceAccountName: otel-collector + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: otel-collector + containers: + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.128.0 + args: + - --config=/etc/otelcol/config.yaml + ports: + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + - name: health + containerPort: 13133 + - name: metrics + containerPort: 8888 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi + readinessProbe: + httpGet: + path: / + port: health + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + httpGet: + path: / + port: health + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + volumeMounts: + - name: config + mountPath: /etc/otelcol + volumes: + - name: config + configMap: + name: otel-collector-config +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: otel-collector +spec: + minAvailable: 1 + selector: + matchLabels: + app: otel-collector +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + labels: + app: otel-collector +spec: + selector: + app: otel-collector + ports: + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + - name: otlp-http + port: 4318 + targetPort: otlp-http + - name: health + port: 13133 + targetPort: health + - name: metrics + port: 8888 + targetPort: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: otel-collector + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: otel-collector + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 5a8c8fb7e..42b447656 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -6,18 +6,60 @@ metadata: release: kube-prometheus-stack spec: groups: + - name: codeplace.recording + interval: 15s + rules: + - record: codeplace:api_request_rate5m + expr: sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)"}[5m])) + - record: codeplace:api_5xx_rate5m + expr: sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)", status_code=~"5.."}[5m])) + - record: codeplace:api_5xx_ratio5m + expr: codeplace:api_5xx_rate5m / clamp_min(codeplace:api_request_rate5m, 0.001) + - record: codeplace:api_p95_latency_seconds5m + expr: histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~"code-place-(dev|prod)"}[5m]))) + - record: codeplace:ingress_request_rate2m + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-dev-.*"}[2m])) or sum(rate(traefik_service_requests_total{service=~"code-place-dev-.*"}[2m])) + labels: + namespace: code-place-dev + - record: codeplace:ingress_5xx_rate2m + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-dev-.*", code=~"5.."}[2m])) or sum(rate(traefik_service_requests_total{service=~"code-place-dev-.*", code=~"5.."}[2m])) + labels: + namespace: code-place-dev + - record: codeplace:ingress_request_rate2m + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-prod-.*"}[2m])) or sum(rate(traefik_service_requests_total{service=~"code-place-prod-.*"}[2m])) + labels: + namespace: code-place-prod + - record: codeplace:ingress_5xx_rate2m + expr: sum(rate(traefik_service_requests_total{exported_service=~"code-place-prod-.*", code=~"5.."}[2m])) or sum(rate(traefik_service_requests_total{service=~"code-place-prod-.*", code=~"5.."}[2m])) + labels: + namespace: code-place-prod + - record: codeplace:public_endpoint_availability5m + expr: avg by (environment, namespace, service, instance) (avg_over_time(probe_success{probe_type="public-http"}[5m])) + - record: codeplace:judge_waiting_queue_length + expr: max by (namespace) (codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"}) - name: codeplace.p0 interval: 15s rules: - alert: BackendTargetDown - expr: up{job="backend", namespace=~"code-place-(dev|prod)"} == 0 + expr: (min(up{job="backend", namespace="code-place-dev"}) == 0) or absent(up{job="backend", namespace="code-place-dev"}) + for: 1m + labels: + namespace: code-place-dev + severity: critical + priority: P0 + annotations: + summary: "Backend metrics target is down in dev" + description: "Prometheus cannot scrape the dev backend /metrics endpoint, or the backend target series is missing, for 1 minute." + - alert: BackendTargetDown + expr: (min(up{job="backend", namespace="code-place-prod"}) == 0) or absent(up{job="backend", namespace="code-place-prod"}) for: 1m labels: + namespace: code-place-prod severity: critical priority: P0 annotations: - summary: "Backend metrics target is down" - description: "Prometheus cannot scrape the backend /metrics endpoint for 1 minute." + summary: "Backend metrics target is down in prod" + description: "Prometheus cannot scrape the prod backend /metrics endpoint, or the backend target series is missing, for 1 minute." - alert: Api5xxSpike expr: sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)", status_code=~"5.."}[2m])) / clamp_min(sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)"}[2m])), 1) > 0.05 for: 2m @@ -28,7 +70,7 @@ spec: summary: "Backend API 5xx rate is high" description: "Backend API 5xx responses are above 5% for 2 minutes." - alert: Ingress5xxSpike - expr: sum(rate(traefik_service_requests_total{service=~"code-place-dev-.*", code=~"5.."}[2m])) / clamp_min(sum(rate(traefik_service_requests_total{service=~"code-place-dev-.*"}[2m])), 1) > 0.05 + expr: codeplace:ingress_5xx_rate2m{namespace="code-place-dev"} / clamp_min(codeplace:ingress_request_rate2m{namespace="code-place-dev"}, 1) > 0.05 for: 2m labels: namespace: code-place-dev @@ -38,7 +80,7 @@ spec: summary: "Ingress 5xx rate is high in dev" description: "Traefik 5xx responses for code-place-dev are above 5% for 2 minutes." - alert: Ingress5xxSpike - expr: sum(rate(traefik_service_requests_total{service=~"code-place-prod-.*", code=~"5.."}[2m])) / clamp_min(sum(rate(traefik_service_requests_total{service=~"code-place-prod-.*"}[2m])), 1) > 0.05 + expr: codeplace:ingress_5xx_rate2m{namespace="code-place-prod"} / clamp_min(codeplace:ingress_request_rate2m{namespace="code-place-prod"}, 1) > 0.05 for: 2m labels: namespace: code-place-prod @@ -47,41 +89,106 @@ spec: annotations: summary: "Ingress 5xx rate is high in prod" description: "Traefik 5xx responses for code-place-prod are above 5% for 2 minutes." - - alert: JudgeAllUnavailable - expr: (sum by (namespace) (codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) == 0) or (max by (namespace) (up{job="backend", namespace=~"code-place-(dev|prod)"}) unless on (namespace) codeplace_judge_server_available{namespace=~"code-place-(dev|prod)"}) + - alert: PublicEndpointDown + expr: (probe_success{environment="prod", probe_type="public-http"} == 0) or absent(probe_success{environment="prod", service="frontend", probe_type="public-http"}) or absent(probe_success{environment="prod", service="hub-auth", probe_type="public-http"}) for: 1m labels: severity: critical priority: P0 + namespace: code-place-prod annotations: - summary: "No judge-server is available" - description: "No enabled judge-server has a fresh heartbeat." - - alert: JudgeHeartbeatCritical - expr: max by (namespace) (codeplace_judge_server_last_heartbeat_age_seconds{namespace=~"code-place-(dev|prod)"}) > 180 + summary: "Public endpoint is down in prod" + description: "A production synthetic HTTPS probe has failed for 1 minute." + - alert: PostgresUnavailable + expr: (sum(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) + for: 1m labels: + namespace: code-place-dev severity: critical priority: P0 annotations: - summary: "Judge-server heartbeat is stale" - description: "A judge-server heartbeat is older than 180 seconds." + summary: "PostgreSQL pod is not ready in dev" + description: "No dev PostgreSQL pod has been ready, or PostgreSQL pod readiness metrics are missing, for 1 minute." - alert: PostgresUnavailable - expr: min by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"postgres-[0-9]+"}) == 0 + expr: (sum(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"postgres-[0-9]+"}) + for: 1m + labels: + namespace: code-place-prod + severity: critical + priority: P0 + annotations: + summary: "PostgreSQL pod is not ready in prod" + description: "No prod PostgreSQL pod has been ready, or PostgreSQL pod readiness metrics are missing, for 1 minute." + - alert: RedisUnavailable + expr: (sum(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) for: 1m labels: + namespace: code-place-dev severity: critical priority: P0 annotations: - summary: "PostgreSQL pod is not ready" - description: "At least one PostgreSQL pod has not been ready for 1 minute." + summary: "Redis pod is not ready in dev" + description: "No dev Redis/Sentinel pod has been ready, or Redis pod readiness metrics are missing, for 1 minute." - alert: RedisUnavailable - expr: min by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0 + expr: (sum(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-prod", condition="true", pod=~"redis-.*|redis-replication-.*"}) + for: 1m + labels: + namespace: code-place-prod + severity: critical + priority: P0 + annotations: + summary: "Redis pod is not ready in prod" + description: "No prod Redis/Sentinel pod has been ready, or Redis pod readiness metrics are missing, for 1 minute." + - alert: LokiUnavailable + expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) + for: 1m + labels: + severity: critical + priority: P0 + namespace: monitoring + annotations: + summary: "Loki is not ready" + description: "No Loki single-binary pod has been ready for 1 minute. Log ingestion and log queries may be unavailable." + - alert: LonghornVolumeFaulted + expr: longhorn_volume_robustness == 3 + for: 1m + labels: + severity: critical + priority: P0 + namespace: longhorn-system + annotations: + summary: "Longhorn volume is faulted" + description: "A Longhorn volume is faulted for 1 minute. A PVC-backed workload may lose storage availability." + - alert: LonghornVolumeReadOnly + expr: longhorn_volume_file_system_read_only == 1 + for: 1m + labels: + severity: critical + priority: P0 + namespace: longhorn-system + annotations: + summary: "Longhorn volume is read-only" + description: "A Longhorn volume filesystem is read-only for 1 minute. Writes from the attached workload may fail." + - alert: VLLMUnavailable + expr: (up{job="vllm", namespace="code-place-prod"} == 0) or absent(up{job="vllm", namespace="code-place-prod"}) + for: 2m + labels: + severity: critical + priority: P0 + namespace: code-place-prod + annotations: + summary: "vLLM metrics target is unavailable" + description: "Prometheus cannot scrape the vLLM /metrics endpoint for 2 minutes. AI hint serving may be unavailable." + - alert: GPUXIDError + expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_XID_ERRORS{namespace="monitoring"}) > 0 for: 1m labels: severity: critical priority: P0 + namespace: monitoring annotations: - summary: "Redis pod is not ready" - description: "At least one Redis pod has not been ready for 1 minute." + summary: "GPU driver reported an XID error" + description: "DCGM reports a non-zero NVIDIA XID error for 1 minute. vLLM GPU health and node driver state should be checked." - name: codeplace.p1 interval: 30s rules: @@ -94,6 +201,25 @@ spec: annotations: summary: "Backend API p95 latency is high" description: "Backend API p95 latency is above 2 seconds for 5 minutes." + - alert: PublicEndpointDown + expr: (probe_success{environment="dev", probe_type="public-http"} == 0) or absent(probe_success{environment="dev", service="frontend", probe_type="public-http"}) or absent(probe_success{environment="dev", service="hub-auth", probe_type="public-http"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: code-place-dev + annotations: + summary: "Public endpoint is down in dev" + description: "A dev synthetic HTTPS probe has failed for 2 minutes." + - alert: PublicEndpointLatencyHigh + expr: probe_duration_seconds{probe_type="public-http", namespace=~"code-place-(dev|prod)"} > 2 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Public frontend endpoint latency is high" + description: "A synthetic HTTPS probe has taken more than 2 seconds for 5 minutes." - alert: JudgeWaitingQueueBacklog expr: codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"} > 5 for: 3m @@ -103,6 +229,33 @@ spec: annotations: summary: "Judge waiting queue is backing up" description: "More than 5 submissions are waiting because no judge-server was available." + - alert: CeleryBrokerQueueBacklog + expr: codeplace_celery_broker_queue_length{namespace=~"code-place-(dev|prod)"} > 20 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Celery broker queue is backing up" + description: "More than 20 Celery tasks have been waiting in the Redis broker default queue for 5 minutes. Check celery-worker capacity, Redis broker health, and task errors." + - alert: SubmissionCreateSystemFailures + expr: sum by (namespace, scope, status) (increase(codeplace_submission_create_outcome_total{namespace=~"code-place-(dev|prod)", status=~"db_error|enqueue_error"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "Submission create path has system failures" + description: "Submission creation is reporting DB or judge enqueue failures. Check backend logs, PostgreSQL, Redis/Celery broker, and judge task queue." + - alert: JudgeTaskFailures + expr: sum by (namespace, scope, status) (increase(codeplace_judge_task_outcome_total{namespace=~"code-place-(dev|prod)", status=~"error|submission_missing|user_missing"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "Judge task failures detected" + description: "Celery judge_task is failing before or during judge dispatch. Check celery-worker logs, submission/user rows, judge-server availability, and result save path." - alert: CeleryWorkerRestarting expr: sum by (namespace) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"celery-worker-.*"}[15m])) > 3 labels: @@ -112,16 +265,16 @@ spec: summary: "Celery worker is restarting repeatedly" description: "Celery worker pods restarted more than 3 times in 15 minutes." - alert: CeleryBeatDown - expr: min by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"celery-beat-.*"}) == 0 + expr: (kube_deployment_status_replicas_available{namespace=~"code-place-(dev|prod)", deployment="celery-beat"} < 1) or (kube_deployment_spec_replicas{namespace=~"code-place-(dev|prod)", deployment="celery-beat"} unless on (namespace, deployment) kube_deployment_status_replicas_available{namespace=~"code-place-(dev|prod)", deployment="celery-beat"}) for: 2m labels: severity: warning priority: P1 annotations: summary: "Celery beat is not ready" - description: "The celery-beat pod has not been ready for 2 minutes." + description: "The celery-beat Deployment has no available replica for 2 minutes, or its availability metric is missing." - alert: PodCrashLooping - expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|celery-.*|judge-server-.*"}[5m])) > 0 + expr: sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~"code-place-(dev|prod)", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|vllm-.*"}[5m])) > 0 for: 5m labels: severity: warning @@ -129,8 +282,144 @@ spec: annotations: summary: "CodePlace pod is restarting" description: "A CodePlace application pod has restarted within the last 5 minutes." + - alert: CodePlacePodNotReady + expr: max by (namespace, pod) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}) == 0 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace pod is not ready" + description: "A CodePlace application or datastore pod has not been ready for 5 minutes." + - alert: KubernetesPodImagePullBackOff + expr: max by (namespace, pod, container, reason) (kube_pod_container_status_waiting_reason{namespace=~"code-place-(dev|prod)", reason=~"ImagePullBackOff|ErrImagePull"}) == 1 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "A Kubernetes pod cannot pull its image" + description: "A pod has been waiting with ImagePullBackOff or ErrImagePull for 2 minutes. Check image tag, registry access, and image pull secret." + - alert: KubernetesPodCrashLoopBackOff + expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{namespace=~"code-place-(dev|prod)", reason="CrashLoopBackOff"}) == 1 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "A Kubernetes pod is in CrashLoopBackOff" + description: "A pod has been waiting with CrashLoopBackOff for 5 minutes. Check the previous container logs and Kubernetes Warning events." + - alert: KubernetesPodOOMKilled + expr: max by (namespace, pod, container) (max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~"code-place-(dev|prod)", reason="OOMKilled"}[10m])) == 1 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "A Kubernetes container was OOMKilled" + description: "A container was terminated by OOMKilled within the last 10 minutes. Check memory limits, recent traffic, and heap/cache growth." + - alert: KubernetesPodUnschedulable + expr: (max by (namespace, pod) (kube_pod_status_unschedulable{namespace=~"code-place-(dev|prod)"}) == 1) or (max by (namespace, pod) (kube_pod_status_phase{namespace=~"code-place-(dev|prod)", phase="Pending"}) == 1) + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "A Kubernetes pod is pending or unschedulable" + description: "A pod has been pending or unschedulable for 10 minutes. Check node capacity, taints, affinity, PVC binding, and FailedScheduling events." + - alert: CodePlaceDeploymentUnavailable + expr: max by (namespace, deployment) (kube_deployment_status_replicas_unavailable{namespace=~"code-place-(dev|prod)", deployment=~"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm"}) > 0 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace deployment has unavailable replicas" + description: "A CodePlace deployment has had unavailable replicas for 5 minutes. Check rollout status, pod events, readiness probes, image pulls, and resource pressure." + - alert: CodePlaceServiceNoReadyEndpoints + expr: (sum by (namespace, endpoint) (kube_endpoint_address{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server", ready="true"}) < 1) or (label_replace(kube_service_info{namespace=~"code-place-(dev|prod)", service=~"backend|frontend|hub-auth|judge-server"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server", ready="true"}) + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace service has no ready endpoints" + description: "A CodePlace Service has had zero ready endpoints for 2 minutes, or the endpoint availability metric is missing while the Service exists. Check Service selector labels, Pod readiness, EndpointSlice state, and rollout events." + - alert: VLLMServiceNoReadyEndpoints + expr: (sum by (namespace, endpoint) (kube_endpoint_address{namespace="code-place-prod", endpoint="vllm", ready="true"}) < 1) or (label_replace(kube_service_info{namespace="code-place-prod", service="vllm"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address{namespace="code-place-prod", endpoint="vllm", ready="true"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "vLLM service has no ready endpoints" + description: "The prod vLLM Service has had zero ready endpoints for 2 minutes, or the endpoint availability metric is missing while the Service exists. AI hint routing may fail even if the Service object exists." + - alert: CodePlaceContainerCPUHigh + expr: sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}[5m])) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="cpu", unit="core", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 0.001) > 0.8 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace container CPU usage is high" + description: "A CodePlace container has used more than 80% of its CPU limit for 10 minutes." + - alert: CodePlaceContainerMemoryHigh + expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"code-place-(dev|prod)", container!="", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}) / clamp_min(sum by (namespace, pod, container) (kube_pod_container_resource_limits{namespace=~"code-place-(dev|prod)", resource="memory", unit="byte", pod=~"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*"}), 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "CodePlace container memory usage is high" + description: "A CodePlace container has used more than 85% of its memory limit for 10 minutes." + - alert: PostgresCollectorError + expr: max by (namespace, pod) (cnpg_collector_last_collection_error{namespace=~"code-place-(dev|prod)", cluster="postgres"}) > 0 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL metrics collection is failing" + description: "CNPG PostgreSQL metrics collection has reported errors for 5 minutes." + - alert: PostgresHADegraded + expr: min by (namespace) (cnpg_collector_nodes_used{namespace=~"code-place-(dev|prod)", cluster="postgres"}) < 3 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL HA placement is degraded" + description: "CNPG reports fewer than 3 distinct nodes for PostgreSQL instances for 10 minutes." + - alert: PostgresReplicaUnavailable + expr: sum by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"postgres-[0-9]+"}) < 3 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "PostgreSQL ready replica count is degraded" + description: "Fewer than 3 PostgreSQL pods have been ready for 5 minutes. Failover capacity or quorum may be reduced." + - alert: RedisReplicaUnavailable + expr: sum by (namespace) (kube_pod_status_ready{namespace=~"code-place-(dev|prod)", condition="true", pod=~"redis-.*|redis-replication-.*"}) < 6 + for: 5m + labels: + severity: warning + priority: P1 + annotations: + summary: "Redis/Sentinel ready replica count is degraded" + description: "Fewer than 6 Redis replication and Sentinel pods have been ready for 5 minutes. Redis failover capacity may be reduced." + - alert: RedisMemoryHigh + expr: (max by (namespace, pod) (redis_memory_used_bytes{namespace=~"code-place-(dev|prod)"}) / max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}) > 0.85) and on (namespace, pod) max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}) > 0 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "Redis memory usage is high" + description: "Redis exporter reports memory usage above 85% of configured maxmemory for 10 minutes." - alert: PVCAlmostFull - expr: kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"} / kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"} > 0.85 + expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 for: 10m labels: severity: warning @@ -138,3 +427,190 @@ spec: annotations: summary: "PVC usage is above 85%" description: "A persistent volume claim has been above 85% usage for 10 minutes." + - alert: LokiPVCAlmostFull + expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace="monitoring", persistentvolumeclaim=~".*loki.*"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace="monitoring", persistentvolumeclaim=~".*loki.*"}), 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki PVC usage is above 85%" + description: "The Loki Longhorn PVC has been above 85% usage for 10 minutes. Log retention or PVC size must be adjusted before ingestion stops." + - alert: AlloyDaemonSetUnavailable + expr: (kube_daemonset_status_number_available{namespace="monitoring", daemonset="alloy"} < kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="alloy"}) or absent(kube_daemonset_status_desired_number_scheduled{namespace="monitoring", daemonset="alloy"}) + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Alloy log collector is not running on every node" + description: "At least one node is missing an available Alloy DaemonSet pod for 5 minutes. Logs from that node may be missing." + - alert: LokiGatewayUnavailable + expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-gateway-.*"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki gateway is not ready" + description: "No Loki gateway replica has been ready for 2 minutes. Alloy writes and Grafana Loki queries may fail." + - alert: LokiRequestErrors + expr: sum(rate(loki_request_duration_seconds_count{namespace="monitoring", status_code=~"5.."}[5m])) > 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Loki is returning 5xx responses" + description: "Loki has returned 5xx responses for 5 minutes. Log writes or Grafana log queries may be failing." + - alert: LonghornVolumeDegraded + expr: longhorn_volume_robustness == 2 + for: 5m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn volume is degraded" + description: "A Longhorn volume has been degraded for 5 minutes. Replica rebuild, node health, and disk pressure should be checked." + - alert: LonghornNodeNotReady + expr: longhorn_node_status{condition="ready"} == 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn node is not ready" + description: "A Longhorn node has not been ready for 5 minutes." + - alert: LonghornDiskNotReady + expr: longhorn_disk_status{condition="ready"} == 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn disk is not ready" + description: "A Longhorn disk has not been ready for 5 minutes." + - alert: LonghornNodeStorageHigh + expr: (longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn node storage usage is high" + description: "A Longhorn node has used or reserved more than 85% of available storage for 10 minutes." + - alert: LonghornDiskUsageHigh + expr: (longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1) > 0.85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: longhorn-system + annotations: + summary: "Longhorn disk usage is high" + description: "A Longhorn disk has used or reserved more than 85% of capacity for 10 minutes." + - alert: VLLMWaitingQueueHigh + expr: max({__name__="vllm:num_requests_waiting", namespace="code-place-prod"}) > 5 + for: 5m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "vLLM request waiting queue is high" + description: "More than 5 vLLM requests have been waiting for 5 minutes. AI hint latency may be elevated." + - alert: VLLMKVCacheHigh + expr: max({__name__="vllm:kv_cache_usage_perc", namespace="code-place-prod"}) > 0.9 + for: 10m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "vLLM KV cache usage is high" + description: "vLLM KV cache usage has been above 90% for 10 minutes." + - alert: VLLMRequestLatencyHigh + expr: histogram_quantile(0.95, sum by (le) (rate({__name__="vllm:e2e_request_latency_seconds_bucket", namespace="code-place-prod"}[10m]))) > 60 + for: 10m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "vLLM request latency is high" + description: "vLLM p95 end-to-end request latency has been above 60 seconds for 10 minutes." + - alert: AIHintBackendErrors + expr: sum by (namespace, status) (increase(codeplace_ai_hint_requests_total{namespace=~"code-place-(dev|prod)", status=~"request_error|stream_parse_error|error"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "Backend AI hint stream failures detected" + description: "Backend AI hint streaming has reported request, parse, or internal errors in the last 5 minutes. Check backend logs, vLLM Service endpoint, vLLM metrics, and request_id-linked traces." + - alert: AIHintBackendLatencyHigh + expr: histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=~"code-place-(dev|prod)", status="success"}[10m]))) > 120 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "Backend AI hint stream latency is high" + description: "Backend AI hint successful stream p95 duration has been above 120 seconds for 10 minutes. Check vLLM queue, GPU utilization, and client stream behavior." + - alert: AIHintAPIFailures + expr: sum by (namespace, scope, status) (increase(codeplace_ai_hint_api_outcome_total{namespace=~"code-place-(dev|prod)", status=~"llm_error|unexpected_error|empty_response"}[5m])) > 0 + for: 2m + labels: + severity: warning + priority: P1 + annotations: + summary: "User-facing AI hint API failures detected" + description: "AI hint API returned LLM errors, unexpected errors, or empty responses in the last 5 minutes. Compare API outcome status with backend stream and vLLM panels." + - alert: GPUUtilizationHigh + expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"}) > 95 + for: 15m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "GPU utilization is high" + description: "DCGM reports GPU utilization above 95% for 15 minutes. vLLM AI hint latency may rise under sustained load." + - alert: GPUMemoryHigh + expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_FB_USED{namespace="monitoring"} / clamp_min(DCGM_FI_DEV_FB_USED{namespace="monitoring"} + DCGM_FI_DEV_FB_FREE{namespace="monitoring"}, 1)) > 0.9 + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "GPU framebuffer memory usage is high" + description: "DCGM reports GPU framebuffer memory usage above 90% for 10 minutes. vLLM KV cache and max sequence settings should be checked." + - alert: GPUTemperatureHigh + expr: max by (Hostname, gpu, UUID) (DCGM_FI_DEV_GPU_TEMP{namespace="monitoring"}) > 85 + for: 10m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "GPU temperature is high" + description: "DCGM reports GPU temperature above 85C for 10 minutes. Node cooling and sustained vLLM load should be checked." + - alert: CodePlaceNodePressure + expr: kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure", status="true"} == 1 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Kubernetes node is under pressure" + description: "A Kubernetes node has reported memory, disk, or PID pressure for 5 minutes." diff --git a/kubernetes/monitoring/public-endpoint-probes.yaml b/kubernetes/monitoring/public-endpoint-probes.yaml new file mode 100644 index 000000000..d46405b3c --- /dev/null +++ b/kubernetes/monitoring/public-endpoint-probes.yaml @@ -0,0 +1,114 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-public-dev-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-public-http-dev + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://k3s.code-place-dev.site/ + labels: + environment: dev + namespace: code-place-dev + service: frontend + probe_type: public-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-public-prod-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-public-http-prod + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://code.pusan.ac.kr/ + labels: + environment: prod + namespace: code-place-prod + service: frontend + probe_type: public-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-grafana-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-grafana-http + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://monitoring.code-place-dev.site/ + labels: + environment: monitoring + namespace: monitoring + service: grafana + probe_type: public-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-hub-auth-dev-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-hub-auth-http-dev + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://hub-auth-dev.code-place-dev.site/ + labels: + environment: dev + namespace: code-place-dev + service: hub-auth + probe_type: public-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: codeplace-hub-auth-prod-http + labels: + release: kube-prometheus-stack +spec: + jobName: codeplace-hub-auth-http-prod + interval: 30s + scrapeTimeout: 15s + module: http_2xx + prober: + url: blackbox-exporter.monitoring.svc.cluster.local:9115 + targets: + staticConfig: + static: + - https://hub-auth.code-place-dev.site/ + labels: + environment: prod + namespace: code-place-prod + service: hub-auth + probe_type: public-http diff --git a/kubernetes/monitoring/secrets/README.md b/kubernetes/monitoring/secrets/README.md new file mode 100644 index 000000000..6d81f9492 --- /dev/null +++ b/kubernetes/monitoring/secrets/README.md @@ -0,0 +1,23 @@ +# Monitoring Secrets + +This directory is reserved for SealedSecret manifests generated per cluster. + +Do not commit plaintext Kubernetes Secret manifests or raw Discord webhook URLs. +The AlertmanagerConfig expects the decrypted Secret below to exist in the +`monitoring` namespace: + +- Secret: `alertmanager-contact-points` +- Key: `webhook-url` + +Generate the SealedSecret with the target cluster public key: + +```sh +kubectl -n monitoring create secret generic alertmanager-contact-points \ + --from-literal=webhook-url="$ALERT_WEBHOOK_URL" \ + --dry-run=client -o yaml \ + | kubeseal --controller-namespace kube-system --format yaml \ + > kubernetes/monitoring/secrets/alertmanager-contact-points.sealedsecret.yaml +``` + +If the SealedSecrets controller runs outside `kube-system`, change +`--controller-namespace` to the actual namespace. diff --git a/kubernetes/monitoring/tempo.yaml b/kubernetes/monitoring/tempo.yaml new file mode 100644 index 000000000..a8b0c3c66 --- /dev/null +++ b/kubernetes/monitoring/tempo.yaml @@ -0,0 +1,144 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo-config +data: + tempo.yaml: | + server: + http_listen_port: 3200 + + distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + ingester: + max_block_duration: 5m + + compactor: + compaction: + block_retention: 72h + + storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tempo-data +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 20Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tempo + labels: + app: tempo +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: tempo + template: + metadata: + labels: + app: tempo + spec: + containers: + - name: tempo + image: grafana/tempo:2.8.1 + args: + - -config.file=/etc/tempo/tempo.yaml + ports: + - name: http + containerPort: 3200 + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + readinessProbe: + httpGet: + path: /ready + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + volumeMounts: + - name: config + mountPath: /etc/tempo + - name: data + mountPath: /var/tempo + volumes: + - name: config + configMap: + name: tempo-config + - name: data + persistentVolumeClaim: + claimName: tempo-data +--- +apiVersion: v1 +kind: Service +metadata: + name: tempo + labels: + app: tempo +spec: + selector: + app: tempo + ports: + - name: http + port: 3200 + targetPort: http + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + - name: otlp-http + port: 4318 + targetPort: otlp-http +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: tempo + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: tempo + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/traefik-service-monitor.yaml b/kubernetes/monitoring/traefik-service-monitor.yaml new file mode 100644 index 000000000..8a43b4bfd --- /dev/null +++ b/kubernetes/monitoring/traefik-service-monitor.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: traefik + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - kube-system + selector: + matchLabels: + app.kubernetes.io/name: traefik + endpoints: + - port: metrics + path: /metrics + interval: 15s + scrapeTimeout: 10s diff --git a/kubernetes/monitoring/vllm-service-monitor.yaml b/kubernetes/monitoring/vllm-service-monitor.yaml new file mode 100644 index 000000000..034c580ba --- /dev/null +++ b/kubernetes/monitoring/vllm-service-monitor.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: vllm + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - code-place-prod + selector: + matchLabels: + app: vllm + jobLabel: app + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index 29a244090..78acaaa2d 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -17,3 +17,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: OTEL_ENABLED + value: "1" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index b46399215..8bbe26979 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -20,3 +20,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: OTEL_ENABLED + value: "1" diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index 6a9b0ae2d..82089303d 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -23,3 +23,5 @@ spec: key: SENTRY_DSN - name: SENTRY_ENVIRONMENT value: "dev" + - name: OTEL_ENABLED + value: "1" diff --git a/kubernetes/setup/longhorn/README.md b/kubernetes/setup/longhorn/README.md index d6dd52878..c652952a7 100644 --- a/kubernetes/setup/longhorn/README.md +++ b/kubernetes/setup/longhorn/README.md @@ -88,3 +88,22 @@ kubectl get pods -n longhorn-system -w 물리 노드가 3대 미만인 경우(예: 2대), 복제본을 배치할 공간이 부족하여 볼륨 상태가 Degraded로 표시될 수 있습니다. 이 경우 Longhorn UI(Settings > General > Default Replica Count)에서 기본 복제본 수를 2로 변경해야 합니다. + +### 5. Monitoring + +kube-prometheus-stack이 설치된 클러스터에서는 CodePlace monitoring kustomization이 Longhorn manager metric을 scrape합니다. + +```sh +kubectl apply -k kubernetes/monitoring +kubectl -n monitoring get servicemonitor longhorn +``` + +Prometheus target에서 `longhorn` scrape가 healthy인지 확인하고, Grafana의 `CodePlace Storage` dashboard에서 다음 상태를 확인합니다. + +- Longhorn manager ready count. +- faulted/degraded/read-only volume count. +- volume robustness와 actual/capacity size. +- Longhorn node/disk 사용률. +- Longhorn manager CPU/memory. + +Longhorn alert는 CodePlace alert rule에 포함됩니다. volume `faulted` 또는 `read-only`는 P0, volume `degraded`와 node/disk readiness 또는 usage 문제는 P1로 처리합니다. diff --git a/kubernetes/setup/traefik/traefik-config.yaml b/kubernetes/setup/traefik/traefik-config.yaml index 0278c1dc8..393932210 100644 --- a/kubernetes/setup/traefik/traefik-config.yaml +++ b/kubernetes/setup/traefik/traefik-config.yaml @@ -9,6 +9,20 @@ spec: - "--certificatesresolvers.default.acme.email=" - "--certificatesresolvers.default.acme.storage=/data/acme.json" - "--certificatesresolvers.default.acme.tlschallenge=true" # 443 포트를 통해 인증 + - "--accesslog=true" + - "--accesslog.format=json" + - "--accesslog.fields.defaultmode=keep" + - "--accesslog.fields.headers.defaultmode=drop" + - "--accesslog.fields.headers.names.X-Request-ID=keep" + - "--accesslog.fields.headers.names.X-Forwarded-For=keep" + + metrics: + prometheus: + addEntryPointsLabels: true + addRoutersLabels: true + addServicesLabels: true + service: + enabled: true service: enabled: true