From 191dd662953097c26fd749b0ba6708bf3c0dbc62 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 14 Oct 2026 11:37:35 +0300 Subject: [PATCH 01/25] Create remove-before-merge.txt --- remove-before-merge.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 remove-before-merge.txt diff --git a/remove-before-merge.txt b/remove-before-merge.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/remove-before-merge.txt @@ -0,0 +1 @@ + From de01e2ddaceb9d02811b2152a0cf20cffa2c4fc7 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Tue, 20 Oct 2026 10:14:04 +0300 Subject: [PATCH 02/25] Add debug profile wothout warmup or rampup --- projects/rhaiis/orchestration/config.d/workloads.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 1ddebc0cd..fa495815e 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -54,6 +54,12 @@ profile7: rampup: 35 warmup: 75 +# Debug profile matching llm-d concurrent-1k-1k without ramp-up or warmup. +debug-guidellm-1k-1k: + data: "prompt_tokens=1000,output_tokens=1000" + rates: [300, 200, 100, 50, 1] + max_seconds: 600 + # Dashboard override stub (set data, rates, max_seconds, rampup, and warmup at runtime) custom: data: "prompt_tokens=1000,output_tokens=1000" From 7886a132c777f2a671f72b15726b5294d14717d8 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 21 Oct 2026 08:56:24 +0300 Subject: [PATCH 03/25] debug profile1 with reverse order --- projects/rhaiis/orchestration/config.d/workloads.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index fa495815e..d960144e1 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -10,6 +10,14 @@ profile1: rampup: 35 warmup: 75 +# Debug profile matching profile1 with the rates in reverse order. +debug-profile1-reverse: + data: "prompt_tokens=1000,output_tokens=1000" + rates: [300,200,100,50,1] + max_seconds: 275 + rampup: 35 + warmup: 75 + profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" rates: [1,50,100,200,300] From 57d2a0c14cb428b4e555689757b6df3a70c970db Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 21 Oct 2026 14:40:26 +0300 Subject: [PATCH 04/25] Use RHAIIS-aligned benchconf branch for llm-d --- projects/llm_d/orchestration/config.d/benchconf.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/benchconf.yaml b/projects/llm_d/orchestration/config.d/benchconf.yaml index 5d6057f73..831f84eea 100644 --- a/projects/llm_d/orchestration/config.d/benchconf.yaml +++ b/projects/llm_d/orchestration/config.d/benchconf.yaml @@ -1,5 +1,5 @@ enabled: true custom_version: - enabled: false - repo: git+https://github.com/openshift-psap/benchconf - version: main + enabled: true + repo: git+https://github.com/tosokin/benchconf + version: toskin-llm-d-profiles-align-with-rhaiis From 1a3176f59345ce9c8a9de6cf9a26f88379cbe3b2 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 21 Oct 2026 18:02:36 +0300 Subject: [PATCH 05/25] fix typo in branch name --- projects/llm_d/orchestration/config.d/benchconf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/benchconf.yaml b/projects/llm_d/orchestration/config.d/benchconf.yaml index 831f84eea..b3fe394e0 100644 --- a/projects/llm_d/orchestration/config.d/benchconf.yaml +++ b/projects/llm_d/orchestration/config.d/benchconf.yaml @@ -2,4 +2,4 @@ enabled: true custom_version: enabled: true repo: git+https://github.com/tosokin/benchconf - version: toskin-llm-d-profiles-align-with-rhaiis + version: tosokin-llm-d-profiles-align-with-rhaiis From dd7874d731ce9c6f2b8c1ec41367f1d2782e6e0d Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 21 Oct 2026 19:46:53 +0300 Subject: [PATCH 06/25] fix another typo --- projects/llm_d/orchestration/config.d/benchconf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/benchconf.yaml b/projects/llm_d/orchestration/config.d/benchconf.yaml index b3fe394e0..b4478e1aa 100644 --- a/projects/llm_d/orchestration/config.d/benchconf.yaml +++ b/projects/llm_d/orchestration/config.d/benchconf.yaml @@ -1,5 +1,5 @@ enabled: true custom_version: enabled: true - repo: git+https://github.com/tosokin/benchconf + repo: git+https://github.com/openshift-psap/benchconf version: tosokin-llm-d-profiles-align-with-rhaiis From 4a5c908d24d6bed01eb078e3465e9fbd19cd729a Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Thu, 22 Oct 2026 08:59:11 +0300 Subject: [PATCH 07/25] Add Prometheus capture to RHAIIS tests temporarily --- projects/kserve/metrics/vllm_latency.yaml | 44 ++++++++--------- projects/kserve/metrics/vllm_throughput.yaml | 26 +++++----- .../llm_d/orchestration/config.d/prom.yaml | 2 +- .../rhaiis/orchestration/config.d/prom.yaml | 41 ++++++++++++++++ projects/rhaiis/orchestration/test_phase.py | 47 +++++++++++++++++++ 5 files changed, 124 insertions(+), 36 deletions(-) create mode 100644 projects/rhaiis/orchestration/config.d/prom.yaml diff --git a/projects/kserve/metrics/vllm_latency.yaml b/projects/kserve/metrics/vllm_latency.yaml index 454a5832a..60f53f3a0 100644 --- a/projects/kserve/metrics/vllm_latency.yaml +++ b/projects/kserve/metrics/vllm_latency.yaml @@ -1,129 +1,129 @@ # params: # LLMISVC_NAMESPACE: Target namespace -# LLMISVC_NAME: Name of the vLLM LLMInferenceService +# POD_NAME: Pod name regex for the vLLM metrics capture - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: ttft_p50_seconds description: Ttft p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: ttft_p90_seconds description: Ttft p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: ttft_p99_seconds description: Ttft p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: tpot_p50_seconds description: Tpot p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: tpot_p90_seconds description: Tpot p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: tpot_p99_seconds description: Tpot p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: e2e_request_latency_p50_seconds description: E2e request latency p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: e2e_request_latency_p90_seconds description: E2e request latency p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: e2e_request_latency_p99_seconds description: E2e request latency p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: prefill_time_p50_seconds description: Prefill time p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: prefill_time_p90_seconds description: Prefill time p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: prefill_time_p99_seconds description: Prefill time p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: decode_time_p50_seconds description: Decode time p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: decode_time_p90_seconds description: Decode time p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: decode_time_p99_seconds description: Decode time p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: inference_time_p50_seconds description: Inference time p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: inference_time_p90_seconds description: Inference time p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: inference_time_p99_seconds description: Inference time p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: queue_time_p50_seconds description: Queue time p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: queue_time_p90_seconds description: Queue time p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) metricName: queue_time_p99_seconds description: Queue time p99 seconds unit: seconds diff --git a/projects/kserve/metrics/vllm_throughput.yaml b/projects/kserve/metrics/vllm_throughput.yaml index 0cc3b6922..0bfaa5a40 100644 --- a/projects/kserve/metrics/vllm_throughput.yaml +++ b/projects/kserve/metrics/vllm_throughput.yaml @@ -1,75 +1,75 @@ # params: # LLMISVC_NAMESPACE: Target namespace -# LLMISVC_NAME: Name of the vLLM LLMInferenceService +# POD_NAME: Pod name regex for the vLLM metrics capture - query: >- - rate({__name__=~"kserve_vllm:generation_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m]) + rate({__name__=~"kserve_vllm:generation_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m]) metricName: generation_token_rate_per_second description: Generation token rate per second unit: ratio - query: >- - sum(rate({__name__=~"kserve_vllm:generation_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) + sum(rate({__name__=~"kserve_vllm:generation_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) metricName: generation_token_rate_sum_per_second description: Generation token rate sum per second unit: ratio - query: >- - kserve_vllm:generation_tokens_total{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} + kserve_vllm:generation_tokens_total{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} metricName: generation_tokens_total description: Generation tokens total unit: ratio - query: >- - rate({__name__=~"kserve_vllm:prompt_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m]) + rate({__name__=~"kserve_vllm:prompt_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m]) metricName: prompt_token_rate_per_second description: Prompt token rate per second unit: ops/s - query: >- - kserve_vllm:prompt_tokens_total{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} + kserve_vllm:prompt_tokens_total{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} metricName: prompt_tokens_total description: Prompt tokens total unit: count - query: >- - sum by (pod, source) (rate({__name__=~"kserve_vllm:prompt_tokens_by_source_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*", source=~"local_compute|local_cache_hit|external_kv_transfer"}[5m])) + sum by (pod, source) (rate({__name__=~"kserve_vllm:prompt_tokens_by_source_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}", source=~"local_compute|local_cache_hit|external_kv_transfer"}[5m])) metricName: prompt_tokens_rate_by_source description: Prompt tokens rate by source unit: ops/s - query: >- - kserve_vllm:num_requests_running{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} + kserve_vllm:num_requests_running{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} metricName: num_requests_running description: Num requests running unit: count - query: >- - kserve_vllm:num_requests_waiting{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} + kserve_vllm:num_requests_waiting{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} metricName: num_requests_waiting description: Num requests waiting unit: count - query: >- - max by (pod, state) (label_replace({__name__=~"kserve_vllm:num_requests_running", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}, "state", "running", "__name__", ".*") or label_replace({__name__=~"kserve_vllm:num_requests_waiting", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}, "state", "waiting", "__name__", ".*")) + max by (pod, state) (label_replace({__name__=~"kserve_vllm:num_requests_running", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}, "state", "running", "__name__", ".*") or label_replace({__name__=~"kserve_vllm:num_requests_waiting", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}, "state", "waiting", "__name__", ".*")) metricName: requests_running_waiting description: Requests running waiting unit: count - query: >- - max by (pod, reason) ({__name__=~"kserve_vllm:num_requests_waiting_by_reason", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} or label_replace({__name__=~"kserve_vllm:num_requests_waiting", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}, "reason", "unspecified", "__name__", ".*")) + max by (pod, reason) ({__name__=~"kserve_vllm:num_requests_waiting_by_reason", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} or label_replace({__name__=~"kserve_vllm:num_requests_waiting", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}, "reason", "unspecified", "__name__", ".*")) metricName: requests_waiting_by_reason description: Requests waiting by reason unit: count - query: >- - rate({__name__=~"kserve_vllm:num_preemptions_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m]) + rate({__name__=~"kserve_vllm:num_preemptions_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m]) metricName: num_preemptions_rate description: Num preemptions rate unit: ops/s - query: >- - sum by (pod) (rate({__name__=~"kserve_vllm:request_success_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) + sum by (pod) (rate({__name__=~"kserve_vllm:request_success_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) metricName: request_success_rate_by_pod description: Request success rate by pod unit: ops/s diff --git a/projects/llm_d/orchestration/config.d/prom.yaml b/projects/llm_d/orchestration/config.d/prom.yaml index 535ec0e47..66865cfaa 100644 --- a/projects/llm_d/orchestration/config.d/prom.yaml +++ b/projects/llm_d/orchestration/config.d/prom.yaml @@ -32,7 +32,7 @@ capture: files: [vllm_latency, vllm_throughput] variables: LLMISVC_NAMESPACE: "{llmisvc_namespace}" - LLMISVC_NAME: "{llmisvc_name}" + POD_NAME: "{llmisvc_name}-kserve-.*" scheduler: enabled: true files: [scheduler] diff --git a/projects/rhaiis/orchestration/config.d/prom.yaml b/projects/rhaiis/orchestration/config.d/prom.yaml new file mode 100644 index 000000000..5af6b0275 --- /dev/null +++ b/projects/rhaiis/orchestration/config.d/prom.yaml @@ -0,0 +1,41 @@ +prepare: + user_workload: + during_prepare: false + during_test: true + +capture: + enabled: true + db: + enabled: true + system_metrics: + enabled: true + user_workload: + enabled: true + fail_if_not_enabled: false + metrics: + enabled: true + include_dirs: + - projects/cluster/metrics + - projects/kserve/metrics + params: + namespace: "@rhaiis.namespace" + pod_name: set_at_runtime + groups: + resources: + enabled: true + files: [resource_cpu, resource_memory, resource_network, workload] + variables: + NAMESPACE: "{namespace}" + POD_NAME: "{pod_name}" + vllm: + enabled: true + files: [vllm_latency, vllm_throughput] + variables: + LLMISVC_NAMESPACE: "{namespace}" + POD_NAME: "{pod_name}" + gpu: + enabled: true + files: [gpu] + variables: + NAMESPACE: "{namespace}" + POD_NAME: "{pod_name}" diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 5f048f6da..681852fd3 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -4,10 +4,16 @@ import logging import os import uuid as _uuid_mod +from datetime import UTC, datetime from pathlib import Path import yaml +from projects.cluster.library.prom.collection import ( + capture_prometheus, + prepare_user_workload_monitoring, + validate_user_workload_monitoring, +) from projects.core.library import env from projects.core.library.fournos_status import patch_fjob_relevant_deployments from projects.core.library.postprocess import create_test_metadata, run_and_postprocess @@ -191,6 +197,7 @@ def _run_test( skipped_optional_phase_workloads, ) run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + benchmark_start: datetime | None = None # Standalone analysis only — no deployment needed if not run_benchmark and not profiler_enabled: @@ -214,6 +221,9 @@ def _run_test( wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) try: + validate_user_workload_monitoring() + prepare_user_workload_monitoring(during="test") + isvc_labels = { "opendatahub.io/dashboard": "true", "deployment_uuid": run_uuid, @@ -337,6 +347,8 @@ def _run_test( # Phase 2: benchmark + post-processing for ALL workloads trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None + if run_benchmark: + benchmark_start = datetime.now(UTC) for wl_key in workload_keys: _run_workload_benchmark( model_key=model_key, @@ -379,6 +391,13 @@ def _run_test( except Exception: logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) finally: + if benchmark_start is not None: + _capture_prometheus_metrics( + deployment_name=deployment_name, + namespace=namespace, + start_time=benchmark_start, + end_time=datetime.now(UTC), + ) _capture_and_cleanup(deployment_name, namespace) if fjob_name: _update_fjob_inference_reference(fjob_name, fjob_ns, None) @@ -399,6 +418,34 @@ def _run_test( return 0 +def _capture_prometheus_metrics( + *, + deployment_name: str, + namespace: str, + start_time: datetime, + end_time: datetime, +) -> None: + """Capture Prometheus data for the RHAIIS benchmark window before cleanup.""" + logger.info( + "Capturing Prometheus metrics for RHAIIS benchmark window: %s - %s", + start_time.isoformat(), + end_time.isoformat(), + ) + try: + with env.NextArtifactDir("prometheus_metrics"): + capture_prometheus( + start_time, + end_time, + runtime_params={ + "namespace": namespace, + "pod_name": f"{deployment_name}-predictor-.*", + }, + ) + except Exception: + logger.warning("Prometheus metrics capture failed; continuing cleanup", exc_info=True) + _warnings.append("Prometheus metrics capture failed") + + def _run_workload_benchmark( *, model_key: str, From a921923ad2cbd5fa1f34b606496be8c5b30d9b33 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Thu, 22 Oct 2026 15:44:53 +0300 Subject: [PATCH 08/25] Revert "Add Prometheus capture to RHAIIS tests temporarily" This reverts commit 4a5c908d24d6bed01eb078e3465e9fbd19cd729a. --- projects/kserve/metrics/vllm_latency.yaml | 44 ++++++++--------- projects/kserve/metrics/vllm_throughput.yaml | 26 +++++----- .../llm_d/orchestration/config.d/prom.yaml | 2 +- .../rhaiis/orchestration/config.d/prom.yaml | 41 ---------------- projects/rhaiis/orchestration/test_phase.py | 47 ------------------- 5 files changed, 36 insertions(+), 124 deletions(-) delete mode 100644 projects/rhaiis/orchestration/config.d/prom.yaml diff --git a/projects/kserve/metrics/vllm_latency.yaml b/projects/kserve/metrics/vllm_latency.yaml index 60f53f3a0..454a5832a 100644 --- a/projects/kserve/metrics/vllm_latency.yaml +++ b/projects/kserve/metrics/vllm_latency.yaml @@ -1,129 +1,129 @@ # params: # LLMISVC_NAMESPACE: Target namespace -# POD_NAME: Pod name regex for the vLLM metrics capture +# LLMISVC_NAME: Name of the vLLM LLMInferenceService - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: ttft_p50_seconds description: Ttft p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: ttft_p90_seconds description: Ttft p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:time_to_first_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: ttft_p99_seconds description: Ttft p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: tpot_p50_seconds description: Tpot p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: tpot_p90_seconds description: Tpot p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_time_per_output_token_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: tpot_p99_seconds description: Tpot p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: e2e_request_latency_p50_seconds description: E2e request latency p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: e2e_request_latency_p90_seconds description: E2e request latency p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:e2e_request_latency_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: e2e_request_latency_p99_seconds description: E2e request latency p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: prefill_time_p50_seconds description: Prefill time p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: prefill_time_p90_seconds description: Prefill time p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_prefill_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: prefill_time_p99_seconds description: Prefill time p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: decode_time_p50_seconds description: Decode time p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: decode_time_p90_seconds description: Decode time p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_decode_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: decode_time_p99_seconds description: Decode time p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: inference_time_p50_seconds description: Inference time p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: inference_time_p90_seconds description: Inference time p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_inference_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: inference_time_p99_seconds description: Inference time p99 seconds unit: seconds - query: >- - histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.5, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: queue_time_p50_seconds description: Queue time p50 seconds unit: seconds - query: >- - histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.9, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: queue_time_p90_seconds description: Queue time p90 seconds unit: seconds - query: >- - histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) by (le)) + histogram_quantile(0.99, sum(rate({__name__=~"kserve_vllm:request_queue_time_seconds_bucket", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) by (le)) metricName: queue_time_p99_seconds description: Queue time p99 seconds unit: seconds diff --git a/projects/kserve/metrics/vllm_throughput.yaml b/projects/kserve/metrics/vllm_throughput.yaml index 0bfaa5a40..0cc3b6922 100644 --- a/projects/kserve/metrics/vllm_throughput.yaml +++ b/projects/kserve/metrics/vllm_throughput.yaml @@ -1,75 +1,75 @@ # params: # LLMISVC_NAMESPACE: Target namespace -# POD_NAME: Pod name regex for the vLLM metrics capture +# LLMISVC_NAME: Name of the vLLM LLMInferenceService - query: >- - rate({__name__=~"kserve_vllm:generation_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m]) + rate({__name__=~"kserve_vllm:generation_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m]) metricName: generation_token_rate_per_second description: Generation token rate per second unit: ratio - query: >- - sum(rate({__name__=~"kserve_vllm:generation_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) + sum(rate({__name__=~"kserve_vllm:generation_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) metricName: generation_token_rate_sum_per_second description: Generation token rate sum per second unit: ratio - query: >- - kserve_vllm:generation_tokens_total{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} + kserve_vllm:generation_tokens_total{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} metricName: generation_tokens_total description: Generation tokens total unit: ratio - query: >- - rate({__name__=~"kserve_vllm:prompt_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m]) + rate({__name__=~"kserve_vllm:prompt_tokens_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m]) metricName: prompt_token_rate_per_second description: Prompt token rate per second unit: ops/s - query: >- - kserve_vllm:prompt_tokens_total{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} + kserve_vllm:prompt_tokens_total{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} metricName: prompt_tokens_total description: Prompt tokens total unit: count - query: >- - sum by (pod, source) (rate({__name__=~"kserve_vllm:prompt_tokens_by_source_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}", source=~"local_compute|local_cache_hit|external_kv_transfer"}[5m])) + sum by (pod, source) (rate({__name__=~"kserve_vllm:prompt_tokens_by_source_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*", source=~"local_compute|local_cache_hit|external_kv_transfer"}[5m])) metricName: prompt_tokens_rate_by_source description: Prompt tokens rate by source unit: ops/s - query: >- - kserve_vllm:num_requests_running{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} + kserve_vllm:num_requests_running{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} metricName: num_requests_running description: Num requests running unit: count - query: >- - kserve_vllm:num_requests_waiting{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} + kserve_vllm:num_requests_waiting{namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} metricName: num_requests_waiting description: Num requests waiting unit: count - query: >- - max by (pod, state) (label_replace({__name__=~"kserve_vllm:num_requests_running", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}, "state", "running", "__name__", ".*") or label_replace({__name__=~"kserve_vllm:num_requests_waiting", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}, "state", "waiting", "__name__", ".*")) + max by (pod, state) (label_replace({__name__=~"kserve_vllm:num_requests_running", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}, "state", "running", "__name__", ".*") or label_replace({__name__=~"kserve_vllm:num_requests_waiting", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}, "state", "waiting", "__name__", ".*")) metricName: requests_running_waiting description: Requests running waiting unit: count - query: >- - max by (pod, reason) ({__name__=~"kserve_vllm:num_requests_waiting_by_reason", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"} or label_replace({__name__=~"kserve_vllm:num_requests_waiting", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}, "reason", "unspecified", "__name__", ".*")) + max by (pod, reason) ({__name__=~"kserve_vllm:num_requests_waiting_by_reason", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"} or label_replace({__name__=~"kserve_vllm:num_requests_waiting", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}, "reason", "unspecified", "__name__", ".*")) metricName: requests_waiting_by_reason description: Requests waiting by reason unit: count - query: >- - rate({__name__=~"kserve_vllm:num_preemptions_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m]) + rate({__name__=~"kserve_vllm:num_preemptions_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m]) metricName: num_preemptions_rate description: Num preemptions rate unit: ops/s - query: >- - sum by (pod) (rate({__name__=~"kserve_vllm:request_success_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.POD_NAME}}"}[5m])) + sum by (pod) (rate({__name__=~"kserve_vllm:request_success_total", namespace="{{.LLMISVC_NAMESPACE}}", pod=~"{{.LLMISVC_NAME}}-kserve-.*"}[5m])) metricName: request_success_rate_by_pod description: Request success rate by pod unit: ops/s diff --git a/projects/llm_d/orchestration/config.d/prom.yaml b/projects/llm_d/orchestration/config.d/prom.yaml index 66865cfaa..535ec0e47 100644 --- a/projects/llm_d/orchestration/config.d/prom.yaml +++ b/projects/llm_d/orchestration/config.d/prom.yaml @@ -32,7 +32,7 @@ capture: files: [vllm_latency, vllm_throughput] variables: LLMISVC_NAMESPACE: "{llmisvc_namespace}" - POD_NAME: "{llmisvc_name}-kserve-.*" + LLMISVC_NAME: "{llmisvc_name}" scheduler: enabled: true files: [scheduler] diff --git a/projects/rhaiis/orchestration/config.d/prom.yaml b/projects/rhaiis/orchestration/config.d/prom.yaml deleted file mode 100644 index 5af6b0275..000000000 --- a/projects/rhaiis/orchestration/config.d/prom.yaml +++ /dev/null @@ -1,41 +0,0 @@ -prepare: - user_workload: - during_prepare: false - during_test: true - -capture: - enabled: true - db: - enabled: true - system_metrics: - enabled: true - user_workload: - enabled: true - fail_if_not_enabled: false - metrics: - enabled: true - include_dirs: - - projects/cluster/metrics - - projects/kserve/metrics - params: - namespace: "@rhaiis.namespace" - pod_name: set_at_runtime - groups: - resources: - enabled: true - files: [resource_cpu, resource_memory, resource_network, workload] - variables: - NAMESPACE: "{namespace}" - POD_NAME: "{pod_name}" - vllm: - enabled: true - files: [vllm_latency, vllm_throughput] - variables: - LLMISVC_NAMESPACE: "{namespace}" - POD_NAME: "{pod_name}" - gpu: - enabled: true - files: [gpu] - variables: - NAMESPACE: "{namespace}" - POD_NAME: "{pod_name}" diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 681852fd3..5f048f6da 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -4,16 +4,10 @@ import logging import os import uuid as _uuid_mod -from datetime import UTC, datetime from pathlib import Path import yaml -from projects.cluster.library.prom.collection import ( - capture_prometheus, - prepare_user_workload_monitoring, - validate_user_workload_monitoring, -) from projects.core.library import env from projects.core.library.fournos_status import patch_fjob_relevant_deployments from projects.core.library.postprocess import create_test_metadata, run_and_postprocess @@ -197,7 +191,6 @@ def _run_test( skipped_optional_phase_workloads, ) run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - benchmark_start: datetime | None = None # Standalone analysis only — no deployment needed if not run_benchmark and not profiler_enabled: @@ -221,9 +214,6 @@ def _run_test( wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) try: - validate_user_workload_monitoring() - prepare_user_workload_monitoring(during="test") - isvc_labels = { "opendatahub.io/dashboard": "true", "deployment_uuid": run_uuid, @@ -347,8 +337,6 @@ def _run_test( # Phase 2: benchmark + post-processing for ALL workloads trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None - if run_benchmark: - benchmark_start = datetime.now(UTC) for wl_key in workload_keys: _run_workload_benchmark( model_key=model_key, @@ -391,13 +379,6 @@ def _run_test( except Exception: logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) finally: - if benchmark_start is not None: - _capture_prometheus_metrics( - deployment_name=deployment_name, - namespace=namespace, - start_time=benchmark_start, - end_time=datetime.now(UTC), - ) _capture_and_cleanup(deployment_name, namespace) if fjob_name: _update_fjob_inference_reference(fjob_name, fjob_ns, None) @@ -418,34 +399,6 @@ def _run_test( return 0 -def _capture_prometheus_metrics( - *, - deployment_name: str, - namespace: str, - start_time: datetime, - end_time: datetime, -) -> None: - """Capture Prometheus data for the RHAIIS benchmark window before cleanup.""" - logger.info( - "Capturing Prometheus metrics for RHAIIS benchmark window: %s - %s", - start_time.isoformat(), - end_time.isoformat(), - ) - try: - with env.NextArtifactDir("prometheus_metrics"): - capture_prometheus( - start_time, - end_time, - runtime_params={ - "namespace": namespace, - "pod_name": f"{deployment_name}-predictor-.*", - }, - ) - except Exception: - logger.warning("Prometheus metrics capture failed; continuing cleanup", exc_info=True) - _warnings.append("Prometheus metrics capture failed") - - def _run_workload_benchmark( *, model_key: str, From 6cbec0d1de43e9d7e923c248c37a91366d479add Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Mon, 26 Oct 2026 10:19:27 +0200 Subject: [PATCH 09/25] add burn-in phase before performance benchmarks, fix benchconf to force-reinstall custom branches --- projects/guidellm/library/benchconf.py | 4 +- projects/llm_d/README.md | 21 ++++- .../llm_d/orchestration/config.d/burnin.yaml | 9 ++ .../llm_d/orchestration/runtime_config.py | 9 ++ projects/llm_d/orchestration/test_phase.py | 82 +++++++++++++++++++ 5 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 projects/llm_d/orchestration/config.d/burnin.yaml diff --git a/projects/guidellm/library/benchconf.py b/projects/guidellm/library/benchconf.py index 1ff4e18a2..5ff75b3d2 100644 --- a/projects/guidellm/library/benchconf.py +++ b/projects/guidellm/library/benchconf.py @@ -37,7 +37,9 @@ def maybe_install_custom_version(_cfg) -> None: spec = f"benchconf @ {custom['repo']}@{custom['version']}" logger.info("Installing benchconf: %s", spec) - subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", spec]) + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "--quiet", "--force-reinstall", "--no-deps", spec] + ) def save_version() -> None: diff --git a/projects/llm_d/README.md b/projects/llm_d/README.md index d0f565542..9d39b8b59 100644 --- a/projects/llm_d/README.md +++ b/projects/llm_d/README.md @@ -76,4 +76,23 @@ Benchmark adaptation notes: - rate-dependent benchmarks (args containing `{rate}` / `{N*rate}` / `{rate*N}`) expand into one GuideLLM run per rate - plain multi-rate benchmarks (no `{rate}` expressions) stay a single GuideLLM invocation - expressions such as `{2*rate}` and `{10*rate}` are resolved in those expanded runs -- Features such as pre-warmup and env passthrough are not modeled yet + +## Burn-in Phase + +Before performance benchmarks, a **burn-in phase** runs to prime the KV cache and CUDA kernels. +This ensures stable, representative benchmark results by warming up the inference engine. + +Configuration is in [`orchestration/config.d/burnin.yaml`](./orchestration/config.d/burnin.yaml): + +| Setting | Default | Description | +|---------|---------|-------------| +| `burnin.enabled` | `true` | Enable/disable burn-in phase | +| `burnin.rate` | `200` | Concurrent streams during burn-in | +| `burnin.max_seconds` | `75` | Duration of burn-in in seconds | + +Burn-in automatically skips for `smoke` tests (functional checks don't need warming). + +To disable burn-in for a specific run: +```text +/var burnin.enabled: false +``` diff --git a/projects/llm_d/orchestration/config.d/burnin.yaml b/projects/llm_d/orchestration/config.d/burnin.yaml new file mode 100644 index 000000000..b193f0780 --- /dev/null +++ b/projects/llm_d/orchestration/config.d/burnin.yaml @@ -0,0 +1,9 @@ +# Burn-in phase configuration +# Runs a separate GuideLLM job before the main benchmark to prime KV cache and CUDA kernels. +# This helps achieve stable, representative benchmark results. + +burnin: + enabled: true + rate: 200 # Concurrent streams during burn-in (high load) + max_seconds: 75 # Duration of burn-in phase in seconds + job_name_prefix: guidellm-burnin diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index ca0383b52..5e3c7e3a5 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -619,6 +619,15 @@ def get_smoke_request() -> dict[str, Any]: return copy.deepcopy(config.project.get_config(f"workloads.smoke_requests.{smoke_request_key}")) +def get_burnin_config() -> dict[str, Any]: + """Get the burn-in phase configuration. + + Returns: + Dictionary with burn-in settings (enabled, rate, max_seconds, job_name_prefix) + """ + return copy.deepcopy(config.project.get_config("burnin", {})) + + def get_run_specs() -> list[RunSpec]: model_names = _normalize_string_or_list(_get_runtime_value("model_name"), "runtime.model_name") profile_names = _normalize_string_or_list( diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 813e4f38b..6c3cf881d 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -433,6 +433,9 @@ def do_test() -> int: run_smoke_request(endpoint_url=endpoint_url) + # Run burn-in phase if enabled (primes KV cache and CUDA kernels) + _run_burnin_if_enabled(endpoint_url=endpoint_url) + benchmark_start = datetime.now(UTC) try: benchmark_times = run_guidellm_benchmark(test_dir, endpoint_url=endpoint_url) @@ -750,6 +753,85 @@ def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: ) +def _should_run_burnin() -> bool: + """Determine if burn-in phase should run. + + Burn-in runs for performance benchmarks only, not for smoke tests. + """ + burnin_cfg = runtime_config.get_burnin_config() + if not burnin_cfg.get("enabled", False): + return False + + # Skip burn-in for smoke/functional tests + benchmark_keys = runtime_config.get_benchmark_keys() + if benchmark_keys and benchmark_keys[0] == "smoke": + return False + + return True + + +def _run_burnin_if_enabled(*, endpoint_url: str) -> None: + """Run burn-in phase if enabled to prime KV cache and CUDA kernels. + + Creates a separate GuideLLM job with high concurrency for a short duration + before the main benchmark to ensure stable, representative results. + """ + if not _should_run_burnin(): + logger.info("Burn-in phase skipped (disabled or smoke test)") + return + + burnin_cfg = runtime_config.get_burnin_config() + namespace = runtime_config.get_namespace() + benchmark = runtime_config.get_benchmark_config() + + burnin_rate = burnin_cfg.get("rate", 200) + burnin_duration = burnin_cfg.get("max_seconds", 75) + job_name_prefix = burnin_cfg.get("job_name_prefix", "guidellm-burnin") + + logger.info( + "Running burn-in phase (rate=%d, duration=%ds) to prime KV cache and CUDA kernels", + burnin_rate, + burnin_duration, + ) + + # Build burn-in GuideLLM args - use same kind ofdata profile as benchmark but fixed rate and duration + burnin_args = [] + + # Get data specification from benchmark config + if benchmark: + benchmark_args = benchmark.get("args", {}) + if "data" in benchmark_args: + burnin_args.append(f"--data={benchmark_args['data']}") + + # Set burn-in profile: high concurrency, short duration + burnin_args.append(f"--profile=kind=concurrent,streams={burnin_rate}") + burnin_args.append(f"--constraint=kind=max_duration,seconds={burnin_duration}") + + # Add tokenizer for the model + burnin_args.append(f"--tokenizer=kind=huggingface_auto,model={runtime_config.get_model_name()}") + + # Get fs_group from workload config + workload = runtime_config.get_workload_config() + fs_group = workload.get("fs_group") if workload else None + + with env.NextArtifactDir("burnin"): + try: + run_guidellm_benchmark_command.run( + endpoint_url=endpoint_url, + name=job_name_prefix, + namespace=namespace, + image=benchmark.get("image") if benchmark else None, + timeout=burnin_duration + 300, # Buffer for pod startup and result collection + pvc_size=benchmark.get("pvc_size") if benchmark else "1Gi", + guidellm_args=burnin_args, + fs_group=fs_group, + use_pvc=benchmark.get("use_pvc") if benchmark else False, + ) + logger.info("Burn-in phase completed successfully") + except Exception: + logger.warning("Burn-in phase failed; continuing with benchmark", exc_info=True) + + def run_guidellm_benchmark(test_dir, *, endpoint_url: str) -> tuple[datetime, datetime] | None: namespace = runtime_config.get_namespace() benchmark = runtime_config.get_benchmark_config() From 0534f116eaacc4a60535818282f105fa410f9f85 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Mon, 26 Oct 2026 14:17:08 +0200 Subject: [PATCH 10/25] change installation of benchconf from my branch to test rhaiis profiles alignment --- projects/guidellm/library/benchconf.py | 4 +--- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/projects/guidellm/library/benchconf.py b/projects/guidellm/library/benchconf.py index 5ff75b3d2..1ff4e18a2 100644 --- a/projects/guidellm/library/benchconf.py +++ b/projects/guidellm/library/benchconf.py @@ -37,9 +37,7 @@ def maybe_install_custom_version(_cfg) -> None: spec = f"benchconf @ {custom['repo']}@{custom['version']}" logger.info("Installing benchconf: %s", spec) - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "--quiet", "--force-reinstall", "--no-deps", spec] - ) + subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", spec]) def save_version() -> None: diff --git a/pyproject.toml b/pyproject.toml index 230209244..21270118e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ agentic_review = [ "langchain-core", ] benchconf = [ - "benchconf @ git+https://github.com/openshift-psap/benchconf@main", + "benchconf @ git+https://github.com/openshift-psap/benchconf@tosokin-llm-d-profiles-align-with-rhaiis", ] [project.scripts] From 2b253ffbd43df899fef3869e5fbc6a27d8532935 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Mon, 26 Oct 2026 16:30:59 +0200 Subject: [PATCH 11/25] fix nested burnin key --- projects/llm_d/orchestration/config.d/burnin.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/burnin.yaml b/projects/llm_d/orchestration/config.d/burnin.yaml index b193f0780..679b6d769 100644 --- a/projects/llm_d/orchestration/config.d/burnin.yaml +++ b/projects/llm_d/orchestration/config.d/burnin.yaml @@ -2,8 +2,7 @@ # Runs a separate GuideLLM job before the main benchmark to prime KV cache and CUDA kernels. # This helps achieve stable, representative benchmark results. -burnin: - enabled: true - rate: 200 # Concurrent streams during burn-in (high load) - max_seconds: 75 # Duration of burn-in phase in seconds - job_name_prefix: guidellm-burnin +enabled: true +rate: 200 # Concurrent streams during burn-in (high load) +max_seconds: 75 # Duration of burn-in phase in seconds +job_name_prefix: guidellm-burnin From 29b1f7268445b5f439542e50915cb343ba7ea255 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 28 Oct 2026 09:15:39 +0200 Subject: [PATCH 12/25] Force custom benchconf branch installation --- projects/guidellm/library/benchconf.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/projects/guidellm/library/benchconf.py b/projects/guidellm/library/benchconf.py index 1ff4e18a2..dba31e0ce 100644 --- a/projects/guidellm/library/benchconf.py +++ b/projects/guidellm/library/benchconf.py @@ -34,10 +34,29 @@ def maybe_install_custom_version(_cfg) -> None: import subprocess import sys + import tempfile spec = f"benchconf @ {custom['repo']}@{custom['version']}" - logger.info("Installing benchconf: %s", spec) - subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", spec]) + install_dir = Path(tempfile.mkdtemp(prefix="benchconf-")) + logger.info("Installing benchconf: %s into %s", spec, install_dir) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + "--quiet", + "--force-reinstall", + "--target", + str(install_dir), + spec, + ] + ) + sys.path.insert(0, str(install_dir)) + for module_name in [ + m for m in sys.modules if m == "benchconf" or m.startswith("benchconf.") + ]: + del sys.modules[module_name] def save_version() -> None: From 9bc8a710e4863417df39ee692c05bb5841b8c71d Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 28 Oct 2026 09:26:30 +0200 Subject: [PATCH 13/25] Format benchconf installer --- projects/guidellm/library/benchconf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/guidellm/library/benchconf.py b/projects/guidellm/library/benchconf.py index dba31e0ce..32becde44 100644 --- a/projects/guidellm/library/benchconf.py +++ b/projects/guidellm/library/benchconf.py @@ -53,9 +53,7 @@ def maybe_install_custom_version(_cfg) -> None: ] ) sys.path.insert(0, str(install_dir)) - for module_name in [ - m for m in sys.modules if m == "benchconf" or m.startswith("benchconf.") - ]: + for module_name in [m for m in sys.modules if m == "benchconf" or m.startswith("benchconf.")]: del sys.modules[module_name] From e7cf49a0ef7132d70c798a108707f6f86333f95a Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 28 Oct 2026 16:07:09 +0200 Subject: [PATCH 14/25] update the multi-turn profile to load parameters from benchconf and add the default-3.6 recipe --- .../llm_d/orchestration/config.d/deployments.yaml | 8 ++++++++ projects/llm_d/orchestration/config.d/workloads.yaml | 10 +--------- .../manifests/scheduler_config/default-3.6GA.yaml | 12 ++++++++++++ 3 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 projects/llm_d/orchestration/manifests/scheduler_config/default-3.6GA.yaml diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index d14a68167..bf1ed4b0a 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -113,6 +113,14 @@ profiles: args: max_model_len: 40960 + default-3.6GA: + scheduler_manifest: manifests/deployments/default-3.6GA.yaml + replicas: 4 + tensor_parallelism: 2 + vllm_extra: + args: + max_model_len: 40960 + # Template flavor configurations with name-based extraction templates: pd: diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 2075359a3..0bad50d0c 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -43,13 +43,5 @@ benchmarks: multi-turn: vllm_args: max_model_len: 40960 - # One Job runs all five rates sequentially; the high-rate cases need more - # than the default one-hour wall-clock budget. timeout_seconds: 7200 - # Uses inline args instead of benchconf because prefix_count and - # max_requests scale with the rate via forge's {rate} expressions. - args: - profile: "kind=concurrent,streams={rate}" - data: "kind=synthetic_text,prompt_tokens=128,output_tokens=128,turns=5,prefix_buckets[0].prefix_tokens=10000,prefix_buckets[0].prefix_count={2*rate}" - constraint: "kind=max_requests,count={10*rate}" - rate: [32, 64, 128, 256, 512] + benchconf: llm-d/multi-turn diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/default-3.6GA.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/default-3.6GA.yaml new file mode 100644 index 000000000..570b6b5ba --- /dev/null +++ b/projects/llm_d/orchestration/manifests/scheduler_config/default-3.6GA.yaml @@ -0,0 +1,12 @@ +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig +plugins: + - type: approx-prefix-cache-producer + - type: inflight-load-producer + - type: prefix-cache-affinity-filter + - type: token-load-scorer +schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-affinity-filter + - pluginRef: token-load-scorer From 23c0ed27f66ec17fbf65b2b2c782df6c30971342 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Wed, 28 Oct 2026 16:41:02 +0200 Subject: [PATCH 15/25] add retires to copy guidellm results --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index e5a0ef105..bd927dfa3 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -823,6 +823,7 @@ def _extract_from_copy_pod_file( result = oc( "cp", + "--retries=99", "-n", ctx.target_namespace, f"{copy_pod}:{remote_gz_path}", @@ -983,6 +984,7 @@ def _extract_from_sidecar_pod( result = oc( "cp", + "--retries=99", "-n", ctx.target_namespace, "-c", From 5aeef2412eeccee0d4fb65f0cce6b00e69ab6a78 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Thu, 29 Oct 2026 09:06:53 +0200 Subject: [PATCH 16/25] update the burnin period to 200 seconds, align with RHAIIS --- projects/llm_d/README.md | 2 +- projects/llm_d/orchestration/config.d/burnin.yaml | 2 +- projects/llm_d/orchestration/test_phase.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/README.md b/projects/llm_d/README.md index 9d39b8b59..c98165808 100644 --- a/projects/llm_d/README.md +++ b/projects/llm_d/README.md @@ -88,7 +88,7 @@ Configuration is in [`orchestration/config.d/burnin.yaml`](./orchestration/confi |---------|---------|-------------| | `burnin.enabled` | `true` | Enable/disable burn-in phase | | `burnin.rate` | `200` | Concurrent streams during burn-in | -| `burnin.max_seconds` | `75` | Duration of burn-in in seconds | +| `burnin.max_seconds` | `200` | Duration of burn-in in seconds | Burn-in automatically skips for `smoke` tests (functional checks don't need warming). diff --git a/projects/llm_d/orchestration/config.d/burnin.yaml b/projects/llm_d/orchestration/config.d/burnin.yaml index 679b6d769..b1ab85f85 100644 --- a/projects/llm_d/orchestration/config.d/burnin.yaml +++ b/projects/llm_d/orchestration/config.d/burnin.yaml @@ -4,5 +4,5 @@ enabled: true rate: 200 # Concurrent streams during burn-in (high load) -max_seconds: 75 # Duration of burn-in phase in seconds +max_seconds: 200 # Duration of burn-in phase in seconds job_name_prefix: guidellm-burnin diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 6c3cf881d..c020440f3 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -785,7 +785,7 @@ def _run_burnin_if_enabled(*, endpoint_url: str) -> None: benchmark = runtime_config.get_benchmark_config() burnin_rate = burnin_cfg.get("rate", 200) - burnin_duration = burnin_cfg.get("max_seconds", 75) + burnin_duration = burnin_cfg.get("max_seconds", 200) job_name_prefix = burnin_cfg.get("job_name_prefix", "guidellm-burnin") logger.info( From 580254904fc93b67b4d84838d7d8d2996667f04f Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Thu, 29 Oct 2026 10:23:28 +0200 Subject: [PATCH 17/25] burnin phase take data from corresponding benchconf profile --- projects/llm_d/orchestration/test_phase.py | 24 +++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index c020440f3..5d2768dbb 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -43,7 +43,7 @@ from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, ) -from projects.llm_d.orchestration.utils import write_yaml +from projects.llm_d.orchestration.utils import load_yaml, write_yaml from projects.llm_d.toolbox.cleanup_test_resources import main as cleanup_test_resources_command logger = logging.getLogger(__name__) @@ -794,14 +794,23 @@ def _run_burnin_if_enabled(*, endpoint_url: str) -> None: burnin_duration, ) - # Build burn-in GuideLLM args - use same kind ofdata profile as benchmark but fixed rate and duration burnin_args = [] - # Get data specification from benchmark config - if benchmark: - benchmark_args = benchmark.get("args", {}) - if "data" in benchmark_args: - burnin_args.append(f"--data={benchmark_args['data']}") + benchconf_ref = benchmark["benchconf"] + benchconf_lib.maybe_install_custom_version() + source_config_path = benchconf_lib.resolve_config_path(benchconf_ref) + source_config = load_yaml(source_config_path) + + try: + burnin_data = source_config["spec"]["data"] + except (KeyError, TypeError) as exc: + raise ValueError(f"Benchconf scenario has no data for burn-in: {benchconf_ref}") from exc + + # Keep only the selected scenario's data. The burn-in profile and duration + # are set separately below and should not inherit the benchmark matrix. + burnin_config_path = env.ARTIFACT_DIR / "burnin-guidellm-config.yaml" + write_yaml(burnin_config_path, {"spec": {"data": burnin_data}}) + logger.info("Using data from Benchconf scenario %s for burn-in", benchconf_ref) # Set burn-in profile: high concurrency, short duration burnin_args.append(f"--profile=kind=concurrent,streams={burnin_rate}") @@ -824,6 +833,7 @@ def _run_burnin_if_enabled(*, endpoint_url: str) -> None: timeout=burnin_duration + 300, # Buffer for pod startup and result collection pvc_size=benchmark.get("pvc_size") if benchmark else "1Gi", guidellm_args=burnin_args, + config_path=burnin_config_path, fs_group=fs_group, use_pvc=benchmark.get("use_pvc") if benchmark else False, ) From a712d55ceb2dbbca27b9933b7eb80aebb9fa3470 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Thu, 29 Oct 2026 10:30:20 +0200 Subject: [PATCH 18/25] update tests for multi-turn profile --- projects/llm_d/tests/test_profiles.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 979d03a12..608dc350f 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -107,10 +107,7 @@ def test_benchmark_workloads_are_available() -> None: assert concurrent["benchconf"] == "llm-d/concurrent-1k-1k" assert heavy["benchconf"] == "llm-d/concurrent-heavy-heterogeneous" - assert multi_turn["rate"] == [32, 64, 128, 256, 512] - assert "turns=5" in multi_turn["args"]["data"] - assert "prefix_count={2*rate}" in multi_turn["args"]["data"] - assert "kind=max_requests,count={10*rate}" == multi_turn["args"]["constraint"] + assert multi_turn["benchconf"] == "llm-d/multi-turn" def test_benchmark_resolution_applies_workload_defaults_and_per_benchmark_overrides() -> None: From e6348cb5216a6a3c80120b4d60de7b3aeb8bb717 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Thu, 29 Oct 2026 13:15:18 +0200 Subject: [PATCH 19/25] add retries when copying prometheus metrics --- projects/core/dsl/utils/k8s.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index a843da00c..b5d75c069 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -372,7 +372,7 @@ def oc_cp_from_pod( Returns: CommandResult with execution details """ - args = ["cp", f"{namespace}/{pod}:{remote_path}", str(local_path)] + args = ["cp", "--retries=99", f"{namespace}/{pod}:{remote_path}", str(local_path)] if container: args.extend(["-c", container]) return oc(*args) From 85db9d3981d0c37af47847df05d4215d871f4050 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Sun, 1 Nov 2026 09:15:15 +0200 Subject: [PATCH 20/25] align with the rest of RHAIIS profiles --- projects/llm_d/orchestration/config.d/workloads.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 0bad50d0c..3a16e9bf4 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -34,12 +34,20 @@ benchmarks: timeout_seconds: 3600 benchconf: llm-d/concurrent-1k-1k + concurrent-8k-1k: + timeout_seconds: 3600 + benchconf: llm-d/concurrent-8k-1k + heavy-heterogeneous: timeout_seconds: 3600 vllm_args: max_model_len: 40960 benchconf: llm-d/concurrent-heavy-heterogeneous + heavy-load: + timeout_seconds: 3600 + benchconf: llm-d/heavy-load + multi-turn: vllm_args: max_model_len: 40960 From 38b81adf24fb526ebafe56273011e1fd2bbfefb4 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Sun, 1 Nov 2026 10:38:52 +0200 Subject: [PATCH 21/25] update the max-model-length argument based on recent regression readout --- projects/llm_d/orchestration/config.d/deployments.yaml | 10 ---------- projects/llm_d/orchestration/config.d/workloads.yaml | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index bf1ed4b0a..8a0d47b10 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -76,9 +76,6 @@ profiles: scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml replicas: 4 tensor_parallelism: 2 - vllm_extra: - args: - max_model_len: 40960 release-precise-prefix-cache: scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml @@ -88,7 +85,6 @@ profiles: benchflow.io/prefix-caching: precise vllm_extra: args: - max_model_len: 40960 prefix_caching_hash_algo: sha256_cbor block_size: 64 # The serving image expands VLLM_ADDITIONAL_ARGS through `eval`; literal @@ -109,17 +105,11 @@ profiles: scheduler: {} replicas: 4 tensor_parallelism: 2 - vllm_extra: - args: - max_model_len: 40960 default-3.6GA: scheduler_manifest: manifests/deployments/default-3.6GA.yaml replicas: 4 tensor_parallelism: 2 - vllm_extra: - args: - max_model_len: 40960 # Template flavor configurations with name-based extraction templates: diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 3a16e9bf4..eb59cc4c3 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -20,6 +20,8 @@ benchmarks: short: timeout_seconds: 3600 + vllm_args: + max_model_len: 40960 args: profile: "kind=concurrent,streams={rate}" data: kind=synthetic_text,prompt_tokens=256,output_tokens=128 @@ -28,14 +30,20 @@ benchmarks: smoke: timeout_seconds: 300 + vllm_args: + max_model_len: 40960 benchconf: llm-d/smoke concurrent-1k-1k: timeout_seconds: 3600 + vllm_args: + max_model_len: 8192 benchconf: llm-d/concurrent-1k-1k concurrent-8k-1k: timeout_seconds: 3600 + vllm_args: + max_model_len: 40960 benchconf: llm-d/concurrent-8k-1k heavy-heterogeneous: @@ -46,6 +54,8 @@ benchmarks: heavy-load: timeout_seconds: 3600 + vllm_args: + max_model_len: 40960 benchconf: llm-d/heavy-load multi-turn: From 1c04a21bb49e85bc7e606c78f1e625a1a91727c0 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Sun, 1 Nov 2026 10:50:08 +0200 Subject: [PATCH 22/25] fix bug introduced in previous commit --- projects/llm_d/orchestration/config.d/workloads.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index eb59cc4c3..b8c56da64 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -20,8 +20,6 @@ benchmarks: short: timeout_seconds: 3600 - vllm_args: - max_model_len: 40960 args: profile: "kind=concurrent,streams={rate}" data: kind=synthetic_text,prompt_tokens=256,output_tokens=128 @@ -30,8 +28,6 @@ benchmarks: smoke: timeout_seconds: 300 - vllm_args: - max_model_len: 40960 benchconf: llm-d/smoke concurrent-1k-1k: From 6572117bab6801b948d1e16a01b0c7e0d6fb9bb5 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Mon, 2 Nov 2026 11:50:59 +0200 Subject: [PATCH 23/25] revert changes done for debugging before merging into main --- .../orchestration/config.d/benchconf.yaml | 4 ++-- .../llm_d/orchestration/presets.d/cpt.yaml | 1 + .../orchestration/config.d/workloads.yaml | 22 ------------------- pyproject.toml | 2 +- remove-before-merge.txt | 1 - 5 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 remove-before-merge.txt diff --git a/projects/llm_d/orchestration/config.d/benchconf.yaml b/projects/llm_d/orchestration/config.d/benchconf.yaml index b4478e1aa..5d6057f73 100644 --- a/projects/llm_d/orchestration/config.d/benchconf.yaml +++ b/projects/llm_d/orchestration/config.d/benchconf.yaml @@ -1,5 +1,5 @@ enabled: true custom_version: - enabled: true + enabled: false repo: git+https://github.com/openshift-psap/benchconf - version: tosokin-llm-d-profiles-align-with-rhaiis + version: main diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index 2dcab3423..78b686963 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -42,6 +42,7 @@ cpt-release-testing-rhoai: - concurrent-1k-1k - heavy-heterogeneous - multi-turn + - concurrent-8k-1k model_cache.pvc.size: 300Gi cpt-release-testing-gpt-oss-120b: diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index d960144e1..6b3742217 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -10,14 +10,6 @@ profile1: rampup: 35 warmup: 75 -# Debug profile matching profile1 with the rates in reverse order. -debug-profile1-reverse: - data: "prompt_tokens=1000,output_tokens=1000" - rates: [300,200,100,50,1] - max_seconds: 275 - rampup: 35 - warmup: 75 - profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" rates: [1,50,100,200,300] @@ -61,17 +53,3 @@ profile7: max_seconds: 275 rampup: 35 warmup: 75 - -# Debug profile matching llm-d concurrent-1k-1k without ramp-up or warmup. -debug-guidellm-1k-1k: - data: "prompt_tokens=1000,output_tokens=1000" - rates: [300, 200, 100, 50, 1] - max_seconds: 600 - -# Dashboard override stub (set data, rates, max_seconds, rampup, and warmup at runtime) -custom: - data: "prompt_tokens=1000,output_tokens=1000" - rates: [1] - max_seconds: 275 - rampup: 35 - warmup: 75 diff --git a/pyproject.toml b/pyproject.toml index 3e8cb4f2a..a2ecd3249 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ agentic_review = [ "langchain-core", ] benchconf = [ - "benchconf @ git+https://github.com/openshift-psap/benchconf@tosokin-llm-d-profiles-align-with-rhaiis", + "benchconf @ git+https://github.com/openshift-psap/benchconf@main", ] [project.scripts] diff --git a/remove-before-merge.txt b/remove-before-merge.txt deleted file mode 100644 index 8b1378917..000000000 --- a/remove-before-merge.txt +++ /dev/null @@ -1 +0,0 @@ - From 45abee39f113fadbd3e288a0e8e091171f71f2a6 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Mon, 2 Nov 2026 11:55:53 +0200 Subject: [PATCH 24/25] leftovers from debugging --- projects/rhaiis/orchestration/config.d/workloads.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 6b3742217..1ddebc0cd 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -53,3 +53,11 @@ profile7: max_seconds: 275 rampup: 35 warmup: 75 + +# Dashboard override stub (set data, rates, max_seconds, rampup, and warmup at runtime) +custom: + data: "prompt_tokens=1000,output_tokens=1000" + rates: [1] + max_seconds: 275 + rampup: 35 + warmup: 75 From 729b1db32dc1b4e90b21316e544d2a0aacabc682 Mon Sep 17 00:00:00 2001 From: Tanya Osokin Date: Mon, 2 Nov 2026 12:07:15 +0200 Subject: [PATCH 25/25] update stale test expectations with additional benchmark key --- projects/llm_d/tests/test_profiles.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 608dc350f..acc64fae7 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -194,6 +194,7 @@ def test_release_preset_expands_benchmark_list_and_merges_workload_args() -> Non "concurrent-1k-1k", "heavy-heterogeneous", "multi-turn", + "concurrent-8k-1k", ] run_specs = runtime_config.get_run_specs() @@ -210,11 +211,12 @@ def test_gpt_release_preset_produces_deployment_workload_matrix() -> None: run_specs = runtime_config.get_run_specs() - assert len(run_specs) == 9 + assert len(run_specs) == 12 assert {spec.benchmark_key for spec in run_specs} == { "concurrent-1k-1k", "heavy-heterogeneous", "multi-turn", + "concurrent-8k-1k", } assert all(spec.model_name == "openai/gpt-oss-120b" for spec in run_specs) assert {spec.deployment_profile_name for spec in run_specs} == { @@ -241,7 +243,7 @@ def test_llama_release_preset_produces_deployment_workload_matrix() -> None: ) run_specs = runtime_config.get_run_specs() - assert len(run_specs) == 9 + assert len(run_specs) == 12 assert {spec.deployment_profile_name for spec in run_specs} == { "release-distributed-default", "release-precise-prefix-cache", @@ -251,6 +253,7 @@ def test_llama_release_preset_produces_deployment_workload_matrix() -> None: "concurrent-1k-1k", "heavy-heterogeneous", "multi-turn", + "concurrent-8k-1k", } assert {spec.namespace for spec in run_specs} == {"forge-llm-d"}