diff --git a/projects/guidellm/library/benchconf.py b/projects/guidellm/library/benchconf.py index 1ff4e18a2..32becde44 100644 --- a/projects/guidellm/library/benchconf.py +++ b/projects/guidellm/library/benchconf.py @@ -34,10 +34,27 @@ def maybe_install_custom_version(_cfg) -> None: import subprocess import sys + import tempfile spec = f"benchconf @ {custom['repo']}@{custom['version']}" - logger.info("Installing benchconf: %s", spec) - subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", spec]) + install_dir = Path(tempfile.mkdtemp(prefix="benchconf-")) + logger.info("Installing benchconf: %s into %s", spec, install_dir) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + "--quiet", + "--force-reinstall", + "--target", + str(install_dir), + spec, + ] + ) + sys.path.insert(0, str(install_dir)) + for module_name in [m for m in sys.modules if m == "benchconf" or m.startswith("benchconf.")]: + del sys.modules[module_name] def save_version() -> None: diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index e5a0ef105..bd927dfa3 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -823,6 +823,7 @@ def _extract_from_copy_pod_file( result = oc( "cp", + "--retries=99", "-n", ctx.target_namespace, f"{copy_pod}:{remote_gz_path}", @@ -983,6 +984,7 @@ def _extract_from_sidecar_pod( result = oc( "cp", + "--retries=99", "-n", ctx.target_namespace, "-c", diff --git a/projects/llm_d/README.md b/projects/llm_d/README.md index d0f565542..9d39b8b59 100644 --- a/projects/llm_d/README.md +++ b/projects/llm_d/README.md @@ -76,4 +76,23 @@ Benchmark adaptation notes: - rate-dependent benchmarks (args containing `{rate}` / `{N*rate}` / `{rate*N}`) expand into one GuideLLM run per rate - plain multi-rate benchmarks (no `{rate}` expressions) stay a single GuideLLM invocation - expressions such as `{2*rate}` and `{10*rate}` are resolved in those expanded runs -- Features such as pre-warmup and env passthrough are not modeled yet + +## Burn-in Phase + +Before performance benchmarks, a **burn-in phase** runs to prime the KV cache and CUDA kernels. +This ensures stable, representative benchmark results by warming up the inference engine. + +Configuration is in [`orchestration/config.d/burnin.yaml`](./orchestration/config.d/burnin.yaml): + +| Setting | Default | Description | +|---------|---------|-------------| +| `burnin.enabled` | `true` | Enable/disable burn-in phase | +| `burnin.rate` | `200` | Concurrent streams during burn-in | +| `burnin.max_seconds` | `75` | Duration of burn-in in seconds | + +Burn-in automatically skips for `smoke` tests (functional checks don't need warming). + +To disable burn-in for a specific run: +```text +/var burnin.enabled: false +``` diff --git a/projects/llm_d/orchestration/config.d/benchconf.yaml b/projects/llm_d/orchestration/config.d/benchconf.yaml index 5d6057f73..b4478e1aa 100644 --- a/projects/llm_d/orchestration/config.d/benchconf.yaml +++ b/projects/llm_d/orchestration/config.d/benchconf.yaml @@ -1,5 +1,5 @@ enabled: true custom_version: - enabled: false + enabled: true repo: git+https://github.com/openshift-psap/benchconf - version: main + version: tosokin-llm-d-profiles-align-with-rhaiis diff --git a/projects/llm_d/orchestration/config.d/burnin.yaml b/projects/llm_d/orchestration/config.d/burnin.yaml new file mode 100644 index 000000000..679b6d769 --- /dev/null +++ b/projects/llm_d/orchestration/config.d/burnin.yaml @@ -0,0 +1,8 @@ +# Burn-in phase configuration +# Runs a separate GuideLLM job before the main benchmark to prime KV cache and CUDA kernels. +# This helps achieve stable, representative benchmark results. + +enabled: true +rate: 200 # Concurrent streams during burn-in (high load) +max_seconds: 75 # Duration of burn-in phase in seconds +job_name_prefix: guidellm-burnin diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index d14a68167..bf1ed4b0a 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -113,6 +113,14 @@ profiles: args: max_model_len: 40960 + default-3.6GA: + scheduler_manifest: manifests/deployments/default-3.6GA.yaml + replicas: 4 + tensor_parallelism: 2 + vllm_extra: + args: + max_model_len: 40960 + # Template flavor configurations with name-based extraction templates: pd: diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 2075359a3..0bad50d0c 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -43,13 +43,5 @@ benchmarks: multi-turn: vllm_args: max_model_len: 40960 - # One Job runs all five rates sequentially; the high-rate cases need more - # than the default one-hour wall-clock budget. timeout_seconds: 7200 - # Uses inline args instead of benchconf because prefix_count and - # max_requests scale with the rate via forge's {rate} expressions. - args: - profile: "kind=concurrent,streams={rate}" - data: "kind=synthetic_text,prompt_tokens=128,output_tokens=128,turns=5,prefix_buckets[0].prefix_tokens=10000,prefix_buckets[0].prefix_count={2*rate}" - constraint: "kind=max_requests,count={10*rate}" - rate: [32, 64, 128, 256, 512] + benchconf: llm-d/multi-turn diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/default-3.6GA.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/default-3.6GA.yaml new file mode 100644 index 000000000..570b6b5ba --- /dev/null +++ b/projects/llm_d/orchestration/manifests/scheduler_config/default-3.6GA.yaml @@ -0,0 +1,12 @@ +apiVersion: llm-d.ai/v1alpha1 +kind: EndpointPickerConfig +plugins: + - type: approx-prefix-cache-producer + - type: inflight-load-producer + - type: prefix-cache-affinity-filter + - type: token-load-scorer +schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-affinity-filter + - pluginRef: token-load-scorer diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index ca0383b52..5e3c7e3a5 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -619,6 +619,15 @@ def get_smoke_request() -> dict[str, Any]: return copy.deepcopy(config.project.get_config(f"workloads.smoke_requests.{smoke_request_key}")) +def get_burnin_config() -> dict[str, Any]: + """Get the burn-in phase configuration. + + Returns: + Dictionary with burn-in settings (enabled, rate, max_seconds, job_name_prefix) + """ + return copy.deepcopy(config.project.get_config("burnin", {})) + + def get_run_specs() -> list[RunSpec]: model_names = _normalize_string_or_list(_get_runtime_value("model_name"), "runtime.model_name") profile_names = _normalize_string_or_list( diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 813e4f38b..6c3cf881d 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -433,6 +433,9 @@ def do_test() -> int: run_smoke_request(endpoint_url=endpoint_url) + # Run burn-in phase if enabled (primes KV cache and CUDA kernels) + _run_burnin_if_enabled(endpoint_url=endpoint_url) + benchmark_start = datetime.now(UTC) try: benchmark_times = run_guidellm_benchmark(test_dir, endpoint_url=endpoint_url) @@ -750,6 +753,85 @@ def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: ) +def _should_run_burnin() -> bool: + """Determine if burn-in phase should run. + + Burn-in runs for performance benchmarks only, not for smoke tests. + """ + burnin_cfg = runtime_config.get_burnin_config() + if not burnin_cfg.get("enabled", False): + return False + + # Skip burn-in for smoke/functional tests + benchmark_keys = runtime_config.get_benchmark_keys() + if benchmark_keys and benchmark_keys[0] == "smoke": + return False + + return True + + +def _run_burnin_if_enabled(*, endpoint_url: str) -> None: + """Run burn-in phase if enabled to prime KV cache and CUDA kernels. + + Creates a separate GuideLLM job with high concurrency for a short duration + before the main benchmark to ensure stable, representative results. + """ + if not _should_run_burnin(): + logger.info("Burn-in phase skipped (disabled or smoke test)") + return + + burnin_cfg = runtime_config.get_burnin_config() + namespace = runtime_config.get_namespace() + benchmark = runtime_config.get_benchmark_config() + + burnin_rate = burnin_cfg.get("rate", 200) + burnin_duration = burnin_cfg.get("max_seconds", 75) + job_name_prefix = burnin_cfg.get("job_name_prefix", "guidellm-burnin") + + logger.info( + "Running burn-in phase (rate=%d, duration=%ds) to prime KV cache and CUDA kernels", + burnin_rate, + burnin_duration, + ) + + # Build burn-in GuideLLM args - use same kind ofdata profile as benchmark but fixed rate and duration + burnin_args = [] + + # Get data specification from benchmark config + if benchmark: + benchmark_args = benchmark.get("args", {}) + if "data" in benchmark_args: + burnin_args.append(f"--data={benchmark_args['data']}") + + # Set burn-in profile: high concurrency, short duration + burnin_args.append(f"--profile=kind=concurrent,streams={burnin_rate}") + burnin_args.append(f"--constraint=kind=max_duration,seconds={burnin_duration}") + + # Add tokenizer for the model + burnin_args.append(f"--tokenizer=kind=huggingface_auto,model={runtime_config.get_model_name()}") + + # Get fs_group from workload config + workload = runtime_config.get_workload_config() + fs_group = workload.get("fs_group") if workload else None + + with env.NextArtifactDir("burnin"): + try: + run_guidellm_benchmark_command.run( + endpoint_url=endpoint_url, + name=job_name_prefix, + namespace=namespace, + image=benchmark.get("image") if benchmark else None, + timeout=burnin_duration + 300, # Buffer for pod startup and result collection + pvc_size=benchmark.get("pvc_size") if benchmark else "1Gi", + guidellm_args=burnin_args, + fs_group=fs_group, + use_pvc=benchmark.get("use_pvc") if benchmark else False, + ) + logger.info("Burn-in phase completed successfully") + except Exception: + logger.warning("Burn-in phase failed; continuing with benchmark", exc_info=True) + + def run_guidellm_benchmark(test_dir, *, endpoint_url: str) -> tuple[datetime, datetime] | None: namespace = runtime_config.get_namespace() benchmark = runtime_config.get_benchmark_config() diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 1ddebc0cd..d960144e1 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -10,6 +10,14 @@ profile1: rampup: 35 warmup: 75 +# Debug profile matching profile1 with the rates in reverse order. +debug-profile1-reverse: + data: "prompt_tokens=1000,output_tokens=1000" + rates: [300,200,100,50,1] + max_seconds: 275 + rampup: 35 + warmup: 75 + profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" rates: [1,50,100,200,300] @@ -54,6 +62,12 @@ profile7: rampup: 35 warmup: 75 +# Debug profile matching llm-d concurrent-1k-1k without ramp-up or warmup. +debug-guidellm-1k-1k: + data: "prompt_tokens=1000,output_tokens=1000" + rates: [300, 200, 100, 50, 1] + max_seconds: 600 + # Dashboard override stub (set data, rates, max_seconds, rampup, and warmup at runtime) custom: data: "prompt_tokens=1000,output_tokens=1000" diff --git a/pyproject.toml b/pyproject.toml index a2ecd3249..3e8cb4f2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ agentic_review = [ "langchain-core", ] benchconf = [ - "benchconf @ git+https://github.com/openshift-psap/benchconf@main", + "benchconf @ git+https://github.com/openshift-psap/benchconf@tosokin-llm-d-profiles-align-with-rhaiis", ] [project.scripts] diff --git a/remove-before-merge.txt b/remove-before-merge.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/remove-before-merge.txt @@ -0,0 +1 @@ +