Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion perfkitbenchmarker/configs/default_benchmark_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -727,8 +727,9 @@ io500_lustre:
openmpi_version: 4.1.6
mpi_ranks_list: [1, 16, 64, 192]
ior_easy_direct_io: True
lustre_set_stripe: True
lustre_stripe_args: '-E 1M -c 1 -E -1 -c -1'
lustre_max_dirty_mb: 2000
io500_wait_for_cleanup: True

gpu_jobs_provisioning:
name: jobs_benchmark
Expand Down
19 changes: 11 additions & 8 deletions perfkitbenchmarker/disk_strategies.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,11 @@
from perfkitbenchmarker import os_types
from perfkitbenchmarker import vm_util

_LUSTRE_SET_STRIPE = flags.DEFINE_boolean(
'lustre_set_stripe',
False,
'Whether to set stripe count to -1 on the Lustre mount point.',
_LUSTRE_STRIPE_ARGS = flags.DEFINE_string(
'lustre_stripe_args',
None,
'Arguments passed to lfs setstripe on the Lustre mount point. Set to empty '
'or None to skip setting striping.',
)
_LUSTRE_MAX_DIRTY_MB = flags.DEFINE_integer(
'lustre_max_dirty_mb',
Expand Down Expand Up @@ -373,13 +374,15 @@ def SetUpDiskOnLinux(self):
)
lustre_disk.UpdateDevicePath(self.disk_spec.mount_point)
self.vm.scratch_disks.append(lustre_disk)
if FLAGS.lustre_set_stripe:
stripe_args = (_LUSTRE_STRIPE_ARGS.value or '').strip()
if stripe_args:
self.vm.RemoteCommand(
f'sudo lfs setstripe -c -1 {self.disk_spec.mount_point}'
f'sudo lfs setstripe {stripe_args} {self.disk_spec.mount_point}'
)
if FLAGS.lustre_max_dirty_mb:
if _LUSTRE_MAX_DIRTY_MB.value:
self.vm.RemoteCommand(
f'sudo lctl set_param osc.*.max_dirty_mb={FLAGS.lustre_max_dirty_mb}',
'sudo lctl set_param'
f' osc.*.max_dirty_mb={_LUSTRE_MAX_DIRTY_MB.value}',
ignore_failure=True,
)

Expand Down
100 changes: 96 additions & 4 deletions perfkitbenchmarker/linux_benchmarks/io500_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from perfkitbenchmarker import linux_packages
from perfkitbenchmarker import regex_util
from perfkitbenchmarker import sample
from perfkitbenchmarker import virtual_machine
from perfkitbenchmarker import vm_util

FLAGS = flags.FLAGS
Expand Down Expand Up @@ -78,6 +79,15 @@
'Total mpi ranks to use. By default, same as total cores available.',
)

# TODO(andytzhu): Verify df -k -P drain behavior on NFS/NetApp Volumes before
# enabling io500_wait_for_cleanup there.
_IO500_WAIT_FOR_CLEANUP = flags.DEFINE_boolean(
'io500_wait_for_cleanup',
False,
'If True, poll df between iterations until used filesystem space drains '
'back near pre-run baseline instead of sleeping a fixed 60s.',
)

# Section names in io500.ini.j2 that can be run
RUNNABLE_SECTIONS = [
'ior_easy_write', 'ior_easy_read',
Expand Down Expand Up @@ -125,6 +135,10 @@
# default ior-easy transfersize is 2MiB by default in io500.
# https://github.com/IO500/io500/blob/main/config-sample-all.ini
DEFAULT_IOR_EASY_TRANSFER_SIZE = 2 * 1024 * 1024 # 2 MiB
_CLEANUP_WAIT_TIMEOUT_SEC = 1800
_CLEANUP_POLL_INTERVAL_SEC = 30
_CLEANUP_TOLERANCE_RATIO = 0.005 # 0.5% of filesystem capacity
_MIN_CLEANUP_TOLERANCE_KB = 64 * 1024 # 64 MiB


def GetConfig(user_config):
Expand Down Expand Up @@ -326,6 +340,7 @@ def _Run(headnode, ranks, ppn, num_clients):
'invalid': '[INVALID]' in line,
'ior_hard_write_direct_io': _IOR_HARD_WRITE_DIRECT_IO.value,
'ior_easy_direct_io': _IOR_EASY_DIRECT_IO.value,
'io500_wait_for_cleanup': _IO500_WAIT_FOR_CLEANUP.value,
},
)
)
Expand All @@ -340,6 +355,7 @@ def _Run(headnode, ranks, ppn, num_clients):
'invalid': '[INVALID]' in line,
'ior_hard_write_direct_io': _IOR_HARD_WRITE_DIRECT_IO.value,
'ior_easy_direct_io': _IOR_EASY_DIRECT_IO.value,
'io500_wait_for_cleanup': _IO500_WAIT_FOR_CLEANUP.value,
}
results.extend([
sample.Sample('BANDWIDTH', raw_result[0], raw_result[1], metadata),
Expand All @@ -349,6 +365,66 @@ def _Run(headnode, ranks, ppn, num_clients):
return results


class _FilesystemNotDrainedError(Exception):
"""Raised while waiting for filesystem space to drain after an io500 run."""


def _GetFilesystemUsageKb(
vm: virtual_machine.BaseVirtualMachine, mount_point: str
) -> tuple[int, int]:
"""Returns `(used_kb, total_kb)` for `mount_point` from POSIX `df` output."""
stdout, _ = vm.RemoteCommand(f'df -k -P {mount_point} | tail -n 1')
parts = stdout.strip().split()
if len(parts) < 5:
raise errors.Benchmarks.RunError(
f'Unexpected df output for {mount_point}: {stdout!r}'
)
try:
total_kb, used_kb = int(parts[1]), int(parts[2])
except ValueError as e:
raise errors.Benchmarks.RunError(
f'Failed to parse df output for {mount_point}: {stdout!r}'
) from e
if total_kb < 0 or used_kb < 0:
raise errors.Benchmarks.RunError(
f'Negative df sizes for {mount_point}: {stdout!r}'
)
return (used_kb, total_kb)


@vm_util.Retry(
poll_interval=_CLEANUP_POLL_INTERVAL_SEC,
max_retries=-1,
timeout=_CLEANUP_WAIT_TIMEOUT_SEC,
fuzz=0,
retryable_exceptions=(
_FilesystemNotDrainedError,
errors.VirtualMachine.RemoteCommandError,
),
)
def _WaitForFilesystemDrain(
vm: virtual_machine.BaseVirtualMachine,
mount_point: str,
baseline_used_kb: int,
) -> None:
"""Polls `df` until used space returns near baseline or timeout expires."""
used_kb, total_kb = _GetFilesystemUsageKb(vm, mount_point)
threshold_kb = baseline_used_kb + max(
_MIN_CLEANUP_TOLERANCE_KB, int(total_kb * _CLEANUP_TOLERANCE_RATIO)
)
if used_kb > threshold_kb:
raise _FilesystemNotDrainedError(
f'Filesystem {mount_point} has not drained yet '
f'({used_kb} > {threshold_kb} KiB).'
)
logging.info(
'Filesystem %s drained (%d <= %d KiB).',
mount_point,
used_kb,
threshold_kb,
)


def Run(benchmark_spec):
"""Run the io500 benchmark on the vms.

Expand Down Expand Up @@ -403,12 +479,19 @@ def Run(benchmark_spec):
else [len(vms)]
)

for num_clients in client_counts:
baseline_used_kb = (
_GetFilesystemUsageKb(headnode, headnode.scratch_disks[0].mount_point)[0]
if _IO500_WAIT_FOR_CLEANUP.value
else 0
)

for client_idx, num_clients in enumerate(client_counts):
active_vms = vms[:num_clients]
num_nodes = len(active_vms)
for total_ranks in [int(r) for r in _MPI_RANKS_LIST.value] or [
ranks_list = [int(r) for r in _MPI_RANKS_LIST.value] or [
num_nodes * num_cpus
]:
]
for rank_idx, total_ranks in enumerate(ranks_list):
ppn = math.ceil(total_ranks / num_nodes)
# Only list the VMs these ranks need, so 1 rank runs on the headnode only.
run_vms = active_vms[: math.ceil(total_ranks / ppn)]
Expand Down Expand Up @@ -444,7 +527,16 @@ def Run(benchmark_spec):
results.extend(_Run(headnode, total_ranks, ppn, len(run_vms)))
if FLAGS.object_storage_fuse_log_trace:
headnode.RemoteCommand('sudo bash -c "cat /tmp/logs/*"')
time.sleep(60)
if _IO500_WAIT_FOR_CLEANUP.value:
if (
client_idx < len(client_counts) - 1
or rank_idx < len(ranks_list) - 1
):
_WaitForFilesystemDrain(
headnode, headnode.scratch_disks[0].mount_point, baseline_used_kb
)
else:
time.sleep(60)
return results


Expand Down
101 changes: 101 additions & 0 deletions tests/disk_strategies_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
# Copyright 2026 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest
from unittest import mock

from absl.testing import flagsaver
from absl.testing import parameterized
from perfkitbenchmarker import configs
from perfkitbenchmarker import disk
from perfkitbenchmarker import disk_strategies
from perfkitbenchmarker import linux_virtual_machine
from perfkitbenchmarker import lustre_service
from tests import mock_command
from tests import pkb_common_test_case


class SetUpLustreDiskStrategyTest(pkb_common_test_case.PkbCommonTestCase):

def _CreateLustreSetup(
self,
) -> tuple[
linux_virtual_machine.BaseLinuxVirtualMachine,
disk_strategies.SetUpLustreDiskStrategy,
mock_command.MockRemoteCommand,
]:
vm = mock.create_autospec(
linux_virtual_machine.BaseLinuxVirtualMachine,
instance=True,
)
vm.scratch_disks = []
mock_cmd = mock_command.MockRemoteCommand({}, vm)
disk_spec = disk.BaseLustreDiskSpec(
'test_lustre',
mount_point='/lustre_dir',
disk_type=disk.LUSTRE,
)
service = mock.create_autospec(
lustre_service.BaseLustreService,
instance=True,
spec_set=True,
)
service.CreateLustreDisk.return_value = disk.LustreDisk(disk_spec)
service.GetMountPoint.return_value = '10.0.0.1@tcp:/lustre'
strategy = disk_strategies.SetUpLustreDiskStrategy(vm, disk_spec, service)
return vm, strategy, mock_cmd

def testDefaultLustreStripeArgsSkipsSetstripe(self):
vm, strategy, mock_cmd = self._CreateLustreSetup()

strategy.SetUpDiskOnLinux()

self.assertLen(vm.scratch_disks, 1)
self.assertEqual(mock_cmd.all_commands, '')

@parameterized.named_parameters(
dict(testcase_name='empty', stripe_args=''),
dict(testcase_name='whitespace', stripe_args=' '),
)
def testBlankLustreStripeArgsSkipsSetstripe(self, stripe_args: str):
vm, strategy, mock_cmd = self._CreateLustreSetup()

with flagsaver.flagsaver(lustre_stripe_args=stripe_args):
strategy.SetUpDiskOnLinux()

self.assertLen(vm.scratch_disks, 1)
self.assertEqual(mock_cmd.all_commands, '')

@flagsaver.flagsaver(lustre_stripe_args='-E 1M -c 1 -E -1 -c -1')
def testConfiguredLustreStripeArgsRunsSetstripe(self):
_, strategy, mock_cmd = self._CreateLustreSetup()

strategy.SetUpDiskOnLinux()

self.assertEqual(
mock_cmd.all_commands,
'sudo lfs setstripe -E 1M -c 1 -E -1 -c -1 /lustre_dir\n',
)

def testDefaultBenchmarkConfigIo500LustreFlags(self):
io500_lustre_config = configs.GetDefaultAndUserConfig()['io500_lustre']
flags_dict = io500_lustre_config['flags']

self.assertEqual(flags_dict['lustre_stripe_args'], '-E 1M -c 1 -E -1 -c -1')
self.assertTrue(flags_dict['io500_wait_for_cleanup'])
self.assertEqual(flags_dict['lustre_max_dirty_mb'], 2000)


if __name__ == '__main__':
unittest.main()
Loading