diff --git a/perfkitbenchmarker/configs/default_benchmark_config.yaml b/perfkitbenchmarker/configs/default_benchmark_config.yaml index 466782d9b5..4358c97453 100644 --- a/perfkitbenchmarker/configs/default_benchmark_config.yaml +++ b/perfkitbenchmarker/configs/default_benchmark_config.yaml @@ -727,8 +727,9 @@ io500_lustre: openmpi_version: 4.1.6 mpi_ranks_list: [1, 16, 64, 192] ior_easy_direct_io: True - lustre_set_stripe: True + lustre_stripe_args: '-E 1M -c 1 -E -1 -c -1' lustre_max_dirty_mb: 2000 + io500_wait_for_cleanup: True gpu_jobs_provisioning: name: jobs_benchmark diff --git a/perfkitbenchmarker/disk_strategies.py b/perfkitbenchmarker/disk_strategies.py index 78d6ca4f8b..9602557920 100644 --- a/perfkitbenchmarker/disk_strategies.py +++ b/perfkitbenchmarker/disk_strategies.py @@ -39,10 +39,11 @@ from perfkitbenchmarker import os_types from perfkitbenchmarker import vm_util -_LUSTRE_SET_STRIPE = flags.DEFINE_boolean( - 'lustre_set_stripe', - False, - 'Whether to set stripe count to -1 on the Lustre mount point.', +_LUSTRE_STRIPE_ARGS = flags.DEFINE_string( + 'lustre_stripe_args', + None, + 'Arguments passed to lfs setstripe on the Lustre mount point. Set to empty ' + 'or None to skip setting striping.', ) _LUSTRE_MAX_DIRTY_MB = flags.DEFINE_integer( 'lustre_max_dirty_mb', @@ -373,13 +374,15 @@ def SetUpDiskOnLinux(self): ) lustre_disk.UpdateDevicePath(self.disk_spec.mount_point) self.vm.scratch_disks.append(lustre_disk) - if FLAGS.lustre_set_stripe: + stripe_args = (_LUSTRE_STRIPE_ARGS.value or '').strip() + if stripe_args: self.vm.RemoteCommand( - f'sudo lfs setstripe -c -1 {self.disk_spec.mount_point}' + f'sudo lfs setstripe {stripe_args} {self.disk_spec.mount_point}' ) - if FLAGS.lustre_max_dirty_mb: + if _LUSTRE_MAX_DIRTY_MB.value: self.vm.RemoteCommand( - f'sudo lctl set_param osc.*.max_dirty_mb={FLAGS.lustre_max_dirty_mb}', + 'sudo lctl set_param' + f' osc.*.max_dirty_mb={_LUSTRE_MAX_DIRTY_MB.value}', ignore_failure=True, ) diff --git a/perfkitbenchmarker/linux_benchmarks/io500_benchmark.py b/perfkitbenchmarker/linux_benchmarks/io500_benchmark.py index 05a286a7fe..c19d2dd52f 100644 --- a/perfkitbenchmarker/linux_benchmarks/io500_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/io500_benchmark.py @@ -31,6 +31,7 @@ from perfkitbenchmarker import linux_packages from perfkitbenchmarker import regex_util from perfkitbenchmarker import sample +from perfkitbenchmarker import virtual_machine from perfkitbenchmarker import vm_util FLAGS = flags.FLAGS @@ -78,6 +79,15 @@ 'Total mpi ranks to use. By default, same as total cores available.', ) +# TODO(andytzhu): Verify df -k -P drain behavior on NFS/NetApp Volumes before +# enabling io500_wait_for_cleanup there. +_IO500_WAIT_FOR_CLEANUP = flags.DEFINE_boolean( + 'io500_wait_for_cleanup', + False, + 'If True, poll df between iterations until used filesystem space drains ' + 'back near pre-run baseline instead of sleeping a fixed 60s.', +) + # Section names in io500.ini.j2 that can be run RUNNABLE_SECTIONS = [ 'ior_easy_write', 'ior_easy_read', @@ -125,6 +135,10 @@ # default ior-easy transfersize is 2MiB by default in io500. # https://github.com/IO500/io500/blob/main/config-sample-all.ini DEFAULT_IOR_EASY_TRANSFER_SIZE = 2 * 1024 * 1024 # 2 MiB +_CLEANUP_WAIT_TIMEOUT_SEC = 1800 +_CLEANUP_POLL_INTERVAL_SEC = 30 +_CLEANUP_TOLERANCE_RATIO = 0.005 # 0.5% of filesystem capacity +_MIN_CLEANUP_TOLERANCE_KB = 64 * 1024 # 64 MiB def GetConfig(user_config): @@ -326,6 +340,7 @@ def _Run(headnode, ranks, ppn, num_clients): 'invalid': '[INVALID]' in line, 'ior_hard_write_direct_io': _IOR_HARD_WRITE_DIRECT_IO.value, 'ior_easy_direct_io': _IOR_EASY_DIRECT_IO.value, + 'io500_wait_for_cleanup': _IO500_WAIT_FOR_CLEANUP.value, }, ) ) @@ -340,6 +355,7 @@ def _Run(headnode, ranks, ppn, num_clients): 'invalid': '[INVALID]' in line, 'ior_hard_write_direct_io': _IOR_HARD_WRITE_DIRECT_IO.value, 'ior_easy_direct_io': _IOR_EASY_DIRECT_IO.value, + 'io500_wait_for_cleanup': _IO500_WAIT_FOR_CLEANUP.value, } results.extend([ sample.Sample('BANDWIDTH', raw_result[0], raw_result[1], metadata), @@ -349,6 +365,66 @@ def _Run(headnode, ranks, ppn, num_clients): return results +class _FilesystemNotDrainedError(Exception): + """Raised while waiting for filesystem space to drain after an io500 run.""" + + +def _GetFilesystemUsageKb( + vm: virtual_machine.BaseVirtualMachine, mount_point: str +) -> tuple[int, int]: + """Returns `(used_kb, total_kb)` for `mount_point` from POSIX `df` output.""" + stdout, _ = vm.RemoteCommand(f'df -k -P {mount_point} | tail -n 1') + parts = stdout.strip().split() + if len(parts) < 5: + raise errors.Benchmarks.RunError( + f'Unexpected df output for {mount_point}: {stdout!r}' + ) + try: + total_kb, used_kb = int(parts[1]), int(parts[2]) + except ValueError as e: + raise errors.Benchmarks.RunError( + f'Failed to parse df output for {mount_point}: {stdout!r}' + ) from e + if total_kb < 0 or used_kb < 0: + raise errors.Benchmarks.RunError( + f'Negative df sizes for {mount_point}: {stdout!r}' + ) + return (used_kb, total_kb) + + +@vm_util.Retry( + poll_interval=_CLEANUP_POLL_INTERVAL_SEC, + max_retries=-1, + timeout=_CLEANUP_WAIT_TIMEOUT_SEC, + fuzz=0, + retryable_exceptions=( + _FilesystemNotDrainedError, + errors.VirtualMachine.RemoteCommandError, + ), +) +def _WaitForFilesystemDrain( + vm: virtual_machine.BaseVirtualMachine, + mount_point: str, + baseline_used_kb: int, +) -> None: + """Polls `df` until used space returns near baseline or timeout expires.""" + used_kb, total_kb = _GetFilesystemUsageKb(vm, mount_point) + threshold_kb = baseline_used_kb + max( + _MIN_CLEANUP_TOLERANCE_KB, int(total_kb * _CLEANUP_TOLERANCE_RATIO) + ) + if used_kb > threshold_kb: + raise _FilesystemNotDrainedError( + f'Filesystem {mount_point} has not drained yet ' + f'({used_kb} > {threshold_kb} KiB).' + ) + logging.info( + 'Filesystem %s drained (%d <= %d KiB).', + mount_point, + used_kb, + threshold_kb, + ) + + def Run(benchmark_spec): """Run the io500 benchmark on the vms. @@ -403,12 +479,19 @@ def Run(benchmark_spec): else [len(vms)] ) - for num_clients in client_counts: + baseline_used_kb = ( + _GetFilesystemUsageKb(headnode, headnode.scratch_disks[0].mount_point)[0] + if _IO500_WAIT_FOR_CLEANUP.value + else 0 + ) + + for client_idx, num_clients in enumerate(client_counts): active_vms = vms[:num_clients] num_nodes = len(active_vms) - for total_ranks in [int(r) for r in _MPI_RANKS_LIST.value] or [ + ranks_list = [int(r) for r in _MPI_RANKS_LIST.value] or [ num_nodes * num_cpus - ]: + ] + for rank_idx, total_ranks in enumerate(ranks_list): ppn = math.ceil(total_ranks / num_nodes) # Only list the VMs these ranks need, so 1 rank runs on the headnode only. run_vms = active_vms[: math.ceil(total_ranks / ppn)] @@ -444,7 +527,16 @@ def Run(benchmark_spec): results.extend(_Run(headnode, total_ranks, ppn, len(run_vms))) if FLAGS.object_storage_fuse_log_trace: headnode.RemoteCommand('sudo bash -c "cat /tmp/logs/*"') - time.sleep(60) + if _IO500_WAIT_FOR_CLEANUP.value: + if ( + client_idx < len(client_counts) - 1 + or rank_idx < len(ranks_list) - 1 + ): + _WaitForFilesystemDrain( + headnode, headnode.scratch_disks[0].mount_point, baseline_used_kb + ) + else: + time.sleep(60) return results diff --git a/tests/disk_strategies_test.py b/tests/disk_strategies_test.py new file mode 100644 index 0000000000..c70f311e67 --- /dev/null +++ b/tests/disk_strategies_test.py @@ -0,0 +1,101 @@ +# Copyright 2026 PerfKitBenchmarker Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock + +from absl.testing import flagsaver +from absl.testing import parameterized +from perfkitbenchmarker import configs +from perfkitbenchmarker import disk +from perfkitbenchmarker import disk_strategies +from perfkitbenchmarker import linux_virtual_machine +from perfkitbenchmarker import lustre_service +from tests import mock_command +from tests import pkb_common_test_case + + +class SetUpLustreDiskStrategyTest(pkb_common_test_case.PkbCommonTestCase): + + def _CreateLustreSetup( + self, + ) -> tuple[ + linux_virtual_machine.BaseLinuxVirtualMachine, + disk_strategies.SetUpLustreDiskStrategy, + mock_command.MockRemoteCommand, + ]: + vm = mock.create_autospec( + linux_virtual_machine.BaseLinuxVirtualMachine, + instance=True, + ) + vm.scratch_disks = [] + mock_cmd = mock_command.MockRemoteCommand({}, vm) + disk_spec = disk.BaseLustreDiskSpec( + 'test_lustre', + mount_point='/lustre_dir', + disk_type=disk.LUSTRE, + ) + service = mock.create_autospec( + lustre_service.BaseLustreService, + instance=True, + spec_set=True, + ) + service.CreateLustreDisk.return_value = disk.LustreDisk(disk_spec) + service.GetMountPoint.return_value = '10.0.0.1@tcp:/lustre' + strategy = disk_strategies.SetUpLustreDiskStrategy(vm, disk_spec, service) + return vm, strategy, mock_cmd + + def testDefaultLustreStripeArgsSkipsSetstripe(self): + vm, strategy, mock_cmd = self._CreateLustreSetup() + + strategy.SetUpDiskOnLinux() + + self.assertLen(vm.scratch_disks, 1) + self.assertEqual(mock_cmd.all_commands, '') + + @parameterized.named_parameters( + dict(testcase_name='empty', stripe_args=''), + dict(testcase_name='whitespace', stripe_args=' '), + ) + def testBlankLustreStripeArgsSkipsSetstripe(self, stripe_args: str): + vm, strategy, mock_cmd = self._CreateLustreSetup() + + with flagsaver.flagsaver(lustre_stripe_args=stripe_args): + strategy.SetUpDiskOnLinux() + + self.assertLen(vm.scratch_disks, 1) + self.assertEqual(mock_cmd.all_commands, '') + + @flagsaver.flagsaver(lustre_stripe_args='-E 1M -c 1 -E -1 -c -1') + def testConfiguredLustreStripeArgsRunsSetstripe(self): + _, strategy, mock_cmd = self._CreateLustreSetup() + + strategy.SetUpDiskOnLinux() + + self.assertEqual( + mock_cmd.all_commands, + 'sudo lfs setstripe -E 1M -c 1 -E -1 -c -1 /lustre_dir\n', + ) + + def testDefaultBenchmarkConfigIo500LustreFlags(self): + io500_lustre_config = configs.GetDefaultAndUserConfig()['io500_lustre'] + flags_dict = io500_lustre_config['flags'] + + self.assertEqual(flags_dict['lustre_stripe_args'], '-E 1M -c 1 -E -1 -c -1') + self.assertTrue(flags_dict['io500_wait_for_cleanup']) + self.assertEqual(flags_dict['lustre_max_dirty_mb'], 2000) + + +if __name__ == '__main__': + unittest.main()