diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 48c927e7ac33..9c439bc2b0f6 100644 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -1642,6 +1642,9 @@ def zero_allgather_bucket_size(self): def zero_compute_grad_norm(self): return self._config.zero_config.compute_grad_norm + def zero_retain_gpu_grad_partition_until_step(self): + return self._config.zero_config.retain_gpu_grad_partition_until_step + def zero_optimization_partition_gradients(self): return self.zero_optimization_stage() >= ZeroStageEnum.gradients @@ -2733,7 +2736,8 @@ def _configure_zero_optimizer(self, optimizer): communication_data_type=self.communication_data_type, elastic_checkpoint=self.zero_elastic_checkpoint(), check_grad_overflow=check_grad_overflow, - compute_grad_norm=self.zero_compute_grad_norm()) + compute_grad_norm=self.zero_compute_grad_norm(), + retain_gpu_grad_partition_until_step=self.zero_retain_gpu_grad_partition_until_step()) elif zero_stage == ZeroStageEnum.weights: self._validate_zero3_moe_compatibility() diff --git a/deepspeed/runtime/zero/config.py b/deepspeed/runtime/zero/config.py index ac2ff3c40e29..72791ecc6ba9 100644 --- a/deepspeed/runtime/zero/config.py +++ b/deepspeed/runtime/zero/config.py @@ -147,6 +147,16 @@ class DeepSpeedZeroConfig(DeepSpeedConfigModel): and callers do not use ``get_global_grad_norm()``. """ + retain_gpu_grad_partition_until_step: bool = False + """ + With ZeRO-2 CPU optimizer offload and fp32 gradient accumulation on a 16-bit model, the fp32 + gradient partition is freed from GPU memory after each request boundary and restored from the host + gradient for the next backward. This is the default and keeps GPU memory use between requests low. + Set to ``true`` to keep the partition in GPU memory until ``step()``: GPU memory use between + requests rises by one fp32 partition per rank, and the per-request host copy and restore are avoided. + Has no effect on configurations that do not use the compact gradient partition. + """ + load_from_fp32_weights: bool = Field(True, json_schema_extra={"deprecated": True}) """ Deprecated and ignored by ``DeepSpeedEngine.load_checkpoint``. Full ZeRO diff --git a/deepspeed/runtime/zero/stage_1_and_2.py b/deepspeed/runtime/zero/stage_1_and_2.py index 371777362788..c7309d0229e0 100644 --- a/deepspeed/runtime/zero/stage_1_and_2.py +++ b/deepspeed/runtime/zero/stage_1_and_2.py @@ -205,10 +205,12 @@ def __init__(self, bf16_optimizer_states=False, elastic_checkpoint=False, check_grad_overflow=True, - compute_grad_norm=True): + compute_grad_norm=True, + retain_gpu_grad_partition_until_step=False): super().__init__() + self.retain_gpu_grad_partition_until_step = retain_gpu_grad_partition_until_step if not compute_grad_norm and clip_grad > 0.0: raise ValueError("zero_optimization.compute_grad_norm=false requires gradient_clipping=0") if not compute_grad_norm and zenflow_config is not None: @@ -653,6 +655,10 @@ def _enforce_cpu_offload(): self._offload_accumulated_param_ids = set() # FP32-master gradients first seen at a boundary stay in the partition until another backward. self._offload_partition_only_param_ids = set() + # Host sums updated by a non-boundary backward since they were last published to the FP32 partition. + self._offload_unpublished_param_ids = set() + # Compact partitions freed at a boundary; the host gradient holds their running sum. + self._host_owned_compact_partitions = set() self.temp_grad_buffer_for_cpu_offload = torch.zeros(largest_param_numel, device=self.device, dtype=self.dtype) @@ -1063,6 +1069,9 @@ def independent_gradient_partition_epilogue(self): # sum until step(): more request boundaries may precede the update. if self.use_grad_accum_attribute: self.all_grad_tensors[i] = None + elif self.is_gradient_accumulation_boundary(): + self._publish_unpublished_offload_grads() + self._offload_compact_grad_partitions() self._release_ipg_buffers() @@ -1110,10 +1119,34 @@ def _finalize_cpu_offload_gradient_accumulation(self): if self._use_compact_grad_partition_buffer(group_idx): self.copy_grads_in_partition(param, group_idx) continue - self._restore_cpu_offload_grad_to_gpu(param) - self.set_norm_for_param_grad_in_gpu(param) - self.update_offload_overflow_tracker_for_param_grad(param) - self.async_inplace_copy_grad_to_fp32_buffer_from_gpu(param) + self._publish_host_accumulated_grad(param) + self._offload_unpublished_param_ids.clear() + self._offload_compact_grad_partitions() + + def _publish_host_accumulated_grad(self, param): + """Read this parameter's host sum back to the device, then into the FP32 partition. + + The device pass lets the shared norm and overflow helpers see the full accumulated gradient. + """ + self._restore_cpu_offload_grad_to_gpu(param) + self.set_norm_for_param_grad_in_gpu(param) + self.update_offload_overflow_tracker_for_param_grad(param) + self.async_inplace_copy_grad_to_fp32_buffer_from_gpu(param) + + def _publish_unpublished_offload_grads(self): + """Bring every host sum a non-boundary backward left behind into the FP32 partition. + + A boundary backward publishes only the parameters it reduced. A parameter whose last + contribution this window came from a non-boundary backward would otherwise reach step(), + safe_get_full_grad() and the grad norm as zero. + """ + for group_idx, group in enumerate(self.params_in_partition): + if self._use_compact_grad_partition_buffer(group_idx): + continue + for param in group: + if self.get_param_id(param) in self._offload_unpublished_param_ids: + self._publish_host_accumulated_grad(param) + self._offload_unpublished_param_ids.clear() def _restore_cpu_offload_grad_to_gpu(self, param): # Last non-boundary epilogue cleared param.grad; reload accumulated CPU grads for boundary helpers. @@ -1264,14 +1297,22 @@ def get_param_gradient_attribute(self, param): return param.grad_accum if self.use_grad_accum_attribute else param.grad def _use_compact_grad_partition_buffer(self, group_idx): - gradient_accumulation_dtype = getattr(self, "gradient_accumulation_dtype", None) - if gradient_accumulation_dtype != torch.float32 or gradient_accumulation_dtype == self.dtype: - return False params_in_partition = self.params_in_partition[group_idx] if len(params_in_partition) > 0 and self._is_muon_group(params_in_partition): return False + gradient_accumulation_dtype = getattr(self, "gradient_accumulation_dtype", None) + if gradient_accumulation_dtype != torch.float32 or gradient_accumulation_dtype == self.dtype: + return False return True + @staticmethod + def _host_grad_holds_accum_exactly(host_dtype, accum_dtype): + # The host gradient may be restored into the device buffer only if no bits are lost: + # same dtype, or an fp32 host holding bf16/fp16 values that were widened from it. + if host_dtype == accum_dtype: + return True + return host_dtype == torch.float32 and accum_dtype in (torch.bfloat16, torch.float16) + def _build_compact_grad_partition_buffer(self, group_idx): partition_size = int(self.partition_size[group_idx]) flat_partition = torch.zeros(partition_size, @@ -1313,9 +1354,68 @@ def _get_compact_grad_partition_buffer(self, group_idx): return grad_group grad_group = self._build_compact_grad_partition_buffer(group_idx) + if self.cpu_offload and group_idx in self._host_owned_compact_partitions: + self._restore_compact_grad_partition(group_idx, grad_group) self.averaged_gradients[group_idx] = grad_group return grad_group + def _offload_compact_grad_partitions(self): + """Copy each compact partition's running sum to the host gradient at a boundary. + + Every parameter accumulated in this window is copied, including one that missed the + boundary backward. Once the host holds the exact sum, the GPU partition is freed so it + does not occupy GPU memory between requests; a later backward in the window restores it. + """ + stream = get_accelerator().current_stream() + if self.overlap_comm and self.reduction_stream is not None: + stream.wait_stream(self.reduction_stream) + for group_idx in range(len(self.bit16_groups)): + grad_group = self.averaged_gradients.get(group_idx) + if grad_group is None or not self._use_compact_grad_partition_buffer(group_idx): + continue + partition_id = dist.get_rank(group=self.real_dp_process_group[group_idx]) + host_grad = self.single_partition_of_fp32_groups[group_idx].grad.view(-1) + for param in self.params_in_partition[group_idx]: + param_id = self.get_param_id(param) + if param_id not in self._offload_accumulated_param_ids: + continue + [_, _, dest_offset, num_elements] = self.grad_position[param_id] + source = self._get_compact_grad_partition_slice(group_idx, partition_id, param_id, num_elements) + self.norm_for_param_grads[param_id] = source.data.to(get_norm_dtype()).norm(2) + # Check the value the optimizer will read: an fp32 sum can pass here and become inf in an + # fp16 host gradient. + host_value = source.to(host_grad.dtype) + self.update_offload_overflow_tracker(host_value) + host_grad.narrow(0, dest_offset, num_elements).copy_(host_value, non_blocking=True) + self._record_offload_copy() + + # Free only when the host copy can restore the device sum bit for bit. Otherwise the + # device buffer stays resident until step(), and ZenFlow orders its own copies. + if not self._host_grad_holds_accum_exactly(host_grad.dtype, grad_group.flat_partition.dtype): + continue + if not self._offload_gradient_safety_enabled: + continue + if self.retain_gpu_grad_partition_until_step: + continue + self._record_gradient_stream(grad_group.flat_partition, stream) + self.averaged_gradients[group_idx] = None + self._host_owned_compact_partitions.add(group_idx) + + def _restore_compact_grad_partition(self, group_idx, grad_group): + self._host_owned_compact_partitions.discard(group_idx) + # The restore may run on another stream than the boundary copy that filled the host gradient. + self._wait_for_offload_copies() + partition_id = dist.get_rank(group=self.real_dp_process_group[group_idx]) + host_grad = self.single_partition_of_fp32_groups[group_idx].grad.view(-1) + for param in self.params_in_partition[group_idx]: + param_id = self.get_param_id(param) + if param_id not in self._offload_accumulated_param_ids: + continue + [_, _, dest_offset, num_elements] = self.grad_position[param_id] + insertion_offset = int(self.grad_partition_insertion_offset[group_idx][partition_id][param_id]) + destination = grad_group.flat_partition.narrow(0, insertion_offset, num_elements) + destination.copy_(host_grad.narrow(0, dest_offset, num_elements), non_blocking=True) + def _get_compact_grad_partition_slice(self, group_idx, partition_id, param_id, num_elements): if not self._use_compact_grad_partition_buffer(group_idx): return None @@ -2126,19 +2226,9 @@ def copy_grads_in_partition(self, param, group_idx=None): if group_idx is None: group_idx = self._get_param_group_idx(param) if self.cpu_offload and self._use_compact_grad_partition_buffer(group_idx): - param_id = self.get_param_id(param) - self._offload_accumulated_param_ids.add(param_id) + # The boundary copy to the host runs per group, in _offload_compact_grad_partitions(). + self._offload_accumulated_param_ids.add(self.get_param_id(param)) if self.is_gradient_accumulation_boundary(): - [_, _, dest_offset, num_elements] = self.grad_position[param_id] - partition_id = dist.get_rank(group=self.real_dp_process_group[group_idx]) - source = self._get_compact_grad_partition_slice(group_idx, partition_id, param_id, num_elements) - self.norm_for_param_grads[param_id] = source.data.to(get_norm_dtype()).norm(2) - self.update_offload_overflow_tracker(source) - destination = self.single_partition_of_fp32_groups[group_idx].grad.view(-1).narrow( - 0, dest_offset, num_elements) - destination.copy_(source.to(destination.dtype), non_blocking=True) - self._record_gradient_stream(source, get_accelerator().current_stream()) - self._record_offload_copy() self.clear_grad_attribute(param) return if self.cpu_offload: @@ -2158,6 +2248,9 @@ def copy_grads_in_partition(self, param, group_idx=None): self.update_offload_overflow_tracker_for_param_grad(param) self.async_inplace_copy_grad_to_fp32_buffer_from_gpu(param) + self._offload_unpublished_param_ids.discard(param_id) + else: + self._offload_unpublished_param_ids.add(param_id) # Include the shared-buffer fast path in subsequent accumulation and finalization. self._offload_accumulated_param_ids.add(param_id) @@ -2916,6 +3009,7 @@ def reset_cpu_buffers(self): self._wait_for_offload_copies() self.norm_for_param_grads = {} self.local_overflow = False + self._host_owned_compact_partitions.clear() # Adam consumes the partition gradient; a parameter idle in the next window must not reuse it. for fp32_partition in self.single_partition_of_fp32_groups: if fp32_partition.grad is not None: @@ -3028,6 +3122,7 @@ def step(self, closure=None): if self.cpu_offload: self._offload_accumulated_param_ids = set() self._offload_partition_only_param_ids = set() + self._offload_unpublished_param_ids = set() if not self.compute_grad_norm: self._global_grad_norm = None @@ -3065,6 +3160,9 @@ def step(self, closure=None): self._release_preflattened_grad_buffers() if self.cpu_offload: self.reset_cpu_buffers() + # A resident compact partition still holds the skipped window's inf/NaN sum. + for k in self.averaged_gradients.keys(): + self.averaged_gradients[k] = None else: for k in self.averaged_gradients.keys(): self.averaged_gradients[k] = None diff --git a/tests/unit/v1/zero/test_zero2_gradient_safety.py b/tests/unit/v1/zero/test_zero2_gradient_safety.py index 2089022539f4..9665c91e0b39 100644 --- a/tests/unit/v1/zero/test_zero2_gradient_safety.py +++ b/tests/unit/v1/zero/test_zero2_gradient_safety.py @@ -30,6 +30,8 @@ def make_optimizer(dtype=torch.bfloat16, device=None, low_precision=False, cpu_o opt = zero.DeepSpeedZeroOptimizer.__new__(zero.DeepSpeedZeroOptimizer) opt.cpu_offload = cpu_offload opt._offload_gradient_safety_enabled = cpu_offload + opt._host_owned_compact_partitions = set() + opt._offload_unpublished_param_ids = set() opt.cpu_offload_pin_memory = False opt.device = "cpu" opt.dtype = dtype @@ -40,6 +42,10 @@ def make_optimizer(dtype=torch.bfloat16, device=None, low_precision=False, cpu_o opt.contiguous_gradients = True opt.overlap_comm = False opt.zenflow = False + # These stubs drive copy_grads_in_partition directly and skip the reduction that fills the + # compact partition buffer. Pin them to the host accumulator path, which Muon and ZenFlow still + # use; the compact layout is covered by the engine tests in test_zero2_offload_multi_backward.py. + opt._use_compact_grad_partition_buffer = lambda group_idx: False opt.has_moe_layers = False opt.autoep_folding_tp_group = None opt.gradient_accumulation_steps = 1 diff --git a/tests/unit/v1/zero/test_zero2_offload_multi_backward.py b/tests/unit/v1/zero/test_zero2_offload_multi_backward.py index 0f513e5587fc..c06fe3757926 100644 --- a/tests/unit/v1/zero/test_zero2_offload_multi_backward.py +++ b/tests/unit/v1/zero/test_zero2_offload_multi_backward.py @@ -167,7 +167,13 @@ def forward(self, inputs, projection_indices=(0, 1)): return loss -def _run_request_boundaries(rank, rendezvous, zero_stage, cpu_offload, accumulation_dtype, checkpointed): +def _run_request_boundaries(rank, + rendezvous, + zero_stage, + cpu_offload, + accumulation_dtype, + checkpointed, + host_accumulator=False): os.environ["LOCAL_RANK"] = str(rank) accelerator = get_accelerator() accelerator.set_device(rank) @@ -215,6 +221,9 @@ def _run_request_boundaries(rank, rendezvous, zero_stage, cpu_offload, accumulat optimizer=torch.optim.SGD(model.parameters(), lr=1 / 32), config=config, dist_init_required=False) + if host_accumulator: + # Muon groups accumulate in the host accumulator instead of the compact partition buffer. + engine.optimizer._use_compact_grad_partition_buffer = lambda group_idx: False calls = ["c0", "c1", "c2", "c3"] windows = [(requests, {}) for requests in [[calls], [["ping"], calls], [["c0"], ["c1"]], [["c0"], ["c1"], ["c2", "c3"]], @@ -222,6 +231,10 @@ def _run_request_boundaries(rank, rendezvous, zero_stage, cpu_offload, accumulat # Each projection first joins a later backward after contributing to the preceding update. for early, late in [(0, 1), (1, 0)]: windows.append(([["c0"], ["c1", "c2"], ["c3"]], {"c0": (early, ), "c1": (late, ), "c2": (late, )})) + # A projection can contribute to a request without joining its boundary backward. + for early, late in [(0, 1), (1, 0)]: + windows.append(([["c0", "c1"], ["c2"]], {"c0": (early, ), "c1": (late, )})) + windows.append(([["c0", "c1"]], {"c0": (early, ), "c1": (late, )})) reference_names = list(references) for step, (requests, selections) in enumerate(windows): expected = {name: torch.zeros_like(reference) for name, reference in references.items()} @@ -278,12 +291,70 @@ def test_multiple_request_boundaries_before_step(tmp_path, zero_stage, cpu_offlo start_method="forkserver") +@pytest.mark.skipif(get_accelerator().device_name() not in ("cpu", "cuda") + or (get_accelerator().device_name() == "cuda" and get_accelerator().device_count() < 2), + reason="Requires CPU or two CUDA devices") +@pytest.mark.parametrize("accumulation_dtype", ["bf16", "fp32"]) +@pytest.mark.parametrize("checkpointed", [False, True], ids=["plain", "reentrant"]) +def test_multiple_request_boundaries_host_accumulator(tmp_path, accumulation_dtype, checkpointed): + # The host-accumulator path is what Muon groups use under stage-2 CPU offload. + torch.multiprocessing.start_processes(_run_request_boundaries, + args=(f"file://{tmp_path / 'rendezvous'}", 2, True, accumulation_dtype, + checkpointed, True), + nprocs=2, + join=True, + start_method="forkserver") + + class TestZero2OffloadOverflowRecovery(DistributedTest): world_size = 2 + @pytest.mark.parametrize("accumulation_dtype", [None, "fp32"]) + def test_fp16_host_gradient_overflow_skips_step(self, accumulation_dtype): + """A sum that exceeds the fp16 range must be caught when it reaches the fp16 host gradient. + + With fp32 accumulation the overflow appears only in the cast to the fp16 master gradient. + """ + if not deepspeed.ops.__compatible_ops__[CPUAdamBuilder.NAME]: + pytest.skip("CPUAdam is required for low-precision offloaded master weights") + if not get_accelerator().is_fp16_supported(): + pytest.skip("FP16 is not supported") + + class Scaled(torch.nn.Module): + + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(16)) + + def forward(self, scale): + return (self.weight * scale).sum() + + config = _base_config(2, cpu_offload=True) + config.pop("bf16", None) + config["fp16"] = {"enabled": True, "loss_scale": 1.0, "fp16_master_weights_and_grads": True} + if accumulation_dtype is not None: + config["data_types"] = {"grad_accum_dtype": accumulation_dtype} + config["train_batch_size"] = 2 + model = Scaled() + engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=config) + try: + initial = _capture_params(engine) + # Each backward adds 40000 per element: finite in fp16 alone, 80000 (> 65504) in the sum. + for i in range(2): + engine.set_gradient_accumulation_boundary(i == 1) + engine.backward(engine(torch.tensor(40000., device=engine.device))) + engine.step() + after = _capture_params(engine) + for name in initial: + assert torch.isfinite(after[name]).all(), f"{name} became non-finite" + _assert_params_match(initial, after, "overflowed sum must skip the update", tol=1e-8) + finally: + engine.destroy() + @pytest.mark.parametrize("precision", ["bf16", "fp16"]) @pytest.mark.parametrize("low_precision_master", [True, False]) - def test_finite_step_after_overflow(self, precision, low_precision_master): + @pytest.mark.parametrize("accumulation_dtype", [None, "fp32"]) + def test_finite_step_after_overflow(self, precision, low_precision_master, accumulation_dtype): if not deepspeed.ops.__compatible_ops__[CPUAdamBuilder.NAME]: pytest.skip("CPUAdam is required for low-precision offloaded master weights") if precision == "bf16" and not get_accelerator().is_bf16_supported(): @@ -307,6 +378,8 @@ def forward(self, overflow): config[precision] = {"enabled": True, f"{precision}_master_weights_and_grads": low_precision_master} if precision == "fp16": config[precision]["loss_scale"] = 1.0 + if accumulation_dtype is not None: + config["data_types"] = {"grad_accum_dtype": accumulation_dtype} config["train_batch_size"] = 2 config["managed_gradient_accumulation"] = False config["optimizer"]["params"]["lr"] = 0.1 diff --git a/tests/unit/v1/zero/test_zero_cpu_offload_grad_accum.py b/tests/unit/v1/zero/test_zero_cpu_offload_grad_accum.py index aa27396fda81..f4311e0c91b9 100644 --- a/tests/unit/v1/zero/test_zero_cpu_offload_grad_accum.py +++ b/tests/unit/v1/zero/test_zero_cpu_offload_grad_accum.py @@ -11,6 +11,7 @@ underestimated. """ +import pytest import torch import deepspeed @@ -19,18 +20,30 @@ from deepspeed.accelerator import get_accelerator -def _cpu_grad_norm(engine): +def _running_grad_norm(engine): + # The running sum lives in the device partition buffer until a boundary copies it to the host and + # frees the buffer; after that the host gradient holds it. Groups on the legacy path (Muon, ZenFlow) + # use the host accumulator. + optimizer = engine.optimizer total_norm_sq = 0.0 - for grad in engine.optimizer.accumulated_grads_in_cpu: - if grad is not None: - total_norm_sq += grad.float().norm(2).item()**2 + for group_idx in range(len(optimizer.bit16_groups)): + grad_group = optimizer.averaged_gradients.get(group_idx) + flat_partition = getattr(grad_group, "flat_partition", None) + if flat_partition is not None: + total_norm_sq += flat_partition.float().norm(2).item()**2 + elif group_idx in optimizer._host_owned_compact_partitions: + host_grad = optimizer.single_partition_of_fp32_groups[group_idx].grad + total_norm_sq += host_grad.float().norm(2).item()**2 + elif optimizer.accumulated_grads_in_cpu[group_idx] is not None: + total_norm_sq += optimizer.accumulated_grads_in_cpu[group_idx].float().norm(2).item()**2 return total_norm_sq**0.5 class TestZero2CPUOffloadGradAccumNorm(DistributedTest): world_size = 1 - def test(self): + @pytest.mark.parametrize("accumulation_dtype", [None, "fp32"]) + def test(self, accumulation_dtype): gradient_accumulation_steps = 4 hidden_dim = 10 @@ -54,6 +67,9 @@ def test(self): }, } + if accumulation_dtype is not None: + config_dict["data_types"] = {"grad_accum_dtype": accumulation_dtype} + if get_accelerator().is_bf16_supported(): config_dict["bf16"] = {"enabled": True} elif get_accelerator().is_fp16_supported(): @@ -79,12 +95,12 @@ def test(self): for batch in data_loader: loss = model(batch[0], batch[1]) model.backward(loss) - norms.append(_cpu_grad_norm(model)) + norms.append(_running_grad_norm(model)) model.destroy() - assert norms[0] > 0, "accumulated_grads_in_cpu should be non-zero after first backward" + assert all(norm > 0 for norm in norms), f"running gradient sum should stay non-zero: {norms}" for i in range(1, len(norms)): - assert norms[i] != norms[0], (f"accumulated_grads_in_cpu norm did not change after micro-step {i}: " + assert norms[i] != norms[0], (f"running gradient sum norm did not change after micro-step {i}: " f"norm[0]={norms[0]:.6f}, norm[{i}]={norms[i]:.6f}. " - "Gradients were not copied back to CPU (PR #7967 regression).") + "Gradients were not accumulated (PR #7967 regression).") diff --git a/tests/unit/v1/zero/test_zero_offload_accum_precision.py b/tests/unit/v1/zero/test_zero_offload_accum_precision.py new file mode 100644 index 000000000000..f54ace64343d --- /dev/null +++ b/tests/unit/v1/zero/test_zero_offload_accum_precision.py @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: Apache-2.0 +# DeepSpeed Team +""" +Accumulation precision of ZeRO-1/2 CPU offload across one optimizer window. + +Each micro-step's gradient is exactly representable in bf16 (values (128 + k) / 128), so the +exact window sum is known. The only rounding comes from the accumulation dtype and the host +copy, which makes the error of each (stage, master dtype, accumulation dtype) cell directly +comparable. +""" + +import os + +import pytest +import torch + +import deepspeed +from deepspeed.utils import safe_get_full_grad + +from unit.common import DistributedTest +from deepspeed.accelerator import get_accelerator + +MICRO_STEPS = 16 +NUMEL = 16 + + +class Weighted(torch.nn.Module): + + def __init__(self, numel): + super().__init__() + self.w = torch.nn.Parameter(torch.zeros(numel)) + + def forward(self, c): + return (self.w * c).sum() + + +def _config(stage, low_precision_master, accumulation_dtype): + config = { + "train_batch_size": 1, + "gradient_accumulation_steps": 1, + "train_micro_batch_size_per_gpu": 1, + "gradient_clipping": 0, + "zero_allow_untested_optimizer": True, + "zero_force_ds_cpu_optimizer": False, + "optimizer": { + "type": "Adam", + "params": { + "lr": 1e-3 + } + }, + "zero_optimization": { + "stage": stage, + "offload_optimizer": { + "device": "cpu", + "pin_memory": False + }, + }, + "bf16": { + "enabled": True, + "bf16_master_weights_and_grads": low_precision_master, + }, + } + if accumulation_dtype is not None: + config["data_types"] = {"grad_accum_dtype": accumulation_dtype} + return config + + +def _window_error(stage, low_precision_master, accumulation_dtype): + """Relative max error of the window's gradient sum against the exact sum, read before step().""" + torch.manual_seed(0) + model = Weighted(NUMEL) + engine, _, _, _ = deepspeed.initialize(model=model, + model_parameters=model.parameters(), + config=_config(stage, low_precision_master, accumulation_dtype)) + try: + generator = torch.Generator().manual_seed(1234) + exact = torch.zeros(NUMEL, dtype=torch.float64) + for step in range(MICRO_STEPS): + k = torch.randint(0, 128, (NUMEL, ), generator=generator) + c = ((128 + k).float() / 128).to(device=engine.device, dtype=torch.bfloat16) + exact += c.double().cpu() + engine.set_gradient_accumulation_boundary(step == MICRO_STEPS - 1) + engine.backward(engine(c)) + grad = safe_get_full_grad(engine.module.w).detach().float().cpu().double() + return ((grad - exact).abs().max() / exact.abs().max()).item() + finally: + engine.destroy() + + +@pytest.mark.parametrize("stage", [1, 2]) +@pytest.mark.parametrize("low_precision_master", [False, True], ids=["fp32-master", "bf16-master"]) +@pytest.mark.parametrize("accumulation_dtype", [None, "fp32"], ids=["bf16-accum", "fp32-accum"]) +class TestZeroOffloadAccumPrecision(DistributedTest): + world_size = 1 + + def test_window_sum(self, stage, low_precision_master, accumulation_dtype): + if not deepspeed.ops.__compatible_ops__[deepspeed.ops.op_builder.CPUAdamBuilder.NAME]: + pytest.skip("CPUAdam is required for offloaded low-precision masters") + if not get_accelerator().is_bf16_supported(): + pytest.skip("BF16 is not supported") + error = _window_error(stage, low_precision_master, accumulation_dtype) + out = os.environ.get("ACCUM_PRECISION_OUT") + if out: + with open(out, "a") as f: + f.write(f"stage={stage} master={'bf16' if low_precision_master else 'fp32'} " + f"accum={accumulation_dtype or 'bf16'} rel_max_err={error:.3e}\n") + if stage == 2 and not low_precision_master and accumulation_dtype == "fp32": + # The supported case: the window sum is exact. Stage 1 rounds its window sum to the + # 16-bit reduction dtype once (same error on the base commit); it is characterized, not required. + assert error == 0.0, f"fp32 accumulation with fp32 masters lost precision: {error:.3e}" + + +def _set_grad_error(scenario, stage=2): + """Drive safe_set_full_grad between two backwards in one window and return the window sum read at its end. + + Returns the value written by safe_get_full_grad minus the sum the caller would expect if the set value + were kept and accumulated into: scenario "mid_window" sets while the GPU partition is resident (after the + first backward of a window); "after_boundary" sets after a boundary freed the partition. + """ + from deepspeed.utils import safe_set_full_grad + torch.manual_seed(0) + model = Weighted(NUMEL) + engine, _, _, _ = deepspeed.initialize(model=model, + model_parameters=model.parameters(), + config=_config(stage, False, "fp32")) + try: + ones = torch.ones(NUMEL, device=engine.device, dtype=torch.bfloat16) + c1 = (torch.arange(NUMEL, device=engine.device) % 7 + 128).float().div(128).to(torch.bfloat16) + c2 = (torch.arange(NUMEL, device=engine.device) % 5 + 129).float().div(128).to(torch.bfloat16) + value = torch.full((NUMEL, ), 0.5, device=engine.device, dtype=torch.float32) + if scenario == "mid_window": + engine.set_gradient_accumulation_boundary(False) + engine.backward(engine(c1)) + safe_set_full_grad(engine.module.w, value) + engine.set_gradient_accumulation_boundary(True) + engine.backward(engine(c2)) + expected = value + c2.float() + else: + engine.set_gradient_accumulation_boundary(True) + engine.backward(engine(c1)) + safe_set_full_grad(engine.module.w, value) + engine.set_gradient_accumulation_boundary(True) + engine.backward(engine(c2)) + expected = value + c2.float() + grad = safe_get_full_grad(engine.module.w).detach().float().cpu() + return (grad - expected.cpu()).abs().max().item(), (grad - (c1 + c2).float().cpu()).abs().max().item() + finally: + engine.destroy() + + +class TestZeroOffloadSetFullGrad(DistributedTest): + world_size = 1 + + @pytest.mark.parametrize("scenario", ["mid_window", "after_boundary"]) + def test_set_full_grad_between_backwards(self, scenario): + if not deepspeed.ops.__compatible_ops__[deepspeed.ops.op_builder.CPUAdamBuilder.NAME]: + pytest.skip("CPUAdam is required for offloaded low-precision masters") + err_expected, err_overwritten = _set_grad_error(scenario) + out = os.environ.get("ACCUM_PRECISION_OUT") + if out: + with open(out, "a") as f: + f.write(f"set_full_grad scenario={scenario} err_if_kept={err_expected:.3e} " + f"err_if_overwritten={err_overwritten:.3e}\n") + if scenario == "after_boundary": + assert err_expected < 1e-6, "a value set after a freed boundary must be kept and accumulated" + else: + # Characterizes the resident case: the boundary overwrites the set value (not kept). + assert err_overwritten < 1e-2, "mid-window set is no longer overwritten at the boundary" + + +def _retain_option_run(retain): + """Three request-boundary backwards and one step; returns residency per boundary, grads, and final weight.""" + torch.manual_seed(0) + model = Weighted(NUMEL) + config = _config(2, False, "fp32") + config["zero_optimization"]["retain_gpu_grad_partition_until_step"] = retain + engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=config) + try: + generator = torch.Generator().manual_seed(99) + exact = torch.zeros(NUMEL, dtype=torch.float64) + resident = [] # True while the GPU partition is held + for _ in range(3): + k = torch.randint(0, 128, (NUMEL, ), generator=generator) + c = ((128 + k).float() / 128).to(device=engine.device, dtype=torch.bfloat16) + exact += c.double().cpu() + engine.set_gradient_accumulation_boundary(True) + engine.backward(engine(c)) + resident.append(engine.optimizer.averaged_gradients.get(0) is not None) + grad = safe_get_full_grad(engine.module.w).detach().float().cpu().double() + engine.step() + weight = engine.module.w.detach().float().cpu().clone() + return resident, (grad - exact).abs().max().item(), weight + finally: + engine.destroy() + + +class TestZeroOffloadReleaseOption(DistributedTest): + world_size = 1 + + def test_release_option_controls_residency_without_changing_results(self): + if not deepspeed.ops.__compatible_ops__[deepspeed.ops.op_builder.CPUAdamBuilder.NAME]: + pytest.skip("CPUAdam is required for offloaded low-precision masters") + resident_released, err_released, w_released = _retain_option_run(False) + resident_kept, err_kept, w_kept = _retain_option_run(True) + # `resident` is True while the GPU partition is held. With the option on, every boundary frees it. + assert resident_released == [False, False, + False], f"partition stayed resident with the option on: {resident_released}" + assert resident_kept == [True, True, True], f"partition was freed with the option off: {resident_kept}" + assert err_kept == 0.0, f"resident path lost precision: {err_kept:.3e}" + assert err_released == 0.0, f"released path lost precision: {err_released:.3e}" + assert torch.equal(w_released, w_kept), "release option changed the optimizer result"