Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions deepspeed/runtime/zero/stage_1_and_2.py
Original file line number Diff line number Diff line change
Expand Up @@ -1789,8 +1789,13 @@ def get_grad_position(self, group_id, tensor_list, first_offset, partition_size)
]

def update_offload_overflow_tracker(self, grad):
if grad is not None and self._has_inf_or_nan(grad.data):
self.local_overflow = True
# Only has_overflow() reads this flag, and only when check_grad_overflow is set. has_inf_or_nan()
# returns a device tensor; OR it into a device flag rather than testing it here, where a Python
# ``if`` would synchronize the host once per gradient.
if grad is None or not self.check_grad_overflow:
return
overflow = self._has_inf_or_nan(grad.data).to(get_accelerator().current_device_name(), non_blocking=True)
self.local_overflow = overflow if self.local_overflow is False else self.local_overflow | overflow

def update_offload_overflow_tracker_for_param_grad(self, param):
grad_accum = self.get_param_gradient_attribute(param)
Expand Down Expand Up @@ -3235,9 +3240,12 @@ def has_overflow_partitioned_grads_serial(self):
return invalid_grad_count.bool()

def has_overflow(self, partition_gradients=True):
overflow = self.local_overflow if self.cpu_offload else self.has_overflow_partitioned_grads_serial()
overflow_gpu = get_accelerator().ByteTensor([overflow]) if self.cpu_offload else overflow.byte().to(
get_accelerator().current_device_name())
device = get_accelerator().current_device_name()
if self.cpu_offload:
# False, or the device flag accumulated by update_offload_overflow_tracker().
overflow_gpu = torch.as_tensor(self.local_overflow, device=device).to(torch.uint8).reshape(1)
else:
overflow_gpu = self.has_overflow_partitioned_grads_serial().byte().to(device)

dist.all_reduce(overflow_gpu, op=dist.ReduceOp.MAX, group=self.dp_process_group)

Expand Down
122 changes: 122 additions & 0 deletions tests/unit/v1/zero/test_offload_overflow_flag.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: Apache-2.0

# DeepSpeed Team

import pytest
import torch

import deepspeed
from deepspeed.accelerator import get_accelerator
from deepspeed.runtime.zero.stage_1_and_2 import DeepSpeedZeroOptimizer
from deepspeed.utils import safe_set_full_grad
from unit.common import DistributedTest
from unit.simple_model import SimpleModel, random_dataloader


class _NoTruthTest(torch.Tensor):
"""A flag that fails if anything tests its truth value, which would synchronize the host."""

def __bool__(self):
raise AssertionError("the offload overflow flag was truth-tested")


def _offload_engine(zero_stage, check_grad_overflow):
config = {
"train_micro_batch_size_per_gpu": 1,
"steps_per_print": 1,
"optimizer": {
"type": "Adam",
"params": {
"lr": 1e-4,
"torch_adam": True
}
},
"bf16": {
"enabled": True,
"check_grad_overflow": check_grad_overflow
},
"zero_optimization": {
"stage": zero_stage,
"offload_optimizer": {
"device": "cpu"
}
},
}
model = SimpleModel(10)
engine, _, _, _ = deepspeed.initialize(config=config, model=model, model_parameters=model.parameters())
return engine


def _step_with_grad(engine, batch, grad_value):
engine.backward(engine(batch[0], batch[1]))
for p in engine.parameters():
safe_set_full_grad(p, torch.full_like(p, grad_value))
engine.step()


@pytest.mark.parametrize("zero_stage", [1, 2])
class TestOffloadOverflowFlag(DistributedTest):
world_size = 2
Comment on lines +58 to +60

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved to tests/unit/v1/zero in 5257815; the selector now picks it in both workflows.


def test_flag_accumulates_on_device(self, zero_stage):
if not get_accelerator().is_bf16_supported():
pytest.skip("bf16 is not supported")
engine = _offload_engine(zero_stage, check_grad_overflow=True)
optimizer = engine.optimizer
original = DeepSpeedZeroOptimizer._has_inf_or_nan
DeepSpeedZeroOptimizer._has_inf_or_nan = staticmethod(lambda x: original(x).as_subclass(_NoTruthTest))
try:
optimizer.reset_cpu_buffers()
device = get_accelerator().current_device_name()
for value in (0.5, -2.0):
optimizer.update_offload_overflow_tracker(torch.full((4, ), value, device=device))
assert torch.is_tensor(optimizer.local_overflow)
assert not optimizer.has_overflow()

optimizer.update_offload_overflow_tracker(torch.tensor([1.0, float("inf")], device=device))
optimizer.update_offload_overflow_tracker(torch.full((4, ), 1.0, device=device))
assert optimizer.has_overflow()

optimizer.reset_cpu_buffers()
assert optimizer.local_overflow is False
assert not optimizer.has_overflow()
finally:
DeepSpeedZeroOptimizer._has_inf_or_nan = original

def test_overflow_still_skips_step(self, zero_stage):
if not get_accelerator().is_bf16_supported():
pytest.skip("bf16 is not supported")
engine = _offload_engine(zero_stage, check_grad_overflow=True)
loader = random_dataloader(model=engine,
total_samples=2,
hidden_dim=10,
device=engine.device,
dtype=torch.bfloat16)
batches = list(loader)
_step_with_grad(engine, batches[0], 0.01)
assert engine.skipped_steps == 0
_step_with_grad(engine, batches[1], float("nan"))
assert engine.skipped_steps == 1

def test_no_check_when_disabled(self, zero_stage):
if not get_accelerator().is_bf16_supported():
pytest.skip("bf16 is not supported")
engine = _offload_engine(zero_stage, check_grad_overflow=False)
original = DeepSpeedZeroOptimizer._has_inf_or_nan

def unexpected(x):
raise AssertionError("gradients were checked for overflow with check_grad_overflow=False")

DeepSpeedZeroOptimizer._has_inf_or_nan = staticmethod(unexpected)
try:
loader = random_dataloader(model=engine,
total_samples=1,
hidden_dim=10,
device=engine.device,
dtype=torch.bfloat16)
_step_with_grad(engine, next(iter(loader)), float("inf"))
assert engine.skipped_steps == 0
assert engine.optimizer.local_overflow is False
finally:
DeepSpeedZeroOptimizer._has_inf_or_nan = original
1 change: 1 addition & 0 deletions tests/unit/v1/zero/test_zero2_gradient_safety.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def make_optimizer(dtype=torch.bfloat16, device=None, low_precision=False, cpu_o
opt.accumulated_grads_in_cpu = {}
opt.norm_for_param_grads = {}
opt.local_overflow = False
opt.check_grad_overflow = True
opt.overflow = False
opt.dp_process_group = None
opt.model_parallel_rank = 0
Expand Down
Loading