Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion deepspeed/runtime/fp16/loss_scaler.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ def _has_inf_or_nan(x):
def update_scale(self, overflow):
if overflow:
# self.cur_scale /= self.scale_factor
if self.delayed_shift == 1 or self.cur_hysteresis == 1:
if self.delayed_shift <= 1 or self.cur_hysteresis <= 1:
if (self.cur_scale == self.min_scale) and self.raise_error_at_min_scale:
raise Exception(
"Current loss scale already at minimum - cannot decrease scale anymore. Exiting run.")
Expand Down
62 changes: 62 additions & 0 deletions tests/unit/runtime/test_dynamic_loss_scaler_hysteresis.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: Apache-2.0

# DeepSpeed Team

import pytest

from deepspeed.runtime.fp16.loss_scaler import DynamicLossScaler


@pytest.fixture(autouse=True)
def _patch_dist(monkeypatch):
"""Stub out dist.get_rank so update_scale can run without a process group."""
import deepspeed.comm as dist
monkeypatch.setattr(dist, "get_rank", lambda: 0)


INIT_SCALE = 2**16
SCALE_WINDOW = 1000
MIN_SCALE = 1


def _make_scaler(delayed_shift, consecutive_hysteresis=False):
return DynamicLossScaler(
init_scale=INIT_SCALE,
scale_window=SCALE_WINDOW,
min_scale=MIN_SCALE,
delayed_shift=delayed_shift,
consecutive_hysteresis=consecutive_hysteresis,
)


class TestDynamicLossScalerHysteresis:
"""Ensure DynamicLossScaler lowers its scale on overflow for all hysteresis values."""

@pytest.mark.parametrize("delayed_shift", [0, -1, -5])
def test_zero_or_negative_hysteresis_lowers_scale(self, delayed_shift):
scaler = _make_scaler(delayed_shift)
scaler.update_scale(overflow=True)
assert scaler.cur_scale < INIT_SCALE
Comment on lines +37 to +40

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Add an engine-level regression for nonpositive hysteresis

For fp16.hysteresis <= 0, the user-visible fix is that ZeRO training can recover after overflow, but these tests invoke update_scale() directly and stub distributed rank. They never exercise configuration propagation, overflow detection, step skipping, or subsequent optimizer progress; the existing ZeRO overflow tests use positive hysteresis. Add a DistributedTest with SimpleModel through deepspeed.initialize() that verifies an overflowing step lowers the scale without updating weights, then a finite step updates weights for zero and negative hysteresis, and execute it on an fp16-capable device. Root AGENTS.md explicitly requires integration tests for training-loop contract changes and actual-device execution.

AGENTS.md reference: AGENTS.md:L81-L82

Useful? React with 👍 / 👎.


def test_hysteresis_one_lowers_scale_immediately(self):
scaler = _make_scaler(delayed_shift=1)
scaler.update_scale(overflow=True)
assert scaler.cur_scale == INIT_SCALE / 2

def test_hysteresis_two_delays_one_step(self):
scaler = _make_scaler(delayed_shift=2)
scaler.update_scale(overflow=True)
assert scaler.cur_scale == INIT_SCALE
scaler.update_scale(overflow=True)
assert scaler.cur_scale == INIT_SCALE / 2

@pytest.mark.parametrize("delayed_shift", [0, 1, 2])
def test_repeated_overflow_reaches_min_scale(self, delayed_shift):
scaler = _make_scaler(delayed_shift)
for _ in range(200):
try:
scaler.update_scale(overflow=True)
except Exception:
break
assert scaler.cur_scale == MIN_SCALE
Loading