From a17d8178bc7b72ddd0b28cc497569e951633d064 Mon Sep 17 00:00:00 2001 From: Siddhant Rajhans <51508945+siddhant-rajhans@users.noreply.github.com> Date: Tue, 3 Nov 2026 05:32:06 +0000 Subject: [PATCH] Fix DynamicLossScaler never lowering scale when hysteresis <= 0 When `fp16.hysteresis` is set to 0 or a negative value, `update_scale` tests `self.cur_hysteresis == 1` which never triggers because the counter starts at 0 (or below) and only moves further away from 1. This means the loss scale is never reduced after overflow, causing every subsequent step to be skipped indefinitely. Change the equality checks to `<= 1` so that a hysteresis of 0 or below is treated as "no delay", matching the behavior of the stage 0 fp16 optimizers. The default `hysteresis: 2` is unaffected. Fixes #8982 Signed-off-by: Siddhant Rajhans <51508945+siddhant-rajhans@users.noreply.github.com> --- deepspeed/runtime/fp16/loss_scaler.py | 2 +- .../test_dynamic_loss_scaler_hysteresis.py | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/unit/runtime/test_dynamic_loss_scaler_hysteresis.py diff --git a/deepspeed/runtime/fp16/loss_scaler.py b/deepspeed/runtime/fp16/loss_scaler.py index f40f34eafabb..540cbd613b2d 100755 --- a/deepspeed/runtime/fp16/loss_scaler.py +++ b/deepspeed/runtime/fp16/loss_scaler.py @@ -261,7 +261,7 @@ def _has_inf_or_nan(x): def update_scale(self, overflow): if overflow: # self.cur_scale /= self.scale_factor - if self.delayed_shift == 1 or self.cur_hysteresis == 1: + if self.delayed_shift <= 1 or self.cur_hysteresis <= 1: if (self.cur_scale == self.min_scale) and self.raise_error_at_min_scale: raise Exception( "Current loss scale already at minimum - cannot decrease scale anymore. Exiting run.") diff --git a/tests/unit/runtime/test_dynamic_loss_scaler_hysteresis.py b/tests/unit/runtime/test_dynamic_loss_scaler_hysteresis.py new file mode 100644 index 000000000000..eb2c898056c1 --- /dev/null +++ b/tests/unit/runtime/test_dynamic_loss_scaler_hysteresis.py @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +import pytest + +from deepspeed.runtime.fp16.loss_scaler import DynamicLossScaler + + +@pytest.fixture(autouse=True) +def _patch_dist(monkeypatch): + """Stub out dist.get_rank so update_scale can run without a process group.""" + import deepspeed.comm as dist + monkeypatch.setattr(dist, "get_rank", lambda: 0) + + +INIT_SCALE = 2**16 +SCALE_WINDOW = 1000 +MIN_SCALE = 1 + + +def _make_scaler(delayed_shift, consecutive_hysteresis=False): + return DynamicLossScaler( + init_scale=INIT_SCALE, + scale_window=SCALE_WINDOW, + min_scale=MIN_SCALE, + delayed_shift=delayed_shift, + consecutive_hysteresis=consecutive_hysteresis, + ) + + +class TestDynamicLossScalerHysteresis: + """Ensure DynamicLossScaler lowers its scale on overflow for all hysteresis values.""" + + @pytest.mark.parametrize("delayed_shift", [0, -1, -5]) + def test_zero_or_negative_hysteresis_lowers_scale(self, delayed_shift): + scaler = _make_scaler(delayed_shift) + scaler.update_scale(overflow=True) + assert scaler.cur_scale < INIT_SCALE + + def test_hysteresis_one_lowers_scale_immediately(self): + scaler = _make_scaler(delayed_shift=1) + scaler.update_scale(overflow=True) + assert scaler.cur_scale == INIT_SCALE / 2 + + def test_hysteresis_two_delays_one_step(self): + scaler = _make_scaler(delayed_shift=2) + scaler.update_scale(overflow=True) + assert scaler.cur_scale == INIT_SCALE + scaler.update_scale(overflow=True) + assert scaler.cur_scale == INIT_SCALE / 2 + + @pytest.mark.parametrize("delayed_shift", [0, 1, 2]) + def test_repeated_overflow_reaches_min_scale(self, delayed_shift): + scaler = _make_scaler(delayed_shift) + for _ in range(200): + try: + scaler.update_scale(overflow=True) + except Exception: + break + assert scaler.cur_scale == MIN_SCALE