diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 709b4f993a40..64ad02577c7e 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -14,8 +14,9 @@ from deepspeed import comm as dist from .layers import * from deepspeed.accelerator import get_accelerator -from .fusedqkv_utils import require_tp_fused_qkvw -from deepspeed.module_inject.tp_shard import AutoTPMeta, get_shard_size, get_shard_size_list +from .fusedqkv_utils import (require_tp_fused_qkvw, replicates_kv_block, is_replicated_kv_attention, + replicated_kv_group_count, replicated_kv_shard_heads, replicated_kv_shard_meta) +from deepspeed.module_inject.tp_shard import (AutoTPMeta, get_shard_size, get_shard_size_list, _MODULE_KV_HEAD_ATTRS) from deepspeed.utils import groups from deepspeed.utils.logging import log_dist, print_dist from deepspeed.module_inject.layers import is_autotp_training_mode @@ -25,6 +26,19 @@ from deepspeed.runtime.zero import GatheredParameters from deepspeed.runtime.zero.utils import is_zero_param +# Module attributes update_mp_params may shard. _SHARDED_KV_HEAD_ATTRS is the subset that must +# stay whole for replicated-K/V attention; keep it a subset of both this list and the broader +# module attribute probe list so a later edit cannot silently drop one side. +_MP_PARAM_ATTRS = ("n_heads", "inner_dim", "num_heads", "num_kv", "num_attention_heads", "num_attn_heads", + "all_head_size", "embed_dim", "hidden_size", "num_key_value_heads", "num_kv_heads", "kv_n_heads", + "d_model", "num_attention_heads_per_partition", "num_multi_query_groups_per_partition", + "hidden_size_per_partition") +_SHARDED_KV_HEAD_ATTRS = ("num_key_value_heads", "num_kv_heads", "kv_n_heads", "num_kv") +assert set(_SHARDED_KV_HEAD_ATTRS) <= set(_MP_PARAM_ATTRS), \ + "_SHARDED_KV_HEAD_ATTRS must be a subset of _MP_PARAM_ATTRS" +assert set(_SHARDED_KV_HEAD_ATTRS) <= set(_MODULE_KV_HEAD_ATTRS), \ + "_SHARDED_KV_HEAD_ATTRS must be a subset of _MODULE_KV_HEAD_ATTRS" + def move(tensor, device, copy=True): if tensor.is_meta: @@ -229,15 +243,27 @@ def __init__(self, self.mp_size = None self.mp_group = None + self.partition_config = partition_config # Per-model TP metadata threaded through every layer / helper so each AutoTP instance # shards by its own model's kv-head / grain values and keeps its own training mode. # None is kept in the layers so they keep following set_autotp_mode, as before. self.tp_meta = AutoTPMeta.from_model_config(model_config, tp_grain_size, training=training_mode) + self._replicates_kv_block = replicates_kv_block(self.module) + if self.partition_config is not None and any( + is_replicated_kv_attention(child, self._replicates_kv_block) for child in self.module.modules()): + # update_mp_params still uses model-wide head counts here, which would leave + # ranks without query rows with empty attention attributes. + raise NotImplementedError("AutoTP does not support an explicit partition_config for replicated-K/V " + "attention yet; remove partition_config to use the query-head heuristic.") + if self.partition_config is None: + # Replicated-K/V layouts keep the whole K/V block on every rank, so shard by + # query heads; K/V head attributes stay whole. This assumes one replicated-K/V + # layout per model. + self.tp_meta = replicated_kv_shard_meta(self.tp_meta, self._replicates_kv_block) self.linear_layer_setting = linear_layer_setting self.orig_layer_impl = orig_layer_impl self.linear_policies = None self.conv_linear_layer = False - self.partition_config = partition_config self.vocab_parallel_lm_head = vocab_parallel_lm_head self._training_mode = training_mode self._vocab_parallel_lm_head_candidate = None @@ -406,10 +432,26 @@ def set_tensor_parallel_config(self, mp_size, mp_group): if self.training_mode: self.mp_group = groups.get_tensor_model_parallel_group() self.mp_size = groups.get_tensor_model_parallel_world_size() - return + else: + self.mp_size = mp_size + self.mp_group = mp_group - self.mp_size = mp_size - self.mp_group = mp_group + self._validate_replicated_kv_tp_width() + + def _validate_replicated_kv_tp_width(self): + """Reject a replicated-K/V layout sharded across more ranks than query heads. + + Runs after custom config, HF tp_plan, or heuristic AutoTP has been chosen. + """ + if self.partition_config is not None or not self._replicates_kv_block: + return + shard_heads = replicated_kv_shard_heads(self.tp_meta, self._replicates_kv_block) + if shard_heads is not None and self.mp_size > shard_heads: + # A rank beyond the query heads has no query rows and an empty output projection. + raise RuntimeError(f"AutoTP cannot shard this replicated key/value attention across {self.mp_size} ranks: " + f"the model has {shard_heads} query heads, so at least one rank would hold " + f"no query rows and an empty attention output projection. Reduce " + f"tensor_parallel.autotp_size to at most the query head count.") def _replace(self, child, name, conv_linear_layer): # This function should clearly define the routing rules for specific layers @@ -1002,19 +1044,25 @@ def update_mp_params(self, child, name=None): if has_unsharded_high_dimensional_param: setattr(child, "replaced", True) return - param_list = [ - "n_heads", "inner_dim", "num_heads", "num_kv", "num_attention_heads", "num_attn_heads", "all_head_size", - "embed_dim", "hidden_size", "num_key_value_heads", "num_kv_heads", "kv_n_heads", "d_model", - "num_attention_heads_per_partition", "num_multi_query_groups_per_partition", "hidden_size_per_partition" - ] + # Replicated-K/V attention follows query heads; K/V aliases stay whole and the + # group count follows the local query shard. + replicated_kv_attention = is_replicated_kv_attention(child, getattr(self, "_replicates_kv_block", False)) + param_list = list(_MP_PARAM_ATTRS) for param in param_list: if "Yuan" in str(child) and 'embed_dim' in param_list: param_list.remove('embed_dim') if hasattr(child, param): + if replicated_kv_attention and param in _SHARDED_KV_HEAD_ATTRS: + continue param_val = getattr(child, param) # get_shard_size selects its partitioning strategy from the module name, so the # attributes must be sharded under the same name as the weights they describe. setattr(child, param, get_shard_size(param_val, self.mp_size, self.tp_meta, name, rank=tp_index)) + # Recompute groups from the local query shard. The module may expose the K/V head count + # under any of the runtime aliases in _MODULE_KV_HEAD_ATTRS. + num_heads = getattr(child, "num_heads", None) + if replicated_kv_attention and hasattr(child, "num_key_value_groups") and num_heads is not None: + child.num_key_value_groups = replicated_kv_group_count(child, num_heads) setattr(child, "replaced", True) def update_linear_policies(self): diff --git a/deepspeed/module_inject/fusedqkv_utils.py b/deepspeed/module_inject/fusedqkv_utils.py index 9540f40aad47..6f7093c39aa5 100644 --- a/deepspeed/module_inject/fusedqkv_utils.py +++ b/deepspeed/module_inject/fusedqkv_utils.py @@ -2,9 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 # DeepSpeed Team +from dataclasses import replace +from typing import Optional + import torch from deepspeed.utils.logging import warning_once -from deepspeed.module_inject.tp_shard import AutoTPMeta, get_shard_size, get_shard_size_list +from deepspeed.module_inject.tp_shard import (AutoTPMeta, get_shard_size, get_shard_size_list, _MODULE_KV_HEAD_ATTRS, + _kv_head_count_from_module) def split_by_qkvlist_and_refuse(qkv_list, split_size, split_dim=0, cat_dim=0): @@ -51,6 +55,68 @@ def get_fused_qkv_type(module): return FUSED_QKV_TYPE_DICT[max(module_name_matches, key=len)] +_REPLICATED_KV_BLOCK_CLASS_NAMES = tuple(name for name, fused_type in FUSED_QKV_TYPE_DICT.items() + if fused_type == 'bigcodetype') + + +def replicates_kv_block(module): + """Whether this model's fused qkv layout keeps the whole K/V block on every rank. + + Uses class names instead of ``get_fused_qkv_type`` to avoid stringifying every module. + """ + return any( + any(name in type(child).__name__ for name in _REPLICATED_KV_BLOCK_CLASS_NAMES) for child in module.modules()) + + +def is_replicated_kv_attention(module, replicated_kv): + """Whether ``module`` is an attention module using the replicated-K/V layout.""" + return bool(replicated_kv and getattr(module, "multi_query", False)) + + +def replicated_kv_shard_meta(meta: AutoTPMeta, replicated_kv: bool) -> AutoTPMeta: + """Metadata whose shard unit is query heads for replicated K/V, otherwise K/V heads. + + ``get_shard_size`` always reads ``num_kv_heads`` as the shard unit. Replacing that field + keeps the fused-qkv partition, the query-row widths, and the attribute sharding on the same + head count instead of each caller deriving it independently. + """ + if not replicated_kv or meta.num_attention_heads is None: + return meta + return replace(meta, num_kv_heads=meta.num_attention_heads) + + +def replicated_kv_shard_heads(meta: AutoTPMeta, replicated_kv: bool) -> Optional[int]: + """Number of attention heads that bound one rank's shard, or None when unspecified.""" + if replicated_kv and meta.num_attention_heads is None: + return None + return replicated_kv_shard_meta(meta, replicated_kv).num_kv_heads + + +def replicated_kv_query_widths(meta: AutoTPMeta, mp_size: int): + """Per-rank query-row widths for a replicated-K/V fused qkv layout, or None without n_embd.""" + shard_meta = replicated_kv_shard_meta(meta, True) + if shard_meta.n_embd is None: + return None + return get_shard_size_list(shard_meta.n_embd, mp_size, shard_meta) + + +def replicated_kv_row_start(meta: AutoTPMeta, mp_size: int, rank: int) -> Optional[int]: + """First K/V row in ``rank``'s local replicated-K/V shard.""" + query_widths = replicated_kv_query_widths(meta, mp_size) + if query_widths is None: + return None + return query_widths[rank] + + +def replicated_kv_group_count(module, local_num_heads: int) -> int: + """Query-to-K/V group count after the query heads have been sharded.""" + kv_heads = _kv_head_count_from_module(module) + if not kv_heads: + raise RuntimeError(f"AutoTP cannot recompute num_key_value_groups for {type(module).__name__}: " + f"none of the module K/V head attributes {_MODULE_KV_HEAD_ATTRS} is set.") + return local_num_heads // kv_heads + + def fused_qkv_subparam_sizes(module, weight_shape, meta: AutoTPMeta): """Sizes of the sub-parameters a fused qkv weight is cut into, or None. @@ -163,8 +229,7 @@ def _bigcode_type_transpose(input, mp_size, meta=meta): assert n_embd is not None q = input[:n_embd] kv = input[n_embd:] - shape = q.shape - split_q = q.split(get_shard_size_list(shape[0], mp_size, meta), dim=0) + split_q = q.split(replicated_kv_query_widths(meta, mp_size), dim=0) return torch.cat((split_q[gpu_index], kv), dim=0) def _phi3_type_transpose(input, mp_size, meta=meta): diff --git a/deepspeed/module_inject/layers.py b/deepspeed/module_inject/layers.py index 911dd3823cca..9c85cdded7b5 100644 --- a/deepspeed/module_inject/layers.py +++ b/deepspeed/module_inject/layers.py @@ -18,7 +18,7 @@ from abc import ABC, abstractmethod from typing import Iterable, Any, Optional, List, Tuple, Dict from .fusedqkv_utils import (shard_value_with_share_qk, prepare_tp_fused_qkvw, fused_qkv_subparam_sizes, - set_fused_qkv_shard_state) + set_fused_qkv_shard_state, replicated_kv_query_widths, replicated_kv_row_start) from deepspeed.runtime.tensor_parallel import AUTOTP_MODE from deepspeed.checkpoint.constants import (DS_AUTOTP_UC_META, UNIVERSAL_CHECKPOINT_VERSION_KEY, UNIVERSAL_CHECKPOINT_VERSION_VALUE) @@ -1467,6 +1467,124 @@ def _mark_uc_metadata(self): is_bias=True) +def _all_reduce_replicated_kv_rows(kv_grad, group): + """All-reduce the replicated K/V rows in place. + + Some backends return this gradient non-contiguously, which collectives reject; reduce a + contiguous copy in that case and preserve the original layout for autograd. + """ + if kv_grad.numel() == 0: + # The key/value block is replicated whole, so it is empty on every rank at once. + return kv_grad + if kv_grad.is_contiguous(): + # Contiguous: reduce the view in place. + dist.all_reduce(kv_grad, group=group) + else: + # Non-contiguous: reduce a copy, then write it back. + reduced = kv_grad.contiguous() + dist.all_reduce(reduced, group=group) + kv_grad.copy_(reduced) + return kv_grad + + +def _sum_replicated_kv_gradient(grad, kv_start, group): + """Sum the K/V gradient rows that every rank holds a copy of.""" + grad = grad.clone() + _all_reduce_replicated_kv_rows(grad.narrow(0, kv_start, grad.shape[0] - kv_start), group) + return grad + + +class _ReplicatedKvGradReducer: + """Sum each micro-batch's replicated K/V gradient exactly once. + + A plain tensor hook reduces the incoming gradient before PyTorch accumulates it, which is + correct with gradient accumulation. Some backends skip that hook under ZeRO 1/2, so a + post-accumulate hook is registered as well. The fallback reduces only the K/V rows added + since its previous call: reducing the whole accumulated gradient again would scale earlier + micro-batches once per later backward. + + When the engine allows deferred reduction, both hooks leave the local partial in place and + the post-accumulate hook reduces the K/V rows that were accumulated since the previous + boundary. The engine only allows this when the full window stays in ``param.grad``. + """ + + def __init__(self, param, kv_start, group): + self.param = param + self.kv_start = kv_start + self.group = group + self._tensor_hook_ran = False + self._grad = None + self._kv = None + self._boundary_reduced = False + self._accumulation_boundary = True + self._defer_allowed = False + + def set_accumulation_state(self, is_boundary, defer_allowed): + """Record the engine's accumulation state for this backward. + + Called from the engine's backward prologue for this reducer only, so two engines in + one process do not overwrite each other's state. A non-boundary backward starts a new + window: even a parameter that is unused there must forget the previous boundary's + reduced snapshot. + """ + self._accumulation_boundary = is_boundary + self._defer_allowed = defer_allowed + if not is_boundary: + self._tensor_hook_ran = False + self._grad = None + self._kv = None + self._boundary_reduced = False + + def tensor_hook(self, grad): + self._tensor_hook_ran = True + if self._defer_allowed: + return grad + return _sum_replicated_kv_gradient(grad, self.kv_start, self.group) + + def _reduce_new_kv_rows(self, grad, kv_grad, use_delta): + if use_delta and self._grad is grad and self._kv is not None: + delta = kv_grad - self._kv + else: + delta = kv_grad.clone() + _all_reduce_replicated_kv_rows(delta, self.group) + if use_delta and self._grad is grad and self._kv is not None: + kv_grad.copy_(self._kv + delta) + else: + kv_grad.copy_(delta) + self._grad = grad + self._kv = kv_grad.clone() + + def post_accumulate_hook(self, *_unused): + # Both hook forms leave the gradient on the parameter; the pre-2.1 + # AccumulateGrad hook also passes (grad_inputs, grad_outputs). + grad = self.param.grad + if grad is None: + self._tensor_hook_ran = False + self._grad = None + self._kv = None + self._boundary_reduced = False + return + kv_grad = grad.narrow(0, self.kv_start, grad.shape[0] - self.kv_start) + if self._defer_allowed: + if not self._accumulation_boundary: + # The next window starts here; forget the previous boundary's snapshot. + self._tensor_hook_ran = False + self._grad = None + self._kv = None + return + # A reentrant checkpoint may run this hook more than once at the boundary. + # Reducing the delta keeps each new local contribution counted once. + self._reduce_new_kv_rows(grad, kv_grad, use_delta=self._boundary_reduced) + self._boundary_reduced = True + self._tensor_hook_ran = False + return + if self._tensor_hook_ran: + # The tensor hook already summed this micro-batch before it was accumulated. + self._tensor_hook_ran = False + else: + self._reduce_new_kv_rows(grad, kv_grad, use_delta=True) + + class FusedModuleWrapper: def __init__(self, fused_module: nn.Module): @@ -1505,21 +1623,22 @@ def _sum_replicated_kv_gradients(self): if get_fused_qkv_type(self.fused_module.module) != 'bigcodetype' or not pg_utils.is_parallel(self.mp_group): return - n_embd = self.tp_meta.n_embd - if n_embd is None: - return # This rank's shard is its query rows followed by the whole key/value block. - kv_start = get_shard_size_list(n_embd, self.tp_world_size, self.tp_meta)[self.tp_index] + kv_start = replicated_kv_row_start(self.tp_meta, self.tp_world_size, self.tp_index) + if kv_start is None: + return group = self.mp_group - def sum_kv_rows(grad): - grad = grad.clone() - dist.all_reduce(grad.narrow(0, kv_start, grad.shape[0] - kv_start), group=group) - return grad + from deepspeed.utils.torch import register_grad_hook + # The tensor hook handles the common path before accumulation; the post-accumulate + # fallback covers torch_npu, which skipped it under ZeRO 1/2 in our tests. for param in (self.weight, self.bias): if param is not None and param.requires_grad: - param.register_hook(sum_kv_rows) + reducer = _ReplicatedKvGradReducer(param, kv_start, group) + param.register_hook(reducer.tensor_hook) + register_grad_hook(param, reducer.post_accumulate_hook) + param._ds_replicated_kv_reducer = reducer def _freeze_partition_sizes(self, total_size): super()._freeze_partition_sizes(total_size) @@ -1574,7 +1693,9 @@ def _bigcode_affine_map(self, shape): # The query rows are not split evenly: the widths follow the head count and the grain # size, so they have to come from the same helper the partition calls rather than # from a division here. - query_widths = list(get_shard_size_list(n_embd, self.tp_world_size, self.tp_meta)) + query_widths = replicated_kv_query_widths(self.tp_meta, self.tp_world_size) + if query_widths is None: + return None return segmented_map(shape, [(n_embd, False), (shape[0] - n_embd, True)], 0, self.tp_world_size, diff --git a/deepspeed/module_inject/tp_shard.py b/deepspeed/module_inject/tp_shard.py index 9fb66eb114b9..406645a00caf 100644 --- a/deepspeed/module_inject/tp_shard.py +++ b/deepspeed/module_inject/tp_shard.py @@ -8,6 +8,9 @@ from deepspeed import comm as dist +# Attribute names that carry the (query) attention head count, in probe order. +_ATTN_HEAD_ATTRS = ('num_attention_heads', 'num_heads', 'n_heads', 'n_head', 'attention_heads') + # Attribute names that carry the key/value head count, in probe order. Modern canonical names # first, then legacy aliases kept for older configs / checkpoints, then the query-head names as # an implicit "kv == q heads" default for plain (non-GQA) transformers. Shared by AutoTP and the @@ -23,6 +26,16 @@ 'attention_heads', ) +# The K/V-only subset of _KV_HEAD_ATTRS. The query-head aliases above are a config fallback for +# plain MHA models; they must not be treated as K/V counts when a replicated-K/V module still +# shards its query dimension. +_KV_ONLY_HEAD_ATTRS = tuple(name for name in _KV_HEAD_ATTRS if name not in _ATTN_HEAD_ATTRS) + +# Runtime module attributes that can carry a K/V head count. `kv_heads` is GPTBigCode's module +# attribute and `num_kv` is DeepSpeed's module alias, so neither appears in the config-only list +# above. Query-head aliases are deliberately absent for the same reason as _KV_ONLY_HEAD_ATTRS. +_MODULE_KV_HEAD_ATTRS = ('kv_heads', ) + _KV_ONLY_HEAD_ATTRS + ('num_kv', ) + def _kv_head_count_from(config) -> Optional[int]: """First non-None kv-head count found on ``config`` via :data:`_KV_HEAD_ATTRS`, else ``None``.""" @@ -33,8 +46,13 @@ def _kv_head_count_from(config) -> Optional[int]: return None -# Attribute names that carry the (query) attention head count, in probe order. -_ATTN_HEAD_ATTRS = ('num_attention_heads', 'num_heads', 'n_heads', 'n_head', 'attention_heads') +def _kv_head_count_from_module(module) -> Optional[int]: + """First non-None K/V head count found on ``module`` via :data:`_MODULE_KV_HEAD_ATTRS`.""" + for name in _MODULE_KV_HEAD_ATTRS: + value = getattr(module, name, None) + if value is not None: + return value + return None def _attention_head_count_from(config) -> Optional[int]: diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 284a9c7b5e38..98e543bfc1ef 100644 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -46,7 +46,8 @@ from deepspeed.runtime.bf16_optimizer import BF16_Optimizer from deepspeed.linear.optimized_linear import LoRAOptimizedLinear -from deepspeed.module_inject.layers import GatherReplacedLayerParams, configure_tensor_parallel_runtime, collect_autotp_universal_checkpoint_info +from deepspeed.module_inject.layers import (GatherReplacedLayerParams, configure_tensor_parallel_runtime, + collect_autotp_universal_checkpoint_info) from deepspeed.module_inject.auto_ep_folding import (clear_autoep_folding_gradient_corrected, is_autoep_folding_gradient_corrected, reduce_autoep_folding_gradient) @@ -563,6 +564,10 @@ def __init__(self, self.lr_scheduler = None self._is_gradient_accumulation_boundary = None + self._replicated_kv_reducers = [] + # Deferral is a per-window decision; see _backward_prologue. + self._replicated_kv_window = None + self._replicated_kv_window_defer = False self.scale_wrt_gas = None self.losses = None self.mesh_device = mesh_device @@ -604,6 +609,9 @@ def __init__(self, # Head counts were recorded against the whole model; the parameters are shards now. from deepspeed import resolve_per_head_muon_after_sharding resolve_per_head_muon_after_sharding(model) + self._replicated_kv_reducers = [ + p._ds_replicated_kv_reducer for p in model.parameters() if hasattr(p, "_ds_replicated_kv_reducer") + ] see_memory_usage("DeepSpeed Engine: After args sanity test", force=self.memory_breakdown()) self._set_distributed_vars(args) @@ -1047,16 +1055,19 @@ def _apply_autotp_partitioning(self, model, tp_config): # from_model_config descends into text_config itself, so multimodal outer configs # work here too. from deepspeed.module_inject.tp_shard import AutoTPMeta + from deepspeed.module_inject.fusedqkv_utils import replicates_kv_block, replicated_kv_shard_heads - num_kv_heads = AutoTPMeta.from_model_config(model_config).num_kv_heads + meta = AutoTPMeta.from_model_config(model_config) + # Replicated-K/V layouts are bounded by query heads, not K/V heads. + replicated_kv = replicates_kv_block(model) + shard_heads = replicated_kv_shard_heads(meta, replicated_kv) - # Ranks beyond the KV head count get no attention shard. This still computes the - # correct result because the row-parallel all-reduce sums their empty contribution, - # but attention work concentrates on the first num_kv_heads ranks. - if num_kv_heads is not None and tp_size > num_kv_heads: + # The replicated-K/V rejection lives in AutoTP.set_tensor_parallel_config and also + # covers inference; keep this warning only for non-replicated layouts. + if not replicated_kv and shard_heads is not None and tp_size > shard_heads: log_dist( f"AutoTP: autotp_size ({tp_size}) exceeds the model's key-value head count " - f"({num_kv_heads}); ranks beyond the head count hold no attention shard and " + f"({shard_heads}); ranks beyond the head count hold no attention shard and " "attention throughput will not scale past that point.", ranks=[0], level=logging.WARNING) @@ -3149,6 +3160,39 @@ def _backward_prologue(self): self._start_timers(self.engine_timers.backward_timers) self._check_autotp_frozen_replicated_params() + if self._replicated_kv_reducers: + # Deferral is safe only when the whole accumulation window stays in param.grad. + # ZeRO-1 with a separate accumulation buffer moves each micro-batch's gradient out + # of param.grad, BF16_Optimizer accumulates into its own fp32 buffer, and ZeRO-2/3 + # reduce or partition the gradient during each micro-batch. no_sync and coalesced + # reduction deliberately delay the engine's own reduction as well. + # + # Latch the decision for the whole window. Deciding per backward lets one window mix + # the deferred and per-micro-batch paths (for example when only the non-boundary + # backwards are inside no_sync); the boundary's whole-window reduce then counts the + # micro-batches that were already reduced before accumulation again. + window = self.micro_steps // self.gradient_accumulation_steps() + if window != self._replicated_kv_window: + self._replicated_kv_window = window + stage = self.zero_optimization_stage() + optimizer = self.optimizer + if stage == ZeroStageEnum.disabled: + gradient_is_local = (not isinstance(optimizer, BF16_Optimizer) + and not self.zero_use_cpu_optimizer()) + else: + gradient_is_local = (stage == ZeroStageEnum.optimizer_states + and isinstance(optimizer, DeepSpeedZeroOptimizer) + and not getattr(optimizer, "use_grad_accum_attribute", True) + and not self.zero_use_cpu_optimizer() and not self.zenflow) + self._replicated_kv_window_defer = ( + gradient_is_local and self.gradient_accumulation_steps() > 1 + and self.managed_gradient_accumulation() and not self.inside_no_sync_ctxt + and not self.pipeline_parallelism) + defer_allowed = self._replicated_kv_window_defer + is_boundary = self.is_gradient_accumulation_boundary() + for reducer in self._replicated_kv_reducers: + reducer.set_accumulation_state(is_boundary, defer_allowed) + # When necessary internal APIs are not available, we disable direct calls to tensor.backward() # and limit to engine.backward(loss) only. if not self._support_torch_style_backward and not self._running_engine_backward: diff --git a/tests/unit/v1/autotp/test_autotp_custom_patterns.py b/tests/unit/v1/autotp/test_autotp_custom_patterns.py index db72bc619726..8b79bdeb50c5 100644 --- a/tests/unit/v1/autotp/test_autotp_custom_patterns.py +++ b/tests/unit/v1/autotp/test_autotp_custom_patterns.py @@ -19,7 +19,9 @@ from deepspeed.checkpoint.constants import (DS_AUTOTP_UC_META, PARAMETER_WITH_ROW_PARALLELISM_PATTERNS, TP_REPLICATED_PARAMETER_PATTERNS) from deepspeed.module_inject.autotp_config import AutoTPConfig -from deepspeed.module_inject.tp_shard import AutoTPMeta, get_shard_size, get_shard_size_list +from deepspeed.module_inject.tp_shard import (AutoTPMeta, get_shard_size, get_shard_size_list, _MODULE_KV_HEAD_ATTRS) +from deepspeed.module_inject.fusedqkv_utils import (replicated_kv_query_widths, replicated_kv_row_start, + replicated_kv_shard_heads) from deepspeed.module_inject.auto_tp import AutoTP from deepspeed.module_inject.auto_tp_model_utils import (build_bloom_alibi_tensor, build_mpt_alibi_tensor, get_alibi_mask, install_head_sharded_helper) @@ -510,6 +512,133 @@ def test_update_mp_params_uses_group_local_rank(monkeypatch): assert child.num_heads == 4 +class _GPTBigCodeLikeConfig: + n_embd = 32 + n_head = 8 + num_key_value_heads = 1 + + +class _GPTBigCodeBlockStub(nn.Module): + + def __init__(self): + super().__init__() + self.attn = nn.Module() + self.attn.multi_query = True + + +def _autotp_stub(*, tp_meta=None, partition_config=None, mp_group=None, mp_size=2): + autotp = object.__new__(AutoTP) + autotp.mp_group = mp_group + autotp.mp_size = mp_size + autotp.partition_config = partition_config + autotp._replicates_kv_block = True + autotp._training_mode = None + autotp.tp_meta = tp_meta if tp_meta is not None else AutoTPMeta(num_attention_heads=8, num_kv_heads=8) + return autotp + + +def _kv_attention_stub(monkeypatch, **attrs): + tp_group = object() + child = nn.Module() + child.proj = nn.Linear(1, 1, bias=False) + setattr(child.proj.weight, DS_AUTOTP_UC_META, {}) + child.multi_query = True + for name, value in attrs.items(): + setattr(child, name, value) + monkeypatch.setattr(dist, "get_rank", lambda group=None: 1 if group is tp_group else 0) + return _autotp_stub(mp_group=tp_group), child + + +def test_explicit_partition_config_rejects_replicated_kv_attention(): + # The heuristic attribute sharding cannot follow an arbitrary explicit config yet. + with pytest.raises(NotImplementedError, match="explicit partition_config"): + AutoTP(module=_GPTBigCodeBlockStub(), + all_reduce_linears=(), + prefix="", + state_dict=None, + linear_layer_setting=(nn.Linear, nn.Embedding), + orig_layer_impl=None, + partition_config=AutoTPConfig(tp_size=2), + model_config=_GPTBigCodeLikeConfig(), + training_mode=True) + + +def test_explicit_partition_config_keeps_non_mqa_layout(monkeypatch): + # Non-MQA explicit configs are still supported: no heuristic tp_meta override and no + # width rejection. + monkeypatch.setattr("deepspeed.module_inject.auto_tp.replicates_kv_block", lambda module: True) + autotp = AutoTP(module=nn.Linear(1, 1), + all_reduce_linears=(), + prefix="", + state_dict=None, + linear_layer_setting=(nn.Linear, nn.Embedding), + orig_layer_impl=None, + partition_config=AutoTPConfig(tp_size=8), + model_config=_GPTBigCodeLikeConfig(), + training_mode=False) + + assert autotp.tp_meta.num_kv_heads == 1 + autotp.set_tensor_parallel_config(8, None) + assert autotp.mp_size == 8 + + +@pytest.mark.parametrize("attrs, expected, error", [ + (dict(num_heads=8, kv_heads=1, + num_key_value_heads=1), dict(num_heads=4, num_key_value_heads=1, num_key_value_groups=None), None), + (dict(num_heads=8, num_key_value_heads=1, + num_key_value_groups=8), dict(num_heads=4, num_key_value_heads=1, num_key_value_groups=4), None), + (dict(num_heads=8, num_key_value_groups=8), {}, "none of the module K/V head attributes"), + (dict(num_heads=8, kv_heads=1, num_key_value_groups=8), dict(num_heads=4, kv_heads=1, + num_key_value_groups=4), None), +]) +def test_update_mp_params_replicated_kv_attributes(monkeypatch, attrs, expected, error): + autotp, child = _kv_attention_stub(monkeypatch, **attrs) + if error: + with pytest.raises(RuntimeError, match=error): + autotp.update_mp_params(child) + return + autotp.update_mp_params(child) + for name, value in expected.items(): + if value is None: + assert not hasattr(child, name) + else: + assert getattr(child, name) == value + + +@pytest.mark.parametrize("kv_attr", _MODULE_KV_HEAD_ATTRS) +def test_update_mp_params_recomputes_groups_from_each_kv_attr(monkeypatch, kv_attr): + # Keep the skip list and the group-count lookup on the same attribute list. + autotp, child = _kv_attention_stub(monkeypatch, num_heads=8, num_key_value_groups=8, **{kv_attr: 1}) + autotp.update_mp_params(child) + assert child.num_heads == 4 + assert getattr(child, kv_attr) == 1 + assert child.num_key_value_groups == 4 + + +def test_replicated_kv_policy_helpers_use_query_heads(): + meta = AutoTPMeta(num_attention_heads=8, num_kv_heads=1, n_embd=32) + + assert replicated_kv_shard_heads(meta, True) == 8 + assert replicated_kv_shard_heads(meta, False) == 1 + assert replicated_kv_query_widths(meta, 2) == [16, 16] + assert replicated_kv_row_start(meta, 2, 1) == 16 + + +@pytest.mark.parametrize("training", [False, True]) +def test_replicated_kv_attention_rejects_tp_beyond_query_heads(monkeypatch, training): + autotp = _autotp_stub(tp_meta=AutoTPMeta(num_attention_heads=4, num_kv_heads=4)) + monkeypatch.setattr("deepspeed.module_inject.auto_tp.is_autotp_training_mode", lambda: training) + if training: + monkeypatch.setattr("deepspeed.module_inject.auto_tp.groups.get_tensor_model_parallel_group", lambda: None) + monkeypatch.setattr("deepspeed.module_inject.auto_tp.groups.get_tensor_model_parallel_world_size", lambda: 8) + mp_size = 1 + else: + mp_size = 8 + + with pytest.raises(RuntimeError, match="4 query heads"): + autotp.set_tensor_parallel_config(mp_size, None) + + def test_update_mp_params_shards_attributes_like_their_weights(monkeypatch): tp_group = object() autotp = object.__new__(AutoTP) diff --git a/tests/unit/v1/autotp/test_autotp_inference.py b/tests/unit/v1/autotp/test_autotp_inference.py new file mode 100644 index 000000000000..339284d2cb28 --- /dev/null +++ b/tests/unit/v1/autotp/test_autotp_inference.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +from copy import deepcopy + +import pytest +import torch + +import deepspeed +from deepspeed.accelerator import get_accelerator +from unit.common import DistributedTest + + +def skip_on_device(): + if get_accelerator().device_name() == 'xpu': + pytest.skip("XPU requires a higher version for test") + + +def _config(transformers, *, n_layer=2, n_head=4): + return transformers.GPTBigCodeConfig(vocab_size=64, + n_positions=32, + n_embd=32, + n_layer=n_layer, + n_head=n_head, + multi_query=True) + + +class TestGPTBigCodeMultiQueryInference(DistributedTest): + """Inference of multi-query GPTBigCode must match the unsharded model under AutoTP. + + The inference injection path builds the same AutoTP instance as training, so it inherits the + query-head split for the fused c_attn and the replicated key/value block. + """ + world_size = 2 + reuse_dist_env = False + + def test_matches_unsharded_logits(self): + skip_on_device() + transformers = pytest.importorskip("transformers") + torch.manual_seed(0) + model = transformers.GPTBigCodeForCausalLM(_config(transformers)).eval() + reference = deepcopy(model) + + device = torch.device(get_accelerator().current_device_name()) + reference = reference.to(device) + input_ids = torch.randint(0, 64, (2, 8), device=device) + with torch.no_grad(): + expected = reference(input_ids).logits + + engine = deepspeed.init_inference(model, + tensor_parallel={"tp_size": self.world_size}, + dtype=torch.float32, + replace_with_kernel_inject=False) + with torch.no_grad(): + actual = engine(input_ids).logits + + torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5) diff --git a/tests/unit/v1/autotp/test_autotp_training.py b/tests/unit/v1/autotp/test_autotp_training.py index c561bcd2a416..632d4016f510 100644 --- a/tests/unit/v1/autotp/test_autotp_training.py +++ b/tests/unit/v1/autotp/test_autotp_training.py @@ -21,14 +21,15 @@ UNIVERSAL_CHECKPOINT_INFO) from unit.simple_model import SimpleModel, random_dataloader from deepspeed.utils import groups -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from torch import nn from deepspeed.module_inject.auto_tp import AutoTP from deepspeed.module_inject.fusedqkv_utils import shared_qk_value_head_ids from deepspeed.module_inject.layers import (LinearAllreduce, LinearLayer, VocabParallelLinear, VocabParallelEmbedding, set_autotp_mode, is_autotp_training_mode, GatherFromTensorParallelRegion, ScatterToTensorParallelRegion, HiddenParallelEmbedding, - Yuan_LinearAllreduce) + Yuan_LinearAllreduce, _sum_replicated_kv_gradient, + _ReplicatedKvGradReducer) from deepspeed.module_inject.tp_shard import AutoTPMeta, get_shard_size_list from unit.checkpoint.common import compare_lr_scheduler_states, compare_optimizer_states import os @@ -1282,18 +1283,166 @@ def test_forward_and_backward_match_the_unsharded_layer(self): set_autotp_mode(training=False) +def test_replicated_kv_gradient_hook_and_staging(monkeypatch): + import deepspeed.module_inject.layers as layers_module + + kv_start = 3 + # Some backends return the weight gradient in a transposed layout. + grad = torch.arange(40.0).reshape(5, 8).t() + assert grad.shape == (8, 5) and not grad.is_contiguous() + + collectives = [] + + def fake_all_reduce(tensor, group=None): + collectives.append((tuple(tensor.shape), tensor.is_contiguous())) + tensor.fill_(-1.0) + + # Collective backends reject non-contiguous inputs, so the test stands in for that + # documented contract and records what actually reaches the backend. + monkeypatch.setattr(layers_module.dist, "all_reduce", fake_all_reduce) + reduced = _sum_replicated_kv_gradient(grad, kv_start, group=None) + + assert collectives == [((5, 5), True)] + torch.testing.assert_close(reduced[:kv_start], grad[:kv_start]) + torch.testing.assert_close(reduced[kv_start:], torch.full((5, 5), -1.0)) + assert not reduced.is_contiguous() + + # PyTorch < 2.1 AccumulateGrad hooks receive (grad_inputs, grad_outputs). + param = nn.Parameter(torch.zeros(4, 2)) + param.grad = torch.ones(4, 2) + monkeypatch.setattr(layers_module.dist, "all_reduce", lambda tensor, group=None: tensor.fill_(7.0)) + reducer = _ReplicatedKvGradReducer(param, kv_start=2, group=None) + reducer.post_accumulate_hook((torch.empty_like(param.grad), ), (torch.empty_like(param.grad), )) + torch.testing.assert_close(param.grad[:2], torch.ones(2, 2)) + torch.testing.assert_close(param.grad[2:], torch.full((2, 2), 7.0)) + + +def _run_two_accumulated_backwards(monkeypatch, register_tensor_hook): + import deepspeed.module_inject.layers as layers_module + from deepspeed.utils.torch import register_grad_hook + + param = nn.Parameter(torch.zeros(4, 1)) + # Simulate a two-rank TP sum on the replicated K/V rows. + monkeypatch.setattr(layers_module.dist, "all_reduce", lambda tensor, group=None: tensor.mul_(2)) + reducer = _ReplicatedKvGradReducer(param, kv_start=2, group=None) + if register_tensor_hook: + param.register_hook(reducer.tensor_hook) + register_grad_hook(param, reducer.post_accumulate_hook) + for _ in range(2): + (param * 1.0).sum().backward() + return param + + +@pytest.mark.parametrize("register_tensor_hook", [True, False]) +def test_replicated_kv_gradient_reduces_each_micro_batch_once(monkeypatch, register_tensor_hook): + # The second case simulates the torch_npu path where the plain tensor hook is skipped under + # ZeRO 1/2. The post-accumulate fallback must reduce only the new K/V rows, not the whole + # accumulated gradient. + param = _run_two_accumulated_backwards(monkeypatch, register_tensor_hook) + torch.testing.assert_close(param.grad, torch.tensor([[2.0], [2.0], [4.0], [4.0]])) + + +def test_replicated_kv_gradient_defers_to_accumulation_boundary(monkeypatch): + import deepspeed.module_inject.layers as layers_module + from deepspeed.utils.torch import register_grad_hook + + param = nn.Parameter(torch.zeros(4, 1)) + calls = [] + + def fake_all_reduce(tensor, group=None): + calls.append(tensor.shape) + tensor.mul_(2) + + monkeypatch.setattr(layers_module, "_all_reduce_replicated_kv_rows", fake_all_reduce) + reducer = _ReplicatedKvGradReducer(param, kv_start=2, group=None) + param.register_hook(reducer.tensor_hook) + register_grad_hook(param, reducer.post_accumulate_hook) + try: + reducer.set_accumulation_state(is_boundary=False, defer_allowed=True) + (param * 1.0).sum().backward() + assert calls == [] + + reducer.set_accumulation_state(is_boundary=True, defer_allowed=True) + (param * 1.0).sum().backward() + assert calls == [(2, 1)] + torch.testing.assert_close(param.grad, torch.tensor([[2.0], [2.0], [4.0], [4.0]])) + finally: + reducer.set_accumulation_state(is_boundary=True, defer_allowed=False) + + +def test_replicated_kv_gradient_deferred_boundary_reduces_new_rows(monkeypatch): + """A reentrant boundary can run the post hook more than once in one backward.""" + import deepspeed.module_inject.layers as layers_module + + param = nn.Parameter(torch.zeros(4, 1)) + calls = [] + + def fake_all_reduce(tensor, group=None): + calls.append(tensor.shape) + tensor.mul_(2) + + monkeypatch.setattr(layers_module, "_all_reduce_replicated_kv_rows", fake_all_reduce) + reducer = _ReplicatedKvGradReducer(param, kv_start=2, group=None) + reducer.set_accumulation_state(is_boundary=True, defer_allowed=True) + + param.grad = torch.ones(4, 1) + reducer.post_accumulate_hook() + torch.testing.assert_close(param.grad, torch.tensor([[1.0], [1.0], [2.0], [2.0]])) + + # A later phase adds one more local contribution to the K/V rows. + param.grad[2:] += 1.0 + reducer.post_accumulate_hook() + assert calls == [(2, 1), (2, 1)] + torch.testing.assert_close(param.grad, torch.tensor([[1.0], [1.0], [4.0], [4.0]])) + + +@contextmanager +def _count_kv_gradient_all_reduces(kv_rows, hidden): + """Count the K/V-row all-reduces issued by the fused c_attn gradient hook. + + The hook runs in the autograd thread, so the test observes it by wrapping the collective it + calls. Only the replicated K/V slices have these shapes; the other all-reduces in the step + operate on full activations or gradients. + """ + import deepspeed.module_inject.layers as layers_module + + counts = {"weight": 0, "bias": 0} + original_all_reduce = layers_module.dist.all_reduce + + def counting_all_reduce(tensor, *args, **kwargs): + shape = tuple(tensor.shape) + if shape == (kv_rows, hidden): + counts["weight"] += 1 + elif shape == (kv_rows, ): + counts["bias"] += 1 + return original_all_reduce(tensor, *args, **kwargs) + + layers_module.dist.all_reduce = counting_all_reduce + try: + yield counts + finally: + layers_module.dist.all_reduce = original_all_reduce + + class TestGPTBigCodeMultiQueryTraining(DistributedTest): - """Multi-query GPTBigCode must train the same model under AutoTP as on one device. + """GPTBigCode MQA must train like one device under AutoTP. - Every rank holds the whole shared key/value block of the fused c_attn but attends with only - its own query heads, so the gradient each rank computes for those rows is partial. Unless the - ranks sum it, each applies a different update to rows that must stay identical (#8927). + TP=2 covers ZeRO 0/1/2/3 with DP=2; TP=4 covers the four-way query split. Each rank's + partial K/V gradient must be summed, and the hook is asserted in every case. """ - world_size = [2, 4] + world_size = 4 reuse_dist_env = False - def test_matches_single_device_training(self): + @pytest.mark.parametrize("zero_stage", [0, 1, 2, 3]) + def test_matches_single_device_training_tp2(self, zero_stage): + self._train_and_compare(2, zero_stage) + + def test_matches_single_device_training_tp4(self): + self._train_and_compare(4, 0) + + def _train_and_compare(self, expected_tp, zero_stage): skip_on_device() + assert dist.get_world_size() == 4, (f"this case expects four ranks, got {dist.get_world_size()}") transformers = pytest.importorskip("transformers") hidden, heads, steps, lr = 32, 4, 4, 0.5 kv_rows = 2 * (hidden // heads) @@ -1326,7 +1475,7 @@ def build_model(): for input_ids in batches: loss = reference(input_ids=input_ids, labels=input_ids).loss loss.backward() - if reference_kv_grads is None: + if zero_stage == 0 and reference_kv_grads is None: c_attn = reference.transformer.h[0].attn.c_attn reference_kv_grads = (c_attn.weight.grad[hidden:].clone(), c_attn.bias.grad[hidden:].clone()) reference_optimizer.step() @@ -1343,35 +1492,237 @@ def build_model(): }, # The reference does not clip, and DeepSpeed clips to norm 1.0 by default. "gradient_clipping": 0.0, + "zero_allow_untested_optimizer": True, "tensor_parallel": { - "autotp_size": dist.get_world_size(), + "autotp_size": expected_tp, }, "zero_optimization": { - "stage": 0, + "stage": zero_stage, }, } try: engine, _, _, _ = deepspeed.initialize(model=build_model(), config=ds_config) c_attn = engine.module.transformer.h[0].attn.c_attn + + if zero_stage == 0: + # Every rank must hold its query rows plus the whole K/V block. + expected_query_width = hidden // expected_tp + assert c_attn.weight.shape[0] == expected_query_width + kv_rows + + # Checkpoint metadata must describe the same shard the partition built. + checkpoint_map = c_attn._bigcode_affine_map((hidden + kv_rows, hidden)) + assert checkpoint_map is not None + assert checkpoint_map.shard_shapes[c_attn.tp_index] == tuple(c_attn.weight.shape) + losses = [] for step, input_ids in enumerate(batches): - loss = engine(input_ids=input_ids, labels=input_ids).loss - engine.backward(loss) if step == 0: - # The shared rows' gradient on every rank must be the whole gradient, not - # the part from this rank's heads. - torch.testing.assert_close(c_attn.weight.grad[-kv_rows:], reference_kv_grads[0]) - torch.testing.assert_close(c_attn.bias.grad[-kv_rows:], reference_kv_grads[1]) + with _count_kv_gradient_all_reduces(kv_rows, hidden) as hook_calls: + loss = engine(input_ids=input_ids, labels=input_ids).loss + engine.backward(loss) + assert hook_calls["weight"] > 0, "the K/V weight gradient hook did not run" + if c_attn.bias is not None: + assert hook_calls["bias"] > 0, "the K/V bias gradient hook did not run" + if zero_stage == 0: + # The shared rows' gradient on every rank must be the whole gradient, + # not the part from this rank's heads. + torch.testing.assert_close(c_attn.weight.grad[-kv_rows:], reference_kv_grads[0]) + torch.testing.assert_close(c_attn.bias.grad[-kv_rows:], reference_kv_grads[1]) + else: + loss = engine(input_ids=input_ids, labels=input_ids).loss + engine.backward(loss) engine.step() losses.append(loss.item()) torch.testing.assert_close(torch.tensor(losses), torch.tensor(reference_losses), atol=1e-5, rtol=0) + if zero_stage == 0: + kv_on_ranks = [None] * dist.get_world_size() + dist.all_gather_object(kv_on_ranks, c_attn.weight.detach()[-kv_rows:].cpu()) + assert all(torch.equal(kv, kv_on_ranks[0]) for kv in kv_on_ranks) + finally: + reset_tp_model_init_state() + + +class TestGPTBigCodeMultiQueryAccumulation(DistributedTest): + """The replicated K/V sum is deferred to the accumulation boundary for ZeRO 0/1. + + A window that starts under no_sync keeps the per-micro-batch path; the decision is + latched for the whole window so the two paths cannot mix. + """ + + world_size = 4 + reuse_dist_env = False + + @pytest.mark.parametrize("zero_stage,use_no_sync", [ + (0, False), + (1, False), + (2, False), + (3, False), + (0, True), + (1, True), + ]) + def test_gas2_matches_single_device(self, zero_stage, use_no_sync): + skip_on_device() + assert dist.get_world_size() == 4 + transformers = pytest.importorskip("transformers") + hidden, heads, gas, steps, lr = 32, 4, 2, 3, 0.5 + kv_rows = 2 * (hidden // heads) + device = torch.device(get_accelerator().current_device_name()) + + def build_model(): + torch.manual_seed(0) + config = transformers.GPTBigCodeConfig(vocab_size=64, + n_positions=32, + n_embd=hidden, + n_layer=2, + n_head=heads, + multi_query=True, + attn_pdrop=0.0, + resid_pdrop=0.0, + embd_pdrop=0.0) + return transformers.GPTBigCodeForCausalLM(config) + + batches = [[ + torch.randint(0, 64, (2, 16), generator=torch.Generator().manual_seed(step * gas + micro)).to(device) + for micro in range(gas) + ] for step in range(steps)] + + reference = build_model().to(device) + reference_optimizer = torch.optim.SGD(reference.parameters(), lr=lr) + reference_losses = [] + for step_batches in batches: + reference_optimizer.zero_grad() + for input_ids in step_batches: + loss = reference(input_ids=input_ids, labels=input_ids).loss + (loss / gas).backward() + reference_losses.append(loss.item()) + reference_optimizer.step() + + ds_config = { + "train_micro_batch_size_per_gpu": 2, + "gradient_accumulation_steps": gas, + "optimizer": { + "type": "SGD", + "params": { + "lr": lr, + }, + }, + "gradient_clipping": 0.0, + "zero_allow_untested_optimizer": True, + "tensor_parallel": { + "autotp_size": 2, + }, + "zero_optimization": { + "stage": zero_stage, + }, + } + try: + engine, _, _, _ = deepspeed.initialize(model=build_model(), config=ds_config) + losses = [] + counts_per_step = [] + for step_batches in batches: + with _count_kv_gradient_all_reduces(kv_rows, hidden) as counts: + for micro, input_ids in enumerate(step_batches): + # The canonical DDP pattern: no_sync covers every non-boundary + # micro-batch. The engine latches the deferral decision per window, + # so the boundary must stay on the same per-micro-batch path. + context = engine.no_sync() if use_no_sync and micro < gas - 1 else nullcontext() + with context: + loss = engine(input_ids=input_ids, labels=input_ids).loss + engine.backward(loss) + engine.step() + losses.append(loss.item()) + counts_per_step.append(dict(counts)) + + torch.testing.assert_close(torch.tensor(losses), torch.tensor(reference_losses), atol=1e-5, rtol=0) + # Two layers, weight and bias. A window that starts under no_sync stays on the + # per-micro-batch path for all of its micro-batches. Otherwise ZeRO 0/1 defer the + # reduce to the boundary (two of each per step) and ZeRO 2/3 reduce every + # micro-batch (four of each per step). + deferred = zero_stage in (0, 1) and not use_no_sync + expected = 2 if deferred else 4 + for counts in counts_per_step: + assert counts == {"weight": expected, "bias": expected}, counts + + c_attn = engine.module.transformer.h[0].attn.c_attn kv_on_ranks = [None] * dist.get_world_size() dist.all_gather_object(kv_on_ranks, c_attn.weight.detach()[-kv_rows:].cpu()) assert all(torch.equal(kv, kv_on_ranks[0]) for kv in kv_on_ranks) finally: reset_tp_model_init_state() + @pytest.mark.parametrize("model_dtype,grad_accum_dtype", [("fp16", "fp32"), ("bf16", "fp32")]) + def test_gas2_low_precision_stage1_keeps_kv_replicated(self, model_dtype, grad_accum_dtype): + skip_on_device() + assert dist.get_world_size() == 4 + transformers = pytest.importorskip("transformers") + hidden, heads, gas, steps, lr = 32, 4, 2, 2, 0.5 + kv_rows = 2 * (hidden // heads) + device = torch.device(get_accelerator().current_device_name()) + dtype = torch.float16 if model_dtype == "fp16" else torch.bfloat16 + + def build_model(): + torch.manual_seed(0) + config = transformers.GPTBigCodeConfig(vocab_size=64, + n_positions=32, + n_embd=hidden, + n_layer=2, + n_head=heads, + multi_query=True, + attn_pdrop=0.0, + resid_pdrop=0.0, + embd_pdrop=0.0) + return transformers.GPTBigCodeForCausalLM(config).to(dtype=dtype) + + batches = [[ + torch.randint(0, 64, (2, 16), generator=torch.Generator().manual_seed(step * gas + micro)).to(device) + for micro in range(gas) + ] for step in range(steps)] + + ds_config = { + "train_micro_batch_size_per_gpu": 2, + "gradient_accumulation_steps": gas, + "optimizer": { + "type": "SGD", + "params": { + "lr": lr, + }, + }, + "gradient_clipping": 0.0, + "zero_allow_untested_optimizer": True, + "data_types": { + "grad_accum_dtype": grad_accum_dtype + }, + "tensor_parallel": { + "autotp_size": 2, + }, + "zero_optimization": { + "stage": 1, + }, + } + ds_config[model_dtype] = {"enabled": True} + if model_dtype == "fp16": + ds_config[model_dtype]["initial_scale_power"] = 8 + try: + engine, _, _, _ = deepspeed.initialize(model=build_model(), config=ds_config) + for step_batches in batches: + with _count_kv_gradient_all_reduces(kv_rows, hidden) as counts: + for input_ids in step_batches: + loss = engine(input_ids=input_ids, labels=input_ids).loss + engine.backward(loss) + engine.step() + # fp16/bf16 with fp32 accumulation stores earlier micro-batches outside + # param.grad, so the K/V reduce must stay per micro-batch. + assert counts == {"weight": 4, "bias": 4}, counts + + c_attn = engine.module.transformer.h[0].attn.c_attn + kv_on_ranks = [None] * dist.get_world_size() + dist.all_gather_object(kv_on_ranks, c_attn.weight.detach()[-kv_rows:].float().cpu()) + for kv in kv_on_ranks[1:]: + torch.testing.assert_close(kv, kv_on_ranks[0], atol=1e-2, rtol=1e-2) + finally: + reset_tp_model_init_state() + class TestLegacyLmHeadGatheredTraining(DistributedTest): world_size = 2