Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 59 additions & 11 deletions deepspeed/module_inject/auto_tp.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
from deepspeed import comm as dist
from .layers import *
from deepspeed.accelerator import get_accelerator
from .fusedqkv_utils import require_tp_fused_qkvw
from deepspeed.module_inject.tp_shard import AutoTPMeta, get_shard_size, get_shard_size_list
from .fusedqkv_utils import (require_tp_fused_qkvw, replicates_kv_block, is_replicated_kv_attention,
replicated_kv_group_count, replicated_kv_shard_heads, replicated_kv_shard_meta)
from deepspeed.module_inject.tp_shard import (AutoTPMeta, get_shard_size, get_shard_size_list, _MODULE_KV_HEAD_ATTRS)
from deepspeed.utils import groups
from deepspeed.utils.logging import log_dist, print_dist
from deepspeed.module_inject.layers import is_autotp_training_mode
Expand All @@ -25,6 +26,19 @@
from deepspeed.runtime.zero import GatheredParameters
from deepspeed.runtime.zero.utils import is_zero_param

# Module attributes update_mp_params may shard. _SHARDED_KV_HEAD_ATTRS is the subset that must
# stay whole for replicated-K/V attention; keep it a subset of both this list and the broader
# module attribute probe list so a later edit cannot silently drop one side.
_MP_PARAM_ATTRS = ("n_heads", "inner_dim", "num_heads", "num_kv", "num_attention_heads", "num_attn_heads",
"all_head_size", "embed_dim", "hidden_size", "num_key_value_heads", "num_kv_heads", "kv_n_heads",
"d_model", "num_attention_heads_per_partition", "num_multi_query_groups_per_partition",
"hidden_size_per_partition")
_SHARDED_KV_HEAD_ATTRS = ("num_key_value_heads", "num_kv_heads", "kv_n_heads", "num_kv")
assert set(_SHARDED_KV_HEAD_ATTRS) <= set(_MP_PARAM_ATTRS), \
"_SHARDED_KV_HEAD_ATTRS must be a subset of _MP_PARAM_ATTRS"
assert set(_SHARDED_KV_HEAD_ATTRS) <= set(_MODULE_KV_HEAD_ATTRS), \
"_SHARDED_KV_HEAD_ATTRS must be a subset of _MODULE_KV_HEAD_ATTRS"


def move(tensor, device, copy=True):
if tensor.is_meta:
Expand Down Expand Up @@ -229,15 +243,27 @@ def __init__(self,

self.mp_size = None
self.mp_group = None
self.partition_config = partition_config
# Per-model TP metadata threaded through every layer / helper so each AutoTP instance
# shards by its own model's kv-head / grain values and keeps its own training mode.
# None is kept in the layers so they keep following set_autotp_mode, as before.
self.tp_meta = AutoTPMeta.from_model_config(model_config, tp_grain_size, training=training_mode)
self._replicates_kv_block = replicates_kv_block(self.module)
if self.partition_config is not None and any(
is_replicated_kv_attention(child, self._replicates_kv_block) for child in self.module.modules()):
# update_mp_params still uses model-wide head counts here, which would leave
# ranks without query rows with empty attention attributes.
raise NotImplementedError("AutoTP does not support an explicit partition_config for replicated-K/V "
"attention yet; remove partition_config to use the query-head heuristic.")
if self.partition_config is None:
# Replicated-K/V layouts keep the whole K/V block on every rank, so shard by
# query heads; K/V head attributes stay whole. This assumes one replicated-K/V
# layout per model.
self.tp_meta = replicated_kv_shard_meta(self.tp_meta, self._replicates_kv_block)
self.linear_layer_setting = linear_layer_setting
self.orig_layer_impl = orig_layer_impl
self.linear_policies = None
self.conv_linear_layer = False
self.partition_config = partition_config
self.vocab_parallel_lm_head = vocab_parallel_lm_head
self._training_mode = training_mode
self._vocab_parallel_lm_head_candidate = None
Expand Down Expand Up @@ -406,10 +432,26 @@ def set_tensor_parallel_config(self, mp_size, mp_group):
if self.training_mode:
self.mp_group = groups.get_tensor_model_parallel_group()
self.mp_size = groups.get_tensor_model_parallel_world_size()
return
else:
self.mp_size = mp_size
self.mp_group = mp_group

self.mp_size = mp_size
self.mp_group = mp_group
self._validate_replicated_kv_tp_width()

def _validate_replicated_kv_tp_width(self):
"""Reject a replicated-K/V layout sharded across more ranks than query heads.

Runs after custom config, HF tp_plan, or heuristic AutoTP has been chosen.
"""
if self.partition_config is not None or not self._replicates_kv_block:
return
shard_heads = replicated_kv_shard_heads(self.tp_meta, self._replicates_kv_block)
if shard_heads is not None and self.mp_size > shard_heads:
# A rank beyond the query heads has no query rows and an empty output projection.
raise RuntimeError(f"AutoTP cannot shard this replicated key/value attention across {self.mp_size} ranks: "
f"the model has {shard_heads} query heads, so at least one rank would hold "
f"no query rows and an empty attention output projection. Reduce "
f"tensor_parallel.autotp_size to at most the query head count.")

def _replace(self, child, name, conv_linear_layer):
# This function should clearly define the routing rules for specific layers
Expand Down Expand Up @@ -1002,19 +1044,25 @@ def update_mp_params(self, child, name=None):
if has_unsharded_high_dimensional_param:
setattr(child, "replaced", True)
return
param_list = [
"n_heads", "inner_dim", "num_heads", "num_kv", "num_attention_heads", "num_attn_heads", "all_head_size",
"embed_dim", "hidden_size", "num_key_value_heads", "num_kv_heads", "kv_n_heads", "d_model",
"num_attention_heads_per_partition", "num_multi_query_groups_per_partition", "hidden_size_per_partition"
]
# Replicated-K/V attention follows query heads; K/V aliases stay whole and the
# group count follows the local query shard.
replicated_kv_attention = is_replicated_kv_attention(child, getattr(self, "_replicates_kv_block", False))
param_list = list(_MP_PARAM_ATTRS)
for param in param_list:
if "Yuan" in str(child) and 'embed_dim' in param_list:
param_list.remove('embed_dim')
if hasattr(child, param):
if replicated_kv_attention and param in _SHARDED_KV_HEAD_ATTRS:
continue
param_val = getattr(child, param)
# get_shard_size selects its partitioning strategy from the module name, so the
# attributes must be sharded under the same name as the weights they describe.
setattr(child, param, get_shard_size(param_val, self.mp_size, self.tp_meta, name, rank=tp_index))
# Recompute groups from the local query shard. The module may expose the K/V head count
# under any of the runtime aliases in _MODULE_KV_HEAD_ATTRS.
num_heads = getattr(child, "num_heads", None)
if replicated_kv_attention and hasattr(child, "num_key_value_groups") and num_heads is not None:
child.num_key_value_groups = replicated_kv_group_count(child, num_heads)
setattr(child, "replaced", True)

def update_linear_policies(self):
Expand Down
71 changes: 68 additions & 3 deletions deepspeed/module_inject/fusedqkv_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@
# SPDX-License-Identifier: Apache-2.0

# DeepSpeed Team
from dataclasses import replace
from typing import Optional

import torch
from deepspeed.utils.logging import warning_once
from deepspeed.module_inject.tp_shard import AutoTPMeta, get_shard_size, get_shard_size_list
from deepspeed.module_inject.tp_shard import (AutoTPMeta, get_shard_size, get_shard_size_list, _MODULE_KV_HEAD_ATTRS,
_kv_head_count_from_module)


def split_by_qkvlist_and_refuse(qkv_list, split_size, split_dim=0, cat_dim=0):
Expand Down Expand Up @@ -51,6 +55,68 @@ def get_fused_qkv_type(module):
return FUSED_QKV_TYPE_DICT[max(module_name_matches, key=len)]


_REPLICATED_KV_BLOCK_CLASS_NAMES = tuple(name for name, fused_type in FUSED_QKV_TYPE_DICT.items()
if fused_type == 'bigcodetype')


def replicates_kv_block(module):
"""Whether this model's fused qkv layout keeps the whole K/V block on every rank.

Uses class names instead of ``get_fused_qkv_type`` to avoid stringifying every module.
"""
return any(
any(name in type(child).__name__ for name in _REPLICATED_KV_BLOCK_CLASS_NAMES) for child in module.modules())


def is_replicated_kv_attention(module, replicated_kv):
"""Whether ``module`` is an attention module using the replicated-K/V layout."""
return bool(replicated_kv and getattr(module, "multi_query", False))


def replicated_kv_shard_meta(meta: AutoTPMeta, replicated_kv: bool) -> AutoTPMeta:
"""Metadata whose shard unit is query heads for replicated K/V, otherwise K/V heads.

``get_shard_size`` always reads ``num_kv_heads`` as the shard unit. Replacing that field
keeps the fused-qkv partition, the query-row widths, and the attribute sharding on the same
head count instead of each caller deriving it independently.
"""
if not replicated_kv or meta.num_attention_heads is None:
return meta
return replace(meta, num_kv_heads=meta.num_attention_heads)


def replicated_kv_shard_heads(meta: AutoTPMeta, replicated_kv: bool) -> Optional[int]:
"""Number of attention heads that bound one rank's shard, or None when unspecified."""
if replicated_kv and meta.num_attention_heads is None:
return None
return replicated_kv_shard_meta(meta, replicated_kv).num_kv_heads


def replicated_kv_query_widths(meta: AutoTPMeta, mp_size: int):
"""Per-rank query-row widths for a replicated-K/V fused qkv layout, or None without n_embd."""
shard_meta = replicated_kv_shard_meta(meta, True)
if shard_meta.n_embd is None:
return None
return get_shard_size_list(shard_meta.n_embd, mp_size, shard_meta)


def replicated_kv_row_start(meta: AutoTPMeta, mp_size: int, rank: int) -> Optional[int]:
"""First K/V row in ``rank``'s local replicated-K/V shard."""
query_widths = replicated_kv_query_widths(meta, mp_size)
if query_widths is None:
return None
return query_widths[rank]


def replicated_kv_group_count(module, local_num_heads: int) -> int:
"""Query-to-K/V group count after the query heads have been sharded."""
kv_heads = _kv_head_count_from_module(module)
if not kv_heads:
raise RuntimeError(f"AutoTP cannot recompute num_key_value_groups for {type(module).__name__}: "
f"none of the module K/V head attributes {_MODULE_KV_HEAD_ATTRS} is set.")
return local_num_heads // kv_heads


def fused_qkv_subparam_sizes(module, weight_shape, meta: AutoTPMeta):
"""Sizes of the sub-parameters a fused qkv weight is cut into, or None.

Expand Down Expand Up @@ -163,8 +229,7 @@ def _bigcode_type_transpose(input, mp_size, meta=meta):
assert n_embd is not None
q = input[:n_embd]
kv = input[n_embd:]
shape = q.shape
split_q = q.split(get_shard_size_list(shape[0], mp_size, meta), dim=0)
split_q = q.split(replicated_kv_query_widths(meta, mp_size), dim=0)
return torch.cat((split_q[gpu_index], kv), dim=0)

def _phi3_type_transpose(input, mp_size, meta=meta):
Expand Down
143 changes: 132 additions & 11 deletions deepspeed/module_inject/layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from abc import ABC, abstractmethod
from typing import Iterable, Any, Optional, List, Tuple, Dict
from .fusedqkv_utils import (shard_value_with_share_qk, prepare_tp_fused_qkvw, fused_qkv_subparam_sizes,
set_fused_qkv_shard_state)
set_fused_qkv_shard_state, replicated_kv_query_widths, replicated_kv_row_start)
from deepspeed.runtime.tensor_parallel import AUTOTP_MODE
from deepspeed.checkpoint.constants import (DS_AUTOTP_UC_META, UNIVERSAL_CHECKPOINT_VERSION_KEY,
UNIVERSAL_CHECKPOINT_VERSION_VALUE)
Expand Down Expand Up @@ -1467,6 +1467,124 @@ def _mark_uc_metadata(self):
is_bias=True)


def _all_reduce_replicated_kv_rows(kv_grad, group):
"""All-reduce the replicated K/V rows in place.

Some backends return this gradient non-contiguously, which collectives reject; reduce a
contiguous copy in that case and preserve the original layout for autograd.
"""
if kv_grad.numel() == 0:
# The key/value block is replicated whole, so it is empty on every rank at once.
return kv_grad
if kv_grad.is_contiguous():
# Contiguous: reduce the view in place.
dist.all_reduce(kv_grad, group=group)
else:
# Non-contiguous: reduce a copy, then write it back.
reduced = kv_grad.contiguous()
dist.all_reduce(reduced, group=group)
kv_grad.copy_(reduced)
return kv_grad


def _sum_replicated_kv_gradient(grad, kv_start, group):
"""Sum the K/V gradient rows that every rank holds a copy of."""
grad = grad.clone()
_all_reduce_replicated_kv_rows(grad.narrow(0, kv_start, grad.shape[0] - kv_start), group)
return grad


class _ReplicatedKvGradReducer:
"""Sum each micro-batch's replicated K/V gradient exactly once.

A plain tensor hook reduces the incoming gradient before PyTorch accumulates it, which is
correct with gradient accumulation. Some backends skip that hook under ZeRO 1/2, so a
post-accumulate hook is registered as well. The fallback reduces only the K/V rows added
since its previous call: reducing the whole accumulated gradient again would scale earlier
micro-batches once per later backward.

When the engine allows deferred reduction, both hooks leave the local partial in place and
the post-accumulate hook reduces the K/V rows that were accumulated since the previous
boundary. The engine only allows this when the full window stays in ``param.grad``.
"""

def __init__(self, param, kv_start, group):
self.param = param
self.kv_start = kv_start
self.group = group
self._tensor_hook_ran = False
self._grad = None
self._kv = None
self._boundary_reduced = False
self._accumulation_boundary = True
self._defer_allowed = False

def set_accumulation_state(self, is_boundary, defer_allowed):
"""Record the engine's accumulation state for this backward.

Called from the engine's backward prologue for this reducer only, so two engines in
one process do not overwrite each other's state. A non-boundary backward starts a new
window: even a parameter that is unused there must forget the previous boundary's
reduced snapshot.
"""
self._accumulation_boundary = is_boundary
self._defer_allowed = defer_allowed
if not is_boundary:
self._tensor_hook_ran = False
self._grad = None
self._kv = None
self._boundary_reduced = False

def tensor_hook(self, grad):
self._tensor_hook_ran = True
if self._defer_allowed:
return grad
return _sum_replicated_kv_gradient(grad, self.kv_start, self.group)

def _reduce_new_kv_rows(self, grad, kv_grad, use_delta):
if use_delta and self._grad is grad and self._kv is not None:
delta = kv_grad - self._kv
else:
delta = kv_grad.clone()
_all_reduce_replicated_kv_rows(delta, self.group)
if use_delta and self._grad is grad and self._kv is not None:
kv_grad.copy_(self._kv + delta)
else:
kv_grad.copy_(delta)
self._grad = grad
self._kv = kv_grad.clone()

def post_accumulate_hook(self, *_unused):
# Both hook forms leave the gradient on the parameter; the pre-2.1
# AccumulateGrad hook also passes (grad_inputs, grad_outputs).
grad = self.param.grad
if grad is None:
self._tensor_hook_ran = False
self._grad = None
self._kv = None
self._boundary_reduced = False
return
kv_grad = grad.narrow(0, self.kv_start, grad.shape[0] - self.kv_start)
if self._defer_allowed:
if not self._accumulation_boundary:
# The next window starts here; forget the previous boundary's snapshot.
self._tensor_hook_ran = False
self._grad = None
self._kv = None
return
# A reentrant checkpoint may run this hook more than once at the boundary.
# Reducing the delta keeps each new local contribution counted once.
self._reduce_new_kv_rows(grad, kv_grad, use_delta=self._boundary_reduced)
self._boundary_reduced = True
self._tensor_hook_ran = False
return
if self._tensor_hook_ran:
# The tensor hook already summed this micro-batch before it was accumulated.
self._tensor_hook_ran = False
else:
self._reduce_new_kv_rows(grad, kv_grad, use_delta=True)


class FusedModuleWrapper:

def __init__(self, fused_module: nn.Module):
Expand Down Expand Up @@ -1505,21 +1623,22 @@ def _sum_replicated_kv_gradients(self):

if get_fused_qkv_type(self.fused_module.module) != 'bigcodetype' or not pg_utils.is_parallel(self.mp_group):
return
n_embd = self.tp_meta.n_embd
if n_embd is None:
return
# This rank's shard is its query rows followed by the whole key/value block.
kv_start = get_shard_size_list(n_embd, self.tp_world_size, self.tp_meta)[self.tp_index]
kv_start = replicated_kv_row_start(self.tp_meta, self.tp_world_size, self.tp_index)
if kv_start is None:
return
group = self.mp_group

def sum_kv_rows(grad):
grad = grad.clone()
dist.all_reduce(grad.narrow(0, kv_start, grad.shape[0] - kv_start), group=group)
return grad
from deepspeed.utils.torch import register_grad_hook

# The tensor hook handles the common path before accumulation; the post-accumulate
# fallback covers torch_npu, which skipped it under ZeRO 1/2 in our tests.
for param in (self.weight, self.bias):
if param is not None and param.requires_grad:
param.register_hook(sum_kv_rows)
reducer = _ReplicatedKvGradReducer(param, kv_start, group)
param.register_hook(reducer.tensor_hook)
register_grad_hook(param, reducer.post_accumulate_hook)
param._ds_replicated_kv_reducer = reducer

def _freeze_partition_sizes(self, total_size):
super()._freeze_partition_sizes(total_size)
Expand Down Expand Up @@ -1574,7 +1693,9 @@ def _bigcode_affine_map(self, shape):
# The query rows are not split evenly: the widths follow the head count and the grain
# size, so they have to come from the same helper the partition calls rather than
# from a division here.
query_widths = list(get_shard_size_list(n_embd, self.tp_world_size, self.tp_meta))
query_widths = replicated_kv_query_widths(self.tp_meta, self.tp_world_size)
if query_widths is None:
return None
return segmented_map(shape, [(n_embd, False), (shape[0] - n_embd, True)],
0,
self.tp_world_size,
Expand Down
Loading
Loading