From 5ff7a502577869d42c8b20cf3ce8c52d5b9d6812 Mon Sep 17 00:00:00 2001 From: RunguoLi Date: Sun, 8 Nov 2026 01:07:51 -0600 Subject: [PATCH 1/3] Refresh a tensor-parallel inference copy of a ZeRO model through affine plans AffineWeightSync plans each parameter once with plan_transfer, from the trainer's storage maps into a tensor-parallel layout held by every inference replica, and runs every sync as one execute_transfers batch, so no rank gathers a whole parameter. WeightArena keeps a rank's shards in one buffer that is never reallocated, so CUDA graphs captured on it stay valid across syncs. Signed-off-by: RunguoLi --- .../runtime/rollout/affine_weight_sync.py | 224 +++++++++++++++ docs/code-docs/source/affine-transfer.rst | 43 +++ .../rollout/test_affine_weight_sync.py | 272 ++++++++++++++++++ 3 files changed, 539 insertions(+) create mode 100644 deepspeed/runtime/rollout/affine_weight_sync.py create mode 100644 tests/unit/runtime/rollout/test_affine_weight_sync.py diff --git a/deepspeed/runtime/rollout/affine_weight_sync.py b/deepspeed/runtime/rollout/affine_weight_sync.py new file mode 100644 index 000000000000..e4f4e2a99df2 --- /dev/null +++ b/deepspeed/runtime/rollout/affine_weight_sync.py @@ -0,0 +1,224 @@ +# Copyright (c) DeepSpeed Team. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +"""Keep a tensor-parallel inference copy of a training model current, shard to shard, at fixed addresses. + +A rollout engine that generates with its own tensor-parallel copy of the weights refreshes that copy after every +optimizer step. Gathering each parameter whole and cutting out the local shard sends every parameter to every rank. +Instead, each parameter is described twice with affine maps (:mod:`deepspeed.checkpoint.affine`): how the trainer +stores it (:func:`zero_param_shaped_storage_map`, or ``autotp_storage_affine_maps`` for an AutoTP model) and which +block each inference rank needs (:func:`replicated_layout_map`). +:func:`~deepspeed.checkpoint.affine_transfer.plan_transfer` composes the two into copies from trainer shards straight +into inference shards, once per layout, and every sync runs the copies of all parameters as one +:func:`~deepspeed.checkpoint.affine_transfer_executor.execute_transfers` batch. + +CUDA graphs record raw device pointers, so a decode graph captured on the inference copy stays valid only while every +update writes into the storage it was captured on. The executor writes targets in place, and :class:`WeightArena` +keeps a rank's shards as views of one buffer that is never reallocated. +""" + +from typing import Dict, Iterator, Mapping, Optional, Sequence, Set, Tuple + +import torch + +import deepspeed.comm as dist +from deepspeed.checkpoint.affine import ParamAffineMap, nest_affine_maps, replicated_map +from deepspeed.checkpoint.affine_transfer import PlanBudget, ReplicaSelectionRequest, TransferPlan, plan_transfer +from deepspeed.checkpoint.affine_transfer_executor import (StagingBudget, TransferEndpoints, TransferRequest, + TransferStats, execute_transfers) +from deepspeed.checkpoint.zero_affine import zero_param_storage_affine_map, zero_storage_affine_map + +__all__ = ['WeightArena', 'zero_param_shaped_storage_map', 'replicated_layout_map', 'AffineWeightSync'] + +# A rank's tensor for each parameter it holds, by parameter name. +Buffers = Mapping[str, torch.Tensor] + +# Each view starts on a 256-byte boundary, as allocator blocks do, so kernels see the alignment they expect. +_ALIGN_BYTES = 256 + + +def _numel(shape: Sequence[int]) -> int: + numel = 1 + for size in shape: + numel *= size + return numel + + +class WeightArena: + """Named views of one flat buffer that is never reallocated. + + Args: + shapes: ``{name: shape}`` in allocation order. + dtype, device: of the buffer. + """ + + def __init__(self, shapes: Mapping[str, Sequence[int]], dtype: torch.dtype, device=None): + element_size = torch.empty(0, dtype=dtype).element_size() + align = max(1, _ALIGN_BYTES // element_size) + offsets = {} + total = 0 + for name, shape in shapes.items(): + total = -(-total // align) * align + offsets[name] = total + total += _numel(shape) + # Zeros, so that padding a kernel may read (rows past the end of a vocabulary shard) holds no garbage. An + # allocation is not always 256-byte aligned (the CPU allocator aligns to 64 bytes), so allocate one alignment + # more and start the buffer at its first aligned element. + storage = torch.zeros(total + align, dtype=dtype, device=device) + skip = -(storage.data_ptr() // element_size) % align + self.buffer = storage.narrow(0, skip, total) + self._views = {} + for name, offset in offsets.items(): + shape = tuple(shapes[name]) + self._views[name] = self.buffer.narrow(0, offset, _numel(shape)).view(shape) + + def __getitem__(self, name: str) -> torch.Tensor: + return self._views[name] + + def items(self) -> Iterator[Tuple[str, torch.Tensor]]: + return iter(self._views.items()) + + +def zero_param_shaped_storage_map(param) -> ParamAffineMap: + """What every rank stores of a ZeRO model's ``param``, in the parameter's own shape and keyed by global rank. + + ``zero_param_storage_affine_map`` describes the flattened parameter, while a tensor-parallel layout describes it + in its own shape, and both sides of a plan must describe the same logical tensor. This rank's shard is still + ``zero_param_storage_shard(param)``. + """ + shape = tuple(getattr(param, 'ds_shape', param.shape)) + return nest_affine_maps(replicated_map(shape, 1), {0: zero_param_storage_affine_map(param)}) + + +def replicated_layout_map(layout: ParamAffineMap, replicas: Sequence[Sequence[int]]) -> ParamAffineMap: + """A parameter's tensor-parallel layout, held by every replica of the inference copy, keyed by global rank. + + ``layout`` describes one replica, keyed by tensor-parallel rank ``0 .. tp - 1``: an AutoTP layer's layout map, or + one built with :mod:`deepspeed.checkpoint.affine` (``contiguous_split_map``, ``replicated_map``, ...). + ``replicas[i][j]`` is the global rank that is tensor-parallel rank ``j`` of replica ``i``. A rank's shard is its + layout shard flattened, so a contiguous tensor of the layout shard's shape holds it. + """ + tp_size = len(layout.shard_shapes) + if sorted(layout.shard_shapes) != list(range(tp_size)): + raise ValueError(f'The layout must be keyed by tensor-parallel ranks 0..{tp_size - 1}, ' + f'not {sorted(layout.shard_shapes)}.') + if not replicas or any(len(ranks) != tp_size for ranks in replicas): + raise ValueError(f'Every replica needs {tp_size} ranks, one per tensor-parallel rank: {list(replicas)}.') + # Rank j of every replica stores layout shard j whole, as data-parallel replicas store a parameter. + storage = {} + for tp_rank, shape in layout.shard_shapes.items(): + storage[tp_rank] = zero_storage_affine_map(_numel(shape), [ranks[tp_rank] for ranks in replicas]) + return nest_affine_maps(layout, storage) + + +class _LocalFirstLeastLoaded: + """Replica selector: a holder in the target's own process, otherwise the holder that sends the fewest elements so + far, ties to the lowest rank. + + Load counts what each rank sends to other processes, as #8936's ``local_first_least_loaded`` does: local copies + use no link and do not count, and regions with a single holder do, since that load cannot move. ``plan_transfer`` + consults the selector only where a region has several holders, so :meth:`settle` recounts the load from each + finished plan; within one parameter, its single-holder regions are counted only once it is planned. + """ + + def __init__(self): + self.load: Dict[int, int] = {} + + def __call__(self, request: ReplicaSelectionRequest) -> int: + holders = [candidate.source_rank for candidate in request.candidates] + if request.target_rank in holders: + return request.target_rank + chosen = min(holders, key=lambda rank: (self.load.get(rank, 0), rank)) + self.load[chosen] = self.load.get(chosen, 0) + _numel(request.shape) + return chosen + + def settle(self, before: Dict[int, int], plan: TransferPlan) -> None: + """Set the load to ``before`` plus everything ``plan`` sends.""" + self.load = dict(before) + for segment in plan.segments: + if segment.source_rank != segment.target_rank: + self.load[segment.source_rank] = self.load.get(segment.source_rank, 0) + segment.numel + + +class AffineWeightSync: + """Copies parameters from where one side stores them into where the other side needs them, planned once. + + Args: + source_maps: ``{name: map}`` of what the source side stores, keyed by global rank, for example + :func:`zero_param_shaped_storage_map` of each parameter of a ZeRO model. + target_maps: ``{name: map}`` of what the target side needs, keyed by global rank, for example from + :func:`replicated_layout_map`. It names the same parameters as ``source_maps``. + dtype: of every buffer; ranks that hold no buffer must agree on it too. + group: the process group the copies cross, the world by default. It must contain every rank the maps name, + and all of its ranks construct the sync with the same maps. + budget: bytes staged per peer and direction, see + :class:`~deepspeed.checkpoint.affine_transfer_executor.StagingBudget`. + limits: planning limits, see :class:`~deepspeed.checkpoint.affine_transfer.PlanBudget`. + + A block several ranks hold is read from the target's own process when it holds one (a local copy), and otherwise + from the holder that sends the fewest elements so far, over all parameters (see :class:`_LocalFirstLeastLoaded`). + Planning has no communication and is deterministic, so every rank derives the same plans. + """ + + def __init__(self, + source_maps: Mapping[str, ParamAffineMap], + target_maps: Mapping[str, ParamAffineMap], + dtype: torch.dtype, + group=None, + budget: StagingBudget = StagingBudget(), + limits: Optional[PlanBudget] = None): + if set(source_maps) != set(target_maps): + only_one = sorted(set(source_maps) ^ set(target_maps)) + raise ValueError(f'Source and target maps must name the same parameters; only one side names {only_one}.') + self.dtype, self.group, self.budget = dtype, group, budget + self.rank = dist.get_rank() + if group is None: + members = list(range(dist.get_world_size())) + else: + members = [dist.get_global_rank(group, rank) for rank in range(dist.get_world_size(group=group))] + process = {rank: index for index, rank in enumerate(members)} + source_ranks = {rank for affine_map in source_maps.values() for rank in affine_map.shard_shapes} + target_ranks = {rank for affine_map in target_maps.values() for rank in affine_map.shard_shapes} + outside = sorted((source_ranks | target_ranks) - set(process)) + if outside: + raise ValueError(f'Ranks {outside} hold parameters but are not in the transfer group.') + self.endpoints = TransferEndpoints(source={rank: process[rank] + for rank in sorted(source_ranks)}, + target={rank: process[rank] + for rank in sorted(target_ranks)}) + # The parameters this rank takes part in on each side. + self._sources_here = {name for name, affine_map in source_maps.items() if self.rank in affine_map.shard_shapes} + self._targets_here = {name for name, affine_map in target_maps.items() if self.rank in affine_map.shard_shapes} + selector = _LocalFirstLeastLoaded() + self.plans = {} + for name in sorted(source_maps): + before = dict(selector.load) + self.plans[name] = plan_transfer(target_maps[name], source_maps[name], limits, replica_selector=selector) + selector.settle(before, self.plans[name]) + + def sync(self, source_buffers: Buffers, target_buffers: Buffers) -> TransferStats: + """Copy every parameter. Collective over ``group``. + + ``source_buffers[name]`` is this rank's source shard of ``name`` and ``target_buffers[name]`` its target + shard, for every parameter the maps place on this rank; entries for other parameters are ignored. Shards are + contiguous tensors, which may be views (a view keeps its offset into its storage). The sources must not change + while the copies run. A rank that lacks a shard refuses the batch in the executor's collective preflight, so + every rank raises and no data moves. + """ + requests = [] + for name, plan in self.plans.items(): + sources, source_error = self._local(name, self._sources_here, source_buffers, 'source') + targets, target_error = self._local(name, self._targets_here, target_buffers, 'target') + # Raising here would leave the other ranks waiting in the preflight, so the executor raises everywhere. + error = source_error or target_error + requests.append(TransferRequest(name, plan, sources, targets, preflight_error=error)) + return execute_transfers(requests, self.endpoints, self.dtype, self.budget, self.group) + + def _local(self, name: str, here: Set[str], buffers: Buffers, side: str): + """``({rank: shard}, None)`` for this rank's side of ``name``, or ``({}, error)`` when the shard is missing.""" + if name not in here: + return {}, None + if name not in buffers: + return {}, f'rank {self.rank} holds part of {name!r} but was given no {side} shard for it' + return {self.rank: buffers[name]}, None diff --git a/docs/code-docs/source/affine-transfer.rst b/docs/code-docs/source/affine-transfer.rst index 6f27352ae90a..7f478a5f45e0 100644 --- a/docs/code-docs/source/affine-transfer.rst +++ b/docs/code-docs/source/affine-transfer.rst @@ -46,3 +46,46 @@ zero-copy transport such as GPUDirect RDMA would instead take each round's removes the arena copy but needs a registration lifetime this API has no place for, so the wire contract (one packed message per peer per round) is expected to change when such a path lands. + +Refreshing an inference copy +---------------------------- + +A rollout engine that generates with its own tensor-parallel copy of a ZeRO +model refreshes that copy after every optimizer step. +``deepspeed.runtime.rollout.affine_weight_sync`` does it shard to shard: no +rank gathers a whole parameter. Both sides are maps keyed by global rank. The +trainer side comes from ``zero_param_shaped_storage_map`` (or +``autotp_storage_affine_maps`` for an AutoTP model). The inference side is a +tensor-parallel layout held by every replica, from +``replicated_layout_map``:: + + from deepspeed.checkpoint.affine import contiguous_split_map + from deepspeed.checkpoint.zero_affine import zero_param_storage_shard + from deepspeed.runtime.rollout.affine_weight_sync import ( + AffineWeightSync, WeightArena, replicated_layout_map, + zero_param_shaped_storage_map) + + replicas = [[0, 1], [2, 3]] # two replicas of TP=2 + layouts = {name: contiguous_split_map(...) for name in names} + params = dict(engine.module.named_parameters()) + sync = AffineWeightSync( + {name: zero_param_shaped_storage_map(params[name]) for name in names}, + {name: replicated_layout_map(layouts[name], replicas) for name in names}, + torch.bfloat16) + arena = WeightArena({name: layouts[name].shard_shapes[tp_rank] for name in names}, + torch.bfloat16, device) + + engine.empty_partition_cache() # ZeRO-3: release gathered copies first + sync.sync({name: zero_param_storage_shard(params[name]) for name in names}, + dict(arena.items())) + +Plans are made once, when the sync is built, without communication. +``plan_transfer`` plans each parameter. A block that several ranks hold is read +from the target's own process if that process holds it, and otherwise from the +holder that sends the least so far; local copies do not count as load. ``sync`` +is collective: it runs every parameter's copies as one ``execute_transfers`` +batch, and a rank that lacks one of its shards makes every rank refuse the batch +before any data moves. ``WeightArena`` keeps a +rank's shards as views of one buffer that is never reallocated, and the +executor writes targets in place. A CUDA graph captured on the arena therefore +reads the new weights after every sync. Maintenance contact: ``@LiRunGuo``. diff --git a/tests/unit/runtime/rollout/test_affine_weight_sync.py b/tests/unit/runtime/rollout/test_affine_weight_sync.py new file mode 100644 index 000000000000..ddcd52cd2527 --- /dev/null +++ b/tests/unit/runtime/rollout/test_affine_weight_sync.py @@ -0,0 +1,272 @@ +# Copyright (c) DeepSpeed Team. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +"""Refreshing a tensor-parallel copy of a ZeRO model through affine plans.""" + +import pytest +import torch + +import deepspeed +import deepspeed.comm as dist +from deepspeed.accelerator import get_accelerator +from deepspeed.checkpoint.affine import contiguous_split_map, replicated_map +from deepspeed.checkpoint.affine_transfer_executor import AffineTransferExecutionError +from deepspeed.checkpoint.zero_affine import zero_param_storage_shard +from deepspeed.runtime.rollout.affine_weight_sync import (AffineWeightSync, WeightArena, replicated_layout_map, + zero_param_shaped_storage_map) +from unit.common import DistributedTest +from unit.simple_model import random_dataloader + +HIDDEN, INNER = 13, 18 + + +class MLPBlock(torch.nn.Module): + """A column-parallel and a row-parallel linear and a norm, as a transformer MLP is split.""" + + def __init__(self): + super().__init__() + self.up = torch.nn.Linear(HIDDEN, INNER) + self.down = torch.nn.Linear(INNER, HIDDEN) + self.norm = torch.nn.LayerNorm(HIDDEN) + + def forward(self, x, y): + return torch.nn.functional.cross_entropy(self.norm(self.down(torch.relu(self.up(x)))), y) + + +def _even(size, parts): + base, extra = divmod(size, parts) + return [base + (part < extra) for part in range(parts)] + + +def _layouts(tp_size): + # INNER does not divide by 4, so TP=4 shards are uneven. + return { + 'up.weight': contiguous_split_map((INNER, HIDDEN), _even(INNER, tp_size), 0), + 'up.bias': contiguous_split_map((INNER, ), _even(INNER, tp_size), 0), + 'down.weight': contiguous_split_map((HIDDEN, INNER), _even(INNER, tp_size), 1), + 'down.bias': replicated_map((HIDDEN, ), tp_size), + 'norm.weight': replicated_map((HIDDEN, ), tp_size), + 'norm.bias': replicated_map((HIDDEN, ), tp_size), + } + + +def _full_param(param): + if hasattr(param, 'ds_id'): + with deepspeed.zero.GatheredParameters([param]): + return param.data.detach().clone() + return param.data.detach().clone() + + +def _total(value): + tensor = torch.tensor([value], dtype=torch.int64, device=get_accelerator().current_device_name()) + dist.all_reduce(tensor) + return int(tensor.item()) + + +def _all(value): + gathered = [None] * dist.get_world_size() + dist.all_gather_object(gathered, value) + return gathered + + +def _tp_rank(replicas, rank): + for ranks in replicas: + if rank in ranks: + return ranks.index(rank) + return None + + +class TestAffineWeightSync(DistributedTest): + world_size = 4 + + # Two TP=2 replicas, one TP=4 replica, and two TP=1 replicas on half of the ranks. + @pytest.mark.parametrize('replicas', [[[0, 1], [2, 3]], [[0, 1, 2, 3]], [[3], [1]]]) + @pytest.mark.parametrize('zero_stage', [1, 2, 3]) + def test_sync_writes_every_replica_in_place(self, zero_stage, replicas): + dtype = torch.bfloat16 if get_accelerator().is_bf16_supported() else torch.float32 + config = { + 'train_micro_batch_size_per_gpu': 1, + 'optimizer': { + 'type': 'Adam', + 'params': { + 'lr': 1e-2, + 'torch_adam': True + } + }, + 'zero_optimization': { + 'stage': zero_stage + }, + } + if dtype == torch.bfloat16: + config['bf16'] = {'enabled': True} + if zero_stage == 3: + with deepspeed.zero.Init(config_dict_or_path=config): + model = MLPBlock() + else: + model = MLPBlock() + engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=config) + params = dict(engine.module.named_parameters()) + layouts = _layouts(len(replicas[0])) + rank = dist.get_rank() + tp_rank = _tp_rank(replicas, rank) + + source_maps = {name: zero_param_shaped_storage_map(param) for name, param in params.items()} + target_maps = {name: replicated_layout_map(layouts[name], replicas) for name in params} + sync = AffineWeightSync(source_maps, target_maps, dtype) + arena = None + if tp_rank is not None: + shapes = {name: layouts[name].shard_shapes[tp_rank] for name in params} + arena = WeightArena(shapes, dtype, engine.device) + + synced = [] + for step in range(2): + if zero_stage == 3: + engine.empty_partition_cache() + sources = {name: zero_param_storage_shard(param) for name, param in params.items()} + targets = dict(arena.items()) if arena is not None else {} + stats = sync.sync(sources, targets) + if zero_stage < 3: + # Every rank holds a replica of every parameter, so each target is filled from its own process. + assert _total(stats.sent_bytes) == 0 + for name, param in params.items(): + full = _full_param(param) + if arena is not None: + assert torch.equal(arena[name], layouts[name].extract(full, tp_rank)), name + if arena is not None: + synced.append((arena.buffer.data_ptr(), arena.buffer.clone())) + for x, y in random_dataloader(engine, 1, HIDDEN, engine.device, dtype): + engine.backward(engine(x, y)) + engine.step() + + if arena is not None: + (first_address, first_values), (second_address, second_values) = synced + assert second_address == first_address + # Otherwise the second sync proves nothing about writing in place. + assert not torch.equal(second_values, first_values) + engine.destroy() + + def test_reads_of_replicated_parameters_spread_over_their_holders(self): + # Ranks 0-2 each hold all six parameters whole, and rank 3 needs them all. + shapes = {f'w{index}': (4, 6) for index in range(6)} + rank = dist.get_rank() + device = get_accelerator().current_device_name() + source_maps, target_maps = {}, {} + for name, shape in shapes.items(): + source_maps[name] = replicated_layout_map(replicated_map(shape, 1), [[0], [1], [2]]) + target_maps[name] = replicated_layout_map(replicated_map(shape, 1), [[3]]) + sync = AffineWeightSync(source_maps, target_maps, torch.float32) + full = { + name: torch.arange(24, dtype=torch.float32, device=device).view(shape) + 100 * index + for index, (name, shape) in enumerate(shapes.items()) + } + arena = WeightArena(shapes, torch.float32, device) if rank == 3 else None + + targets = dict(arena.items()) if arena is not None else {} + # Every rank passes the same sources; rank 3 stores none, so its entries are ignored. + stats = sync.sync(full, targets) + + sent = _all(stats.sent_bytes) + assert sent[0] == sent[1] == sent[2] == 2 * 24 * 4 and sent[3] == 0 + if arena is not None: + for name in shapes: + assert torch.equal(arena[name], full[name]) + + def test_local_copies_are_free_and_single_holders_count_as_load(self): + # Rank 0 alone holds 'a', which rank 3 needs, so rank 0 must send it. Ranks 0 and 1 hold 'b', which ranks 1 + # and 3 need: rank 1 copies its own, which uses no link, so rank 3 reads 'b' from rank 1, which sends less. + shape = (4, 6) + rank = dist.get_rank() + device = get_accelerator().current_device_name() + whole = replicated_map(shape, 1) + source_maps = {'a': replicated_layout_map(whole, [[0]]), 'b': replicated_layout_map(whole, [[0], [1]])} + target_maps = {'a': replicated_layout_map(whole, [[3]]), 'b': replicated_layout_map(whole, [[1], [3]])} + sync = AffineWeightSync(source_maps, target_maps, torch.float32) + full = {name: torch.full(shape, float(index + 1), device=device) for index, name in enumerate(source_maps)} + needed = [name for name in target_maps if rank in target_maps[name].shard_shapes] + arena = WeightArena({name: shape for name in needed}, torch.float32, device) if needed else None + targets = dict(arena.items()) if arena is not None else {} + + stats = sync.sync(full, targets) + + assert _all(stats.sent_bytes) == [24 * 4, 24 * 4, 0, 0] + for name in needed: + assert torch.equal(arena[name], full[name]), name + + def test_a_rank_missing_a_shard_makes_every_rank_refuse_before_anything_moves(self): + shape = (4, 6) + rank = dist.get_rank() + device = get_accelerator().current_device_name() + sync = AffineWeightSync({'w': replicated_layout_map(replicated_map(shape, 1), [[0]])}, + {'w': replicated_layout_map(contiguous_split_map(shape, [2, 2], 0), [[2, 3]])}, + torch.float32) + arena = WeightArena({'w': (2, 6)}, torch.float32, device) if rank in (2, 3) else None + # Rank 3 forgets its target shard. Had it raised alone, the other ranks would wait for it in the preflight. + targets = dict(arena.items()) if arena is not None and rank != 3 else {} + with pytest.raises(AffineTransferExecutionError): + sync.sync({'w': torch.ones(shape, device=device)}, targets) + if rank == 2: + assert torch.all(arena['w'] == 0) + + def test_a_graph_captured_before_a_sync_reads_the_synced_weights(self): + if get_accelerator().device_name() != 'cuda': + pytest.skip('requires CUDA graphs') + shape = (6, 8) + layout = contiguous_split_map(shape, [3, 3], 0) + rank = dist.get_rank() + device = get_accelerator().current_device_name() + sync = AffineWeightSync({'w': replicated_layout_map(replicated_map(shape, 1), [[0]])}, + {'w': replicated_layout_map(layout, [[2, 3]])}, torch.float32) + tp_rank = _tp_rank([[2, 3]], rank) + arena = None + if tp_rank is not None: + arena = WeightArena({'w': layout.shard_shapes[tp_rank]}, torch.float32, device) + x = torch.randn(4, shape[1], device=device) + if arena is not None: + # Warm up outside the graph, as capture requires. + y = x @ arena['w'].t() + graph = get_accelerator().create_graph() + with get_accelerator().capture_to_graph(graph): + y = x @ arena['w'].t() + + for version in range(2): + full = torch.randn(shape, device=device) + version + targets = dict(arena.items()) if arena is not None else {} + sync.sync({'w': full}, targets) + # Every rank needs the source values to check its shard. + dist.broadcast(full, 0) + if arena is not None: + get_accelerator().replay_graph(graph) + assert torch.equal(y, x @ layout.extract(full, tp_rank).t()) + + +def test_arena_views_are_aligned_disjoint_and_zeroed(): + shapes = {'a': (3, 5), 'b': (7, ), 'c': (2, 2, 3)} + # Several arenas alive at once, at different addresses: an allocation may happen to be aligned. + arenas = [WeightArena(shapes, torch.float16) for _ in range(8)] + for arena in arenas: + assert torch.all(arena.buffer == 0) + for index, (name, view) in enumerate(arena.items()): + assert view.shape == shapes[name] + assert view.data_ptr() % 256 == 0, name + view.fill_(index + 1) + # A view that overlapped another would have overwritten part of it. + for index, (name, view) in enumerate(arena.items()): + assert torch.all(view == index + 1), name + + +def test_replicated_layout_map_gives_each_replica_its_layout_shard(): + full = torch.arange(60, dtype=torch.float32).view(6, 10) + layout = contiguous_split_map((6, 10), [4, 3, 3], 1) + replicas = [[5, 0, 3], [1, 4, 2]] + target = replicated_layout_map(layout, replicas) + for ranks in replicas: + for tp_rank, rank in enumerate(ranks): + shard = target.extract(full, rank).view(layout.shard_shapes[tp_rank]) + assert torch.equal(shard, layout.extract(full, tp_rank)) + + +@pytest.mark.parametrize('replicas', [[[0, 1], [2]], [[0, 1], [1, 2]], []]) +def test_replicated_layout_map_refuses_replicas_that_do_not_fit_the_layout(replicas): + with pytest.raises(ValueError): + replicated_layout_map(contiguous_split_map((4, 4), [2, 2], 0), replicas) From 70831cbb1c73b905b7fba9044257bec021b7a57d Mon Sep 17 00:00:00 2001 From: RunguoLi Date: Tue, 10 Nov 2026 19:33:12 -0600 Subject: [PATCH 2/3] Review notes: the replica choice changes only the cost; maps keyed by global rank with any group; AutoTP source untested Test that a custom group keeps the maps keyed by global rank. Signed-off-by: RunguoLi --- .../runtime/rollout/affine_weight_sync.py | 16 +++++++++--- docs/code-docs/source/affine-transfer.rst | 11 +++++--- .../rollout/test_affine_weight_sync.py | 25 ++++++++++++++++++- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/deepspeed/runtime/rollout/affine_weight_sync.py b/deepspeed/runtime/rollout/affine_weight_sync.py index e4f4e2a99df2..752d6ad61e0c 100644 --- a/deepspeed/runtime/rollout/affine_weight_sync.py +++ b/deepspeed/runtime/rollout/affine_weight_sync.py @@ -7,12 +7,17 @@ A rollout engine that generates with its own tensor-parallel copy of the weights refreshes that copy after every optimizer step. Gathering each parameter whole and cutting out the local shard sends every parameter to every rank. Instead, each parameter is described twice with affine maps (:mod:`deepspeed.checkpoint.affine`): how the trainer -stores it (:func:`zero_param_shaped_storage_map`, or ``autotp_storage_affine_maps`` for an AutoTP model) and which -block each inference rank needs (:func:`replicated_layout_map`). +stores it (:func:`zero_param_shaped_storage_map`; ``autotp_storage_affine_maps`` should serve for an AutoTP model, +but that is not tested yet) and which block each inference rank needs (:func:`replicated_layout_map`). :func:`~deepspeed.checkpoint.affine_transfer.plan_transfer` composes the two into copies from trainer shards straight into inference shards, once per layout, and every sync runs the copies of all parameters as one :func:`~deepspeed.checkpoint.affine_transfer_executor.execute_transfers` batch. +Which holder a replicated block is read from changes only the cost, never the result: once both maps pass +``plan_transfer``'s checks, every plan writes the same values whichever replica it reads, provided the replicas hold +the same values. Under ZeRO-1/2 the replicas are the data-parallel copies of each parameter, equal after every step's +all-gather; under ZeRO-3 each element has a single holder. + CUDA graphs record raw device pointers, so a decode graph captured on the inference copy stays valid only while every update writes into the storage it was captured on. The executor writes targets in place, and :class:`WeightArena` keeps a rank's shards as views of one buffer that is never reallocated. @@ -120,6 +125,10 @@ class _LocalFirstLeastLoaded: use no link and do not count, and regions with a single holder do, since that load cannot move. ``plan_transfer`` consults the selector only where a region has several holders, so :meth:`settle` recounts the load from each finished plan; within one parameter, its single-holder regions are counted only once it is planned. + + The load is kept across parameters: the increment in :meth:`__call__` balances the regions of the parameter being + planned, and :meth:`settle` replaces it with what that plan actually sends, added to the load of the parameters + planned before. """ def __init__(self): @@ -151,7 +160,8 @@ class AffineWeightSync: :func:`replicated_layout_map`. It names the same parameters as ``source_maps``. dtype: of every buffer; ranks that hold no buffer must agree on it too. group: the process group the copies cross, the world by default. It must contain every rank the maps name, - and all of its ranks construct the sync with the same maps. + and all of its ranks construct the sync with the same maps. The maps stay keyed by global rank with any + group; the sync translates them to the group's ranks for the executor. budget: bytes staged per peer and direction, see :class:`~deepspeed.checkpoint.affine_transfer_executor.StagingBudget`. limits: planning limits, see :class:`~deepspeed.checkpoint.affine_transfer.PlanBudget`. diff --git a/docs/code-docs/source/affine-transfer.rst b/docs/code-docs/source/affine-transfer.rst index 7f478a5f45e0..de07a91144d1 100644 --- a/docs/code-docs/source/affine-transfer.rst +++ b/docs/code-docs/source/affine-transfer.rst @@ -53,9 +53,10 @@ Refreshing an inference copy A rollout engine that generates with its own tensor-parallel copy of a ZeRO model refreshes that copy after every optimizer step. ``deepspeed.runtime.rollout.affine_weight_sync`` does it shard to shard: no -rank gathers a whole parameter. Both sides are maps keyed by global rank. The -trainer side comes from ``zero_param_shaped_storage_map`` (or -``autotp_storage_affine_maps`` for an AutoTP model). The inference side is a +rank gathers a whole parameter. Both sides are maps keyed by global rank, also +when the copies run over a custom process group. The trainer side comes from +``zero_param_shaped_storage_map`` (``autotp_storage_affine_maps`` should serve +for an AutoTP model, but that is not tested yet). The inference side is a tensor-parallel layout held by every replica, from ``replicated_layout_map``:: @@ -82,7 +83,9 @@ tensor-parallel layout held by every replica, from Plans are made once, when the sync is built, without communication. ``plan_transfer`` plans each parameter. A block that several ranks hold is read from the target's own process if that process holds it, and otherwise from the -holder that sends the least so far; local copies do not count as load. ``sync`` +holder that sends the least so far, over all parameters; local copies do not +count as load. The choice changes only the cost: replicas hold the same values, +so every choice writes the same bits. ``sync`` is collective: it runs every parameter's copies as one ``execute_transfers`` batch, and a rank that lacks one of its shards makes every rank refuse the batch before any data moves. ``WeightArena`` keeps a diff --git a/tests/unit/runtime/rollout/test_affine_weight_sync.py b/tests/unit/runtime/rollout/test_affine_weight_sync.py index ddcd52cd2527..2110043e76da 100644 --- a/tests/unit/runtime/rollout/test_affine_weight_sync.py +++ b/tests/unit/runtime/rollout/test_affine_weight_sync.py @@ -147,7 +147,8 @@ def test_sync_writes_every_replica_in_place(self, zero_stage, replicas): engine.destroy() def test_reads_of_replicated_parameters_spread_over_their_holders(self): - # Ranks 0-2 each hold all six parameters whole, and rank 3 needs them all. + # Ranks 0-2 each hold all six parameters whole, and rank 3 needs them all. Each parameter is a single region, + # so the even split comes from the load carried across parameters. shapes = {f'w{index}': (4, 6) for index in range(6)} rank = dist.get_rank() device = get_accelerator().current_device_name() @@ -193,6 +194,28 @@ def test_local_copies_are_free_and_single_holders_count_as_load(self): for name in needed: assert torch.equal(arena[name], full[name]), name + def test_maps_stay_keyed_by_global_rank_with_a_custom_group(self): + # The group leaves rank 0 out, so its ranks differ from global ranks: rank 1 (group rank 0) holds 'w' and + # rank 3 (group rank 2) needs it. + shape = (4, 6) + rank = dist.get_rank() + device = get_accelerator().current_device_name() + group = dist.new_group([1, 2, 3]) + if rank == 0: + return + whole = replicated_map(shape, 1) + sync = AffineWeightSync({'w': replicated_layout_map(whole, [[1]])}, {'w': replicated_layout_map(whole, [[3]])}, + torch.float32, + group=group) + full = torch.arange(24, dtype=torch.float32, device=device).view(shape) + arena = WeightArena({'w': shape}, torch.float32, device) if rank == 3 else None + + stats = sync.sync({'w': full} if rank == 1 else {}, dict(arena.items()) if arena is not None else {}) + + assert stats.sent_bytes == (24 * 4 if rank == 1 else 0) + if arena is not None: + assert torch.equal(arena['w'], full) + def test_a_rank_missing_a_shard_makes_every_rank_refuse_before_anything_moves(self): shape = (4, 6) rank = dist.get_rank() From 48a3575a11771c81e29b981a020e64b934556662 Mon Sep 17 00:00:00 2001 From: RunguoLi Date: Thu, 12 Nov 2026 08:26:50 -0600 Subject: [PATCH 3/3] Plan the sync with plan_transfers and a replica policy (#8936); shards given at each sync follow offload/reload Signed-off-by: RunguoLi --- .../runtime/rollout/affine_weight_sync.py | 80 +++++++------------ docs/code-docs/source/affine-transfer.rst | 7 +- .../rollout/test_affine_weight_sync.py | 65 ++++++++++++++- 3 files changed, 92 insertions(+), 60 deletions(-) diff --git a/deepspeed/runtime/rollout/affine_weight_sync.py b/deepspeed/runtime/rollout/affine_weight_sync.py index 752d6ad61e0c..57ed82985fbc 100644 --- a/deepspeed/runtime/rollout/affine_weight_sync.py +++ b/deepspeed/runtime/rollout/affine_weight_sync.py @@ -9,13 +9,13 @@ Instead, each parameter is described twice with affine maps (:mod:`deepspeed.checkpoint.affine`): how the trainer stores it (:func:`zero_param_shaped_storage_map`; ``autotp_storage_affine_maps`` should serve for an AutoTP model, but that is not tested yet) and which block each inference rank needs (:func:`replicated_layout_map`). -:func:`~deepspeed.checkpoint.affine_transfer.plan_transfer` composes the two into copies from trainer shards straight +:func:`~deepspeed.checkpoint.affine_transfer.plan_transfers` composes the two into copies from trainer shards straight into inference shards, once per layout, and every sync runs the copies of all parameters as one :func:`~deepspeed.checkpoint.affine_transfer_executor.execute_transfers` batch. -Which holder a replicated block is read from changes only the cost, never the result: once both maps pass -``plan_transfer``'s checks, every plan writes the same values whichever replica it reads, provided the replicas hold -the same values. Under ZeRO-1/2 the replicas are the data-parallel copies of each parameter, equal after every step's +Which holder a replicated block is read from changes only the cost, never the result: once both maps pass the +planner's checks, every plan writes the same values whichever replica it reads, provided the replicas hold the same +values. Under ZeRO-1/2 the replicas are the data-parallel copies of each parameter, equal after every step's all-gather; under ZeRO-3 each element has a single holder. CUDA graphs record raw device pointers, so a decode graph captured on the inference copy stays valid only while every @@ -23,13 +23,13 @@ keeps a rank's shards as views of one buffer that is never reallocated. """ -from typing import Dict, Iterator, Mapping, Optional, Sequence, Set, Tuple +from typing import Iterator, Mapping, Optional, Sequence, Set, Tuple import torch import deepspeed.comm as dist from deepspeed.checkpoint.affine import ParamAffineMap, nest_affine_maps, replicated_map -from deepspeed.checkpoint.affine_transfer import PlanBudget, ReplicaSelectionRequest, TransferPlan, plan_transfer +from deepspeed.checkpoint.affine_transfer import PlanBudget, ReplicaPolicy, plan_transfers from deepspeed.checkpoint.affine_transfer_executor import (StagingBudget, TransferEndpoints, TransferRequest, TransferStats, execute_transfers) from deepspeed.checkpoint.zero_affine import zero_param_storage_affine_map, zero_storage_affine_map @@ -117,39 +117,6 @@ def replicated_layout_map(layout: ParamAffineMap, replicas: Sequence[Sequence[in return nest_affine_maps(layout, storage) -class _LocalFirstLeastLoaded: - """Replica selector: a holder in the target's own process, otherwise the holder that sends the fewest elements so - far, ties to the lowest rank. - - Load counts what each rank sends to other processes, as #8936's ``local_first_least_loaded`` does: local copies - use no link and do not count, and regions with a single holder do, since that load cannot move. ``plan_transfer`` - consults the selector only where a region has several holders, so :meth:`settle` recounts the load from each - finished plan; within one parameter, its single-holder regions are counted only once it is planned. - - The load is kept across parameters: the increment in :meth:`__call__` balances the regions of the parameter being - planned, and :meth:`settle` replaces it with what that plan actually sends, added to the load of the parameters - planned before. - """ - - def __init__(self): - self.load: Dict[int, int] = {} - - def __call__(self, request: ReplicaSelectionRequest) -> int: - holders = [candidate.source_rank for candidate in request.candidates] - if request.target_rank in holders: - return request.target_rank - chosen = min(holders, key=lambda rank: (self.load.get(rank, 0), rank)) - self.load[chosen] = self.load.get(chosen, 0) + _numel(request.shape) - return chosen - - def settle(self, before: Dict[int, int], plan: TransferPlan) -> None: - """Set the load to ``before`` plus everything ``plan`` sends.""" - self.load = dict(before) - for segment in plan.segments: - if segment.source_rank != segment.target_rank: - self.load[segment.source_rank] = self.load.get(segment.source_rank, 0) + segment.numel - - class AffineWeightSync: """Copies parameters from where one side stores them into where the other side needs them, planned once. @@ -165,9 +132,12 @@ class AffineWeightSync: budget: bytes staged per peer and direction, see :class:`~deepspeed.checkpoint.affine_transfer_executor.StagingBudget`. limits: planning limits, see :class:`~deepspeed.checkpoint.affine_transfer.PlanBudget`. + replica_policy: where a block several ranks hold is read from, see + :func:`~deepspeed.checkpoint.affine_transfer.plan_transfers`. By default + :func:`~deepspeed.checkpoint.affine_replica_policy.local_first_least_loaded`: from the target's own process + when it holds one (a local copy), and otherwise from the holder that sends the fewest bytes so far, over + all parameters. Every rank must pass a policy that gives the same answers. - A block several ranks hold is read from the target's own process when it holds one (a local copy), and otherwise - from the holder that sends the fewest elements so far, over all parameters (see :class:`_LocalFirstLeastLoaded`). Planning has no communication and is deterministic, so every rank derives the same plans. """ @@ -177,7 +147,8 @@ def __init__(self, dtype: torch.dtype, group=None, budget: StagingBudget = StagingBudget(), - limits: Optional[PlanBudget] = None): + limits: Optional[PlanBudget] = None, + replica_policy: Optional[ReplicaPolicy] = None): if set(source_maps) != set(target_maps): only_one = sorted(set(source_maps) ^ set(target_maps)) raise ValueError(f'Source and target maps must name the same parameters; only one side names {only_one}.') @@ -193,19 +164,21 @@ def __init__(self, outside = sorted((source_ranks | target_ranks) - set(process)) if outside: raise ValueError(f'Ranks {outside} hold parameters but are not in the transfer group.') - self.endpoints = TransferEndpoints(source={rank: process[rank] - for rank in sorted(source_ranks)}, - target={rank: process[rank] - for rank in sorted(target_ranks)}) + source_processes = {rank: process[rank] for rank in sorted(source_ranks)} + target_processes = {rank: process[rank] for rank in sorted(target_ranks)} + self.endpoints = TransferEndpoints(source=source_processes, target=target_processes) # The parameters this rank takes part in on each side. self._sources_here = {name for name, affine_map in source_maps.items() if self.rank in affine_map.shard_shapes} self._targets_here = {name for name, affine_map in target_maps.items() if self.rank in affine_map.shard_shapes} - selector = _LocalFirstLeastLoaded() - self.plans = {} - for name in sorted(source_maps): - before = dict(selector.load) - self.plans[name] = plan_transfer(target_maps[name], source_maps[name], limits, replica_selector=selector) - selector.settle(before, self.plans[name]) + pairs = {name: (target_maps[name], source_maps[name]) for name in source_maps} + element_size = dict.fromkeys(source_maps, torch.empty(0, dtype=dtype).element_size()) + plans = plan_transfers(pairs, + limits, + source_processes=source_processes, + target_processes=target_processes, + element_size=element_size, + replica_policy=replica_policy) + self.plans = {name: plans[name] for name in sorted(plans)} def sync(self, source_buffers: Buffers, target_buffers: Buffers) -> TransferStats: """Copy every parameter. Collective over ``group``. @@ -213,7 +186,8 @@ def sync(self, source_buffers: Buffers, target_buffers: Buffers) -> TransferStat ``source_buffers[name]`` is this rank's source shard of ``name`` and ``target_buffers[name]`` its target shard, for every parameter the maps place on this rank; entries for other parameters are ignored. Shards are contiguous tensors, which may be views (a view keeps its offset into its storage). The sources must not change - while the copies run. A rank that lacks a shard refuses the batch in the executor's collective preflight, so + while the copies run. The sync keeps no view of a shard between calls, so resolve the shards at every call: + ``reload_states()`` after ``offload_states()`` moves every ZeRO-3 partition to new storage. A rank that lacks a shard refuses the batch in the executor's collective preflight, so every rank raises and no data moves. """ requests = [] diff --git a/docs/code-docs/source/affine-transfer.rst b/docs/code-docs/source/affine-transfer.rst index db204b3e70a9..d44fc65ae715 100644 --- a/docs/code-docs/source/affine-transfer.rst +++ b/docs/code-docs/source/affine-transfer.rst @@ -122,9 +122,10 @@ tensor-parallel layout held by every replica, from dict(arena.items())) Plans are made once, when the sync is built, without communication. -``plan_transfer`` plans each parameter. A block that several ranks hold is read -from the target's own process if that process holds it, and otherwise from the -holder that sends the least so far, over all parameters; local copies do not +``plan_transfers`` plans every parameter at once, with ``replica_policy`` +(``local_first_least_loaded`` by default): a block that several ranks hold is +read from the target's own process if that process holds it, and otherwise from +the holder that sends the least so far, over all parameters; local copies do not count as load. The choice changes only the cost: replicas hold the same values, so every choice writes the same bits. ``sync`` is collective: it runs every parameter's copies as one ``execute_transfers`` diff --git a/tests/unit/runtime/rollout/test_affine_weight_sync.py b/tests/unit/runtime/rollout/test_affine_weight_sync.py index 2110043e76da..dcfffbbb80bf 100644 --- a/tests/unit/runtime/rollout/test_affine_weight_sync.py +++ b/tests/unit/runtime/rollout/test_affine_weight_sync.py @@ -11,6 +11,7 @@ import deepspeed.comm as dist from deepspeed.accelerator import get_accelerator from deepspeed.checkpoint.affine import contiguous_split_map, replicated_map +from deepspeed.checkpoint.affine_replica_policy import first_holder from deepspeed.checkpoint.affine_transfer_executor import AffineTransferExecutionError from deepspeed.checkpoint.zero_affine import zero_param_storage_shard from deepspeed.runtime.rollout.affine_weight_sync import (AffineWeightSync, WeightArena, replicated_layout_map, @@ -146,9 +147,11 @@ def test_sync_writes_every_replica_in_place(self, zero_stage, replicas): assert not torch.equal(second_values, first_values) engine.destroy() - def test_reads_of_replicated_parameters_spread_over_their_holders(self): + @pytest.mark.parametrize('policy', ['default', 'first_holder']) + def test_reads_of_replicated_parameters_spread_over_their_holders(self, policy): # Ranks 0-2 each hold all six parameters whole, and rank 3 needs them all. Each parameter is a single region, - # so the even split comes from the load carried across parameters. + # so the even split comes from the load carried across parameters. first_holder reads everything from rank 0: + # the policy decides only where data is read from. shapes = {f'w{index}': (4, 6) for index in range(6)} rank = dist.get_rank() device = get_accelerator().current_device_name() @@ -156,7 +159,10 @@ def test_reads_of_replicated_parameters_spread_over_their_holders(self): for name, shape in shapes.items(): source_maps[name] = replicated_layout_map(replicated_map(shape, 1), [[0], [1], [2]]) target_maps[name] = replicated_layout_map(replicated_map(shape, 1), [[3]]) - sync = AffineWeightSync(source_maps, target_maps, torch.float32) + sync = AffineWeightSync(source_maps, + target_maps, + torch.float32, + replica_policy=first_holder if policy == 'first_holder' else None) full = { name: torch.arange(24, dtype=torch.float32, device=device).view(shape) + 100 * index for index, (name, shape) in enumerate(shapes.items()) @@ -168,7 +174,10 @@ def test_reads_of_replicated_parameters_spread_over_their_holders(self): stats = sync.sync(full, targets) sent = _all(stats.sent_bytes) - assert sent[0] == sent[1] == sent[2] == 2 * 24 * 4 and sent[3] == 0 + if policy == 'first_holder': + assert sent == [6 * 24 * 4, 0, 0, 0] + else: + assert sent[0] == sent[1] == sent[2] == 2 * 24 * 4 and sent[3] == 0 if arena is not None: for name in shapes: assert torch.equal(arena[name], full[name]) @@ -231,6 +240,54 @@ def test_a_rank_missing_a_shard_makes_every_rank_refuse_before_anything_moves(se if rank == 2: assert torch.all(arena['w'] == 0) + def test_shards_given_at_each_sync_follow_offload_and_reload(self): + # reload_states() gives every ZeRO-3 partition new storage. The sync keeps no view of a source between calls, + # so shards resolved at each call follow the move; plans depend only on the maps. + dtype = torch.bfloat16 if get_accelerator().is_bf16_supported() else torch.float32 + config = { + 'train_micro_batch_size_per_gpu': 1, + # offload_states() supports only DeepSpeed's FusedAdam. + 'optimizer': { + 'type': 'Adam', + 'params': { + 'lr': 1e-2 + } + }, + 'zero_optimization': { + 'stage': 3 + }, + } + if dtype == torch.bfloat16: + config['bf16'] = {'enabled': True} + with deepspeed.zero.Init(config_dict_or_path=config): + model = MLPBlock() + engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=config) + params = dict(engine.module.named_parameters()) + replicas = [[0, 1], [2, 3]] + layouts = _layouts(2) + tp_rank = _tp_rank(replicas, dist.get_rank()) + source_maps = {name: zero_param_shaped_storage_map(param) for name, param in params.items()} + target_maps = {name: replicated_layout_map(layouts[name], replicas) for name in params} + sync = AffineWeightSync(source_maps, target_maps, dtype) + arena = WeightArena({name: layouts[name].shard_shapes[tp_rank] for name in params}, dtype, engine.device) + + for x, y in random_dataloader(engine, 1, HIDDEN, engine.device, dtype): + engine.backward(engine(x, y)) + engine.step() + engine.empty_partition_cache() + before = {name: zero_param_storage_shard(param).data_ptr() for name, param in params.items()} + engine.offload_states(pin_memory=get_accelerator().device_name() != 'cpu') + engine.reload_states() + moved = any(zero_param_storage_shard(param).data_ptr() != before[name] for name, param in params.items()) + if get_accelerator().device_name() != 'cpu': + # Otherwise this test would not cover a move. + assert moved + + sync.sync({name: zero_param_storage_shard(param) for name, param in params.items()}, dict(arena.items())) + for name, param in params.items(): + assert torch.equal(arena[name], layouts[name].extract(_full_param(param), tp_rank)), name + engine.destroy() + def test_a_graph_captured_before_a_sync_reads_the_synced_weights(self): if get_accelerator().device_name() != 'cuda': pytest.skip('requires CUDA graphs')