diff --git a/deepspeed/runtime/rollout/affine_weight_sync.py b/deepspeed/runtime/rollout/affine_weight_sync.py new file mode 100644 index 000000000000..57ed82985fbc --- /dev/null +++ b/deepspeed/runtime/rollout/affine_weight_sync.py @@ -0,0 +1,208 @@ +# Copyright (c) DeepSpeed Team. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +"""Keep a tensor-parallel inference copy of a training model current, shard to shard, at fixed addresses. + +A rollout engine that generates with its own tensor-parallel copy of the weights refreshes that copy after every +optimizer step. Gathering each parameter whole and cutting out the local shard sends every parameter to every rank. +Instead, each parameter is described twice with affine maps (:mod:`deepspeed.checkpoint.affine`): how the trainer +stores it (:func:`zero_param_shaped_storage_map`; ``autotp_storage_affine_maps`` should serve for an AutoTP model, +but that is not tested yet) and which block each inference rank needs (:func:`replicated_layout_map`). +:func:`~deepspeed.checkpoint.affine_transfer.plan_transfers` composes the two into copies from trainer shards straight +into inference shards, once per layout, and every sync runs the copies of all parameters as one +:func:`~deepspeed.checkpoint.affine_transfer_executor.execute_transfers` batch. + +Which holder a replicated block is read from changes only the cost, never the result: once both maps pass the +planner's checks, every plan writes the same values whichever replica it reads, provided the replicas hold the same +values. Under ZeRO-1/2 the replicas are the data-parallel copies of each parameter, equal after every step's +all-gather; under ZeRO-3 each element has a single holder. + +CUDA graphs record raw device pointers, so a decode graph captured on the inference copy stays valid only while every +update writes into the storage it was captured on. The executor writes targets in place, and :class:`WeightArena` +keeps a rank's shards as views of one buffer that is never reallocated. +""" + +from typing import Iterator, Mapping, Optional, Sequence, Set, Tuple + +import torch + +import deepspeed.comm as dist +from deepspeed.checkpoint.affine import ParamAffineMap, nest_affine_maps, replicated_map +from deepspeed.checkpoint.affine_transfer import PlanBudget, ReplicaPolicy, plan_transfers +from deepspeed.checkpoint.affine_transfer_executor import (StagingBudget, TransferEndpoints, TransferRequest, + TransferStats, execute_transfers) +from deepspeed.checkpoint.zero_affine import zero_param_storage_affine_map, zero_storage_affine_map + +__all__ = ['WeightArena', 'zero_param_shaped_storage_map', 'replicated_layout_map', 'AffineWeightSync'] + +# A rank's tensor for each parameter it holds, by parameter name. +Buffers = Mapping[str, torch.Tensor] + +# Each view starts on a 256-byte boundary, as allocator blocks do, so kernels see the alignment they expect. +_ALIGN_BYTES = 256 + + +def _numel(shape: Sequence[int]) -> int: + numel = 1 + for size in shape: + numel *= size + return numel + + +class WeightArena: + """Named views of one flat buffer that is never reallocated. + + Args: + shapes: ``{name: shape}`` in allocation order. + dtype, device: of the buffer. + """ + + def __init__(self, shapes: Mapping[str, Sequence[int]], dtype: torch.dtype, device=None): + element_size = torch.empty(0, dtype=dtype).element_size() + align = max(1, _ALIGN_BYTES // element_size) + offsets = {} + total = 0 + for name, shape in shapes.items(): + total = -(-total // align) * align + offsets[name] = total + total += _numel(shape) + # Zeros, so that padding a kernel may read (rows past the end of a vocabulary shard) holds no garbage. An + # allocation is not always 256-byte aligned (the CPU allocator aligns to 64 bytes), so allocate one alignment + # more and start the buffer at its first aligned element. + storage = torch.zeros(total + align, dtype=dtype, device=device) + skip = -(storage.data_ptr() // element_size) % align + self.buffer = storage.narrow(0, skip, total) + self._views = {} + for name, offset in offsets.items(): + shape = tuple(shapes[name]) + self._views[name] = self.buffer.narrow(0, offset, _numel(shape)).view(shape) + + def __getitem__(self, name: str) -> torch.Tensor: + return self._views[name] + + def items(self) -> Iterator[Tuple[str, torch.Tensor]]: + return iter(self._views.items()) + + +def zero_param_shaped_storage_map(param) -> ParamAffineMap: + """What every rank stores of a ZeRO model's ``param``, in the parameter's own shape and keyed by global rank. + + ``zero_param_storage_affine_map`` describes the flattened parameter, while a tensor-parallel layout describes it + in its own shape, and both sides of a plan must describe the same logical tensor. This rank's shard is still + ``zero_param_storage_shard(param)``. + """ + shape = tuple(getattr(param, 'ds_shape', param.shape)) + return nest_affine_maps(replicated_map(shape, 1), {0: zero_param_storage_affine_map(param)}) + + +def replicated_layout_map(layout: ParamAffineMap, replicas: Sequence[Sequence[int]]) -> ParamAffineMap: + """A parameter's tensor-parallel layout, held by every replica of the inference copy, keyed by global rank. + + ``layout`` describes one replica, keyed by tensor-parallel rank ``0 .. tp - 1``: an AutoTP layer's layout map, or + one built with :mod:`deepspeed.checkpoint.affine` (``contiguous_split_map``, ``replicated_map``, ...). + ``replicas[i][j]`` is the global rank that is tensor-parallel rank ``j`` of replica ``i``. A rank's shard is its + layout shard flattened, so a contiguous tensor of the layout shard's shape holds it. + """ + tp_size = len(layout.shard_shapes) + if sorted(layout.shard_shapes) != list(range(tp_size)): + raise ValueError(f'The layout must be keyed by tensor-parallel ranks 0..{tp_size - 1}, ' + f'not {sorted(layout.shard_shapes)}.') + if not replicas or any(len(ranks) != tp_size for ranks in replicas): + raise ValueError(f'Every replica needs {tp_size} ranks, one per tensor-parallel rank: {list(replicas)}.') + # Rank j of every replica stores layout shard j whole, as data-parallel replicas store a parameter. + storage = {} + for tp_rank, shape in layout.shard_shapes.items(): + storage[tp_rank] = zero_storage_affine_map(_numel(shape), [ranks[tp_rank] for ranks in replicas]) + return nest_affine_maps(layout, storage) + + +class AffineWeightSync: + """Copies parameters from where one side stores them into where the other side needs them, planned once. + + Args: + source_maps: ``{name: map}`` of what the source side stores, keyed by global rank, for example + :func:`zero_param_shaped_storage_map` of each parameter of a ZeRO model. + target_maps: ``{name: map}`` of what the target side needs, keyed by global rank, for example from + :func:`replicated_layout_map`. It names the same parameters as ``source_maps``. + dtype: of every buffer; ranks that hold no buffer must agree on it too. + group: the process group the copies cross, the world by default. It must contain every rank the maps name, + and all of its ranks construct the sync with the same maps. The maps stay keyed by global rank with any + group; the sync translates them to the group's ranks for the executor. + budget: bytes staged per peer and direction, see + :class:`~deepspeed.checkpoint.affine_transfer_executor.StagingBudget`. + limits: planning limits, see :class:`~deepspeed.checkpoint.affine_transfer.PlanBudget`. + replica_policy: where a block several ranks hold is read from, see + :func:`~deepspeed.checkpoint.affine_transfer.plan_transfers`. By default + :func:`~deepspeed.checkpoint.affine_replica_policy.local_first_least_loaded`: from the target's own process + when it holds one (a local copy), and otherwise from the holder that sends the fewest bytes so far, over + all parameters. Every rank must pass a policy that gives the same answers. + + Planning has no communication and is deterministic, so every rank derives the same plans. + """ + + def __init__(self, + source_maps: Mapping[str, ParamAffineMap], + target_maps: Mapping[str, ParamAffineMap], + dtype: torch.dtype, + group=None, + budget: StagingBudget = StagingBudget(), + limits: Optional[PlanBudget] = None, + replica_policy: Optional[ReplicaPolicy] = None): + if set(source_maps) != set(target_maps): + only_one = sorted(set(source_maps) ^ set(target_maps)) + raise ValueError(f'Source and target maps must name the same parameters; only one side names {only_one}.') + self.dtype, self.group, self.budget = dtype, group, budget + self.rank = dist.get_rank() + if group is None: + members = list(range(dist.get_world_size())) + else: + members = [dist.get_global_rank(group, rank) for rank in range(dist.get_world_size(group=group))] + process = {rank: index for index, rank in enumerate(members)} + source_ranks = {rank for affine_map in source_maps.values() for rank in affine_map.shard_shapes} + target_ranks = {rank for affine_map in target_maps.values() for rank in affine_map.shard_shapes} + outside = sorted((source_ranks | target_ranks) - set(process)) + if outside: + raise ValueError(f'Ranks {outside} hold parameters but are not in the transfer group.') + source_processes = {rank: process[rank] for rank in sorted(source_ranks)} + target_processes = {rank: process[rank] for rank in sorted(target_ranks)} + self.endpoints = TransferEndpoints(source=source_processes, target=target_processes) + # The parameters this rank takes part in on each side. + self._sources_here = {name for name, affine_map in source_maps.items() if self.rank in affine_map.shard_shapes} + self._targets_here = {name for name, affine_map in target_maps.items() if self.rank in affine_map.shard_shapes} + pairs = {name: (target_maps[name], source_maps[name]) for name in source_maps} + element_size = dict.fromkeys(source_maps, torch.empty(0, dtype=dtype).element_size()) + plans = plan_transfers(pairs, + limits, + source_processes=source_processes, + target_processes=target_processes, + element_size=element_size, + replica_policy=replica_policy) + self.plans = {name: plans[name] for name in sorted(plans)} + + def sync(self, source_buffers: Buffers, target_buffers: Buffers) -> TransferStats: + """Copy every parameter. Collective over ``group``. + + ``source_buffers[name]`` is this rank's source shard of ``name`` and ``target_buffers[name]`` its target + shard, for every parameter the maps place on this rank; entries for other parameters are ignored. Shards are + contiguous tensors, which may be views (a view keeps its offset into its storage). The sources must not change + while the copies run. The sync keeps no view of a shard between calls, so resolve the shards at every call: + ``reload_states()`` after ``offload_states()`` moves every ZeRO-3 partition to new storage. A rank that lacks a shard refuses the batch in the executor's collective preflight, so + every rank raises and no data moves. + """ + requests = [] + for name, plan in self.plans.items(): + sources, source_error = self._local(name, self._sources_here, source_buffers, 'source') + targets, target_error = self._local(name, self._targets_here, target_buffers, 'target') + # Raising here would leave the other ranks waiting in the preflight, so the executor raises everywhere. + error = source_error or target_error + requests.append(TransferRequest(name, plan, sources, targets, preflight_error=error)) + return execute_transfers(requests, self.endpoints, self.dtype, self.budget, self.group) + + def _local(self, name: str, here: Set[str], buffers: Buffers, side: str): + """``({rank: shard}, None)`` for this rank's side of ``name``, or ``({}, error)`` when the shard is missing.""" + if name not in here: + return {}, None + if name not in buffers: + return {}, f'rank {self.rank} holds part of {name!r} but was given no {side} shard for it' + return {self.rank: buffers[name]}, None diff --git a/docs/code-docs/source/affine-transfer.rst b/docs/code-docs/source/affine-transfer.rst index ad3c59a8d1a4..d44fc65ae715 100644 --- a/docs/code-docs/source/affine-transfer.rst +++ b/docs/code-docs/source/affine-transfer.rst @@ -87,3 +87,50 @@ zero-copy transport such as GPUDirect RDMA would instead take each round's removes the arena copy but needs a registration lifetime this API has no place for, so the wire contract (one packed message per peer per round) is expected to change when such a path lands. + +Refreshing an inference copy +---------------------------- + +A rollout engine that generates with its own tensor-parallel copy of a ZeRO +model refreshes that copy after every optimizer step. +``deepspeed.runtime.rollout.affine_weight_sync`` does it shard to shard: no +rank gathers a whole parameter. Both sides are maps keyed by global rank, also +when the copies run over a custom process group. The trainer side comes from +``zero_param_shaped_storage_map`` (``autotp_storage_affine_maps`` should serve +for an AutoTP model, but that is not tested yet). The inference side is a +tensor-parallel layout held by every replica, from +``replicated_layout_map``:: + + from deepspeed.checkpoint.affine import contiguous_split_map + from deepspeed.checkpoint.zero_affine import zero_param_storage_shard + from deepspeed.runtime.rollout.affine_weight_sync import ( + AffineWeightSync, WeightArena, replicated_layout_map, + zero_param_shaped_storage_map) + + replicas = [[0, 1], [2, 3]] # two replicas of TP=2 + layouts = {name: contiguous_split_map(...) for name in names} + params = dict(engine.module.named_parameters()) + sync = AffineWeightSync( + {name: zero_param_shaped_storage_map(params[name]) for name in names}, + {name: replicated_layout_map(layouts[name], replicas) for name in names}, + torch.bfloat16) + arena = WeightArena({name: layouts[name].shard_shapes[tp_rank] for name in names}, + torch.bfloat16, device) + + engine.empty_partition_cache() # ZeRO-3: release gathered copies first + sync.sync({name: zero_param_storage_shard(params[name]) for name in names}, + dict(arena.items())) + +Plans are made once, when the sync is built, without communication. +``plan_transfers`` plans every parameter at once, with ``replica_policy`` +(``local_first_least_loaded`` by default): a block that several ranks hold is +read from the target's own process if that process holds it, and otherwise from +the holder that sends the least so far, over all parameters; local copies do not +count as load. The choice changes only the cost: replicas hold the same values, +so every choice writes the same bits. ``sync`` +is collective: it runs every parameter's copies as one ``execute_transfers`` +batch, and a rank that lacks one of its shards makes every rank refuse the batch +before any data moves. ``WeightArena`` keeps a +rank's shards as views of one buffer that is never reallocated, and the +executor writes targets in place. A CUDA graph captured on the arena therefore +reads the new weights after every sync. Maintenance contact: ``@LiRunGuo``. diff --git a/tests/unit/runtime/rollout/test_affine_weight_sync.py b/tests/unit/runtime/rollout/test_affine_weight_sync.py new file mode 100644 index 000000000000..dcfffbbb80bf --- /dev/null +++ b/tests/unit/runtime/rollout/test_affine_weight_sync.py @@ -0,0 +1,352 @@ +# Copyright (c) DeepSpeed Team. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +"""Refreshing a tensor-parallel copy of a ZeRO model through affine plans.""" + +import pytest +import torch + +import deepspeed +import deepspeed.comm as dist +from deepspeed.accelerator import get_accelerator +from deepspeed.checkpoint.affine import contiguous_split_map, replicated_map +from deepspeed.checkpoint.affine_replica_policy import first_holder +from deepspeed.checkpoint.affine_transfer_executor import AffineTransferExecutionError +from deepspeed.checkpoint.zero_affine import zero_param_storage_shard +from deepspeed.runtime.rollout.affine_weight_sync import (AffineWeightSync, WeightArena, replicated_layout_map, + zero_param_shaped_storage_map) +from unit.common import DistributedTest +from unit.simple_model import random_dataloader + +HIDDEN, INNER = 13, 18 + + +class MLPBlock(torch.nn.Module): + """A column-parallel and a row-parallel linear and a norm, as a transformer MLP is split.""" + + def __init__(self): + super().__init__() + self.up = torch.nn.Linear(HIDDEN, INNER) + self.down = torch.nn.Linear(INNER, HIDDEN) + self.norm = torch.nn.LayerNorm(HIDDEN) + + def forward(self, x, y): + return torch.nn.functional.cross_entropy(self.norm(self.down(torch.relu(self.up(x)))), y) + + +def _even(size, parts): + base, extra = divmod(size, parts) + return [base + (part < extra) for part in range(parts)] + + +def _layouts(tp_size): + # INNER does not divide by 4, so TP=4 shards are uneven. + return { + 'up.weight': contiguous_split_map((INNER, HIDDEN), _even(INNER, tp_size), 0), + 'up.bias': contiguous_split_map((INNER, ), _even(INNER, tp_size), 0), + 'down.weight': contiguous_split_map((HIDDEN, INNER), _even(INNER, tp_size), 1), + 'down.bias': replicated_map((HIDDEN, ), tp_size), + 'norm.weight': replicated_map((HIDDEN, ), tp_size), + 'norm.bias': replicated_map((HIDDEN, ), tp_size), + } + + +def _full_param(param): + if hasattr(param, 'ds_id'): + with deepspeed.zero.GatheredParameters([param]): + return param.data.detach().clone() + return param.data.detach().clone() + + +def _total(value): + tensor = torch.tensor([value], dtype=torch.int64, device=get_accelerator().current_device_name()) + dist.all_reduce(tensor) + return int(tensor.item()) + + +def _all(value): + gathered = [None] * dist.get_world_size() + dist.all_gather_object(gathered, value) + return gathered + + +def _tp_rank(replicas, rank): + for ranks in replicas: + if rank in ranks: + return ranks.index(rank) + return None + + +class TestAffineWeightSync(DistributedTest): + world_size = 4 + + # Two TP=2 replicas, one TP=4 replica, and two TP=1 replicas on half of the ranks. + @pytest.mark.parametrize('replicas', [[[0, 1], [2, 3]], [[0, 1, 2, 3]], [[3], [1]]]) + @pytest.mark.parametrize('zero_stage', [1, 2, 3]) + def test_sync_writes_every_replica_in_place(self, zero_stage, replicas): + dtype = torch.bfloat16 if get_accelerator().is_bf16_supported() else torch.float32 + config = { + 'train_micro_batch_size_per_gpu': 1, + 'optimizer': { + 'type': 'Adam', + 'params': { + 'lr': 1e-2, + 'torch_adam': True + } + }, + 'zero_optimization': { + 'stage': zero_stage + }, + } + if dtype == torch.bfloat16: + config['bf16'] = {'enabled': True} + if zero_stage == 3: + with deepspeed.zero.Init(config_dict_or_path=config): + model = MLPBlock() + else: + model = MLPBlock() + engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=config) + params = dict(engine.module.named_parameters()) + layouts = _layouts(len(replicas[0])) + rank = dist.get_rank() + tp_rank = _tp_rank(replicas, rank) + + source_maps = {name: zero_param_shaped_storage_map(param) for name, param in params.items()} + target_maps = {name: replicated_layout_map(layouts[name], replicas) for name in params} + sync = AffineWeightSync(source_maps, target_maps, dtype) + arena = None + if tp_rank is not None: + shapes = {name: layouts[name].shard_shapes[tp_rank] for name in params} + arena = WeightArena(shapes, dtype, engine.device) + + synced = [] + for step in range(2): + if zero_stage == 3: + engine.empty_partition_cache() + sources = {name: zero_param_storage_shard(param) for name, param in params.items()} + targets = dict(arena.items()) if arena is not None else {} + stats = sync.sync(sources, targets) + if zero_stage < 3: + # Every rank holds a replica of every parameter, so each target is filled from its own process. + assert _total(stats.sent_bytes) == 0 + for name, param in params.items(): + full = _full_param(param) + if arena is not None: + assert torch.equal(arena[name], layouts[name].extract(full, tp_rank)), name + if arena is not None: + synced.append((arena.buffer.data_ptr(), arena.buffer.clone())) + for x, y in random_dataloader(engine, 1, HIDDEN, engine.device, dtype): + engine.backward(engine(x, y)) + engine.step() + + if arena is not None: + (first_address, first_values), (second_address, second_values) = synced + assert second_address == first_address + # Otherwise the second sync proves nothing about writing in place. + assert not torch.equal(second_values, first_values) + engine.destroy() + + @pytest.mark.parametrize('policy', ['default', 'first_holder']) + def test_reads_of_replicated_parameters_spread_over_their_holders(self, policy): + # Ranks 0-2 each hold all six parameters whole, and rank 3 needs them all. Each parameter is a single region, + # so the even split comes from the load carried across parameters. first_holder reads everything from rank 0: + # the policy decides only where data is read from. + shapes = {f'w{index}': (4, 6) for index in range(6)} + rank = dist.get_rank() + device = get_accelerator().current_device_name() + source_maps, target_maps = {}, {} + for name, shape in shapes.items(): + source_maps[name] = replicated_layout_map(replicated_map(shape, 1), [[0], [1], [2]]) + target_maps[name] = replicated_layout_map(replicated_map(shape, 1), [[3]]) + sync = AffineWeightSync(source_maps, + target_maps, + torch.float32, + replica_policy=first_holder if policy == 'first_holder' else None) + full = { + name: torch.arange(24, dtype=torch.float32, device=device).view(shape) + 100 * index + for index, (name, shape) in enumerate(shapes.items()) + } + arena = WeightArena(shapes, torch.float32, device) if rank == 3 else None + + targets = dict(arena.items()) if arena is not None else {} + # Every rank passes the same sources; rank 3 stores none, so its entries are ignored. + stats = sync.sync(full, targets) + + sent = _all(stats.sent_bytes) + if policy == 'first_holder': + assert sent == [6 * 24 * 4, 0, 0, 0] + else: + assert sent[0] == sent[1] == sent[2] == 2 * 24 * 4 and sent[3] == 0 + if arena is not None: + for name in shapes: + assert torch.equal(arena[name], full[name]) + + def test_local_copies_are_free_and_single_holders_count_as_load(self): + # Rank 0 alone holds 'a', which rank 3 needs, so rank 0 must send it. Ranks 0 and 1 hold 'b', which ranks 1 + # and 3 need: rank 1 copies its own, which uses no link, so rank 3 reads 'b' from rank 1, which sends less. + shape = (4, 6) + rank = dist.get_rank() + device = get_accelerator().current_device_name() + whole = replicated_map(shape, 1) + source_maps = {'a': replicated_layout_map(whole, [[0]]), 'b': replicated_layout_map(whole, [[0], [1]])} + target_maps = {'a': replicated_layout_map(whole, [[3]]), 'b': replicated_layout_map(whole, [[1], [3]])} + sync = AffineWeightSync(source_maps, target_maps, torch.float32) + full = {name: torch.full(shape, float(index + 1), device=device) for index, name in enumerate(source_maps)} + needed = [name for name in target_maps if rank in target_maps[name].shard_shapes] + arena = WeightArena({name: shape for name in needed}, torch.float32, device) if needed else None + targets = dict(arena.items()) if arena is not None else {} + + stats = sync.sync(full, targets) + + assert _all(stats.sent_bytes) == [24 * 4, 24 * 4, 0, 0] + for name in needed: + assert torch.equal(arena[name], full[name]), name + + def test_maps_stay_keyed_by_global_rank_with_a_custom_group(self): + # The group leaves rank 0 out, so its ranks differ from global ranks: rank 1 (group rank 0) holds 'w' and + # rank 3 (group rank 2) needs it. + shape = (4, 6) + rank = dist.get_rank() + device = get_accelerator().current_device_name() + group = dist.new_group([1, 2, 3]) + if rank == 0: + return + whole = replicated_map(shape, 1) + sync = AffineWeightSync({'w': replicated_layout_map(whole, [[1]])}, {'w': replicated_layout_map(whole, [[3]])}, + torch.float32, + group=group) + full = torch.arange(24, dtype=torch.float32, device=device).view(shape) + arena = WeightArena({'w': shape}, torch.float32, device) if rank == 3 else None + + stats = sync.sync({'w': full} if rank == 1 else {}, dict(arena.items()) if arena is not None else {}) + + assert stats.sent_bytes == (24 * 4 if rank == 1 else 0) + if arena is not None: + assert torch.equal(arena['w'], full) + + def test_a_rank_missing_a_shard_makes_every_rank_refuse_before_anything_moves(self): + shape = (4, 6) + rank = dist.get_rank() + device = get_accelerator().current_device_name() + sync = AffineWeightSync({'w': replicated_layout_map(replicated_map(shape, 1), [[0]])}, + {'w': replicated_layout_map(contiguous_split_map(shape, [2, 2], 0), [[2, 3]])}, + torch.float32) + arena = WeightArena({'w': (2, 6)}, torch.float32, device) if rank in (2, 3) else None + # Rank 3 forgets its target shard. Had it raised alone, the other ranks would wait for it in the preflight. + targets = dict(arena.items()) if arena is not None and rank != 3 else {} + with pytest.raises(AffineTransferExecutionError): + sync.sync({'w': torch.ones(shape, device=device)}, targets) + if rank == 2: + assert torch.all(arena['w'] == 0) + + def test_shards_given_at_each_sync_follow_offload_and_reload(self): + # reload_states() gives every ZeRO-3 partition new storage. The sync keeps no view of a source between calls, + # so shards resolved at each call follow the move; plans depend only on the maps. + dtype = torch.bfloat16 if get_accelerator().is_bf16_supported() else torch.float32 + config = { + 'train_micro_batch_size_per_gpu': 1, + # offload_states() supports only DeepSpeed's FusedAdam. + 'optimizer': { + 'type': 'Adam', + 'params': { + 'lr': 1e-2 + } + }, + 'zero_optimization': { + 'stage': 3 + }, + } + if dtype == torch.bfloat16: + config['bf16'] = {'enabled': True} + with deepspeed.zero.Init(config_dict_or_path=config): + model = MLPBlock() + engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=config) + params = dict(engine.module.named_parameters()) + replicas = [[0, 1], [2, 3]] + layouts = _layouts(2) + tp_rank = _tp_rank(replicas, dist.get_rank()) + source_maps = {name: zero_param_shaped_storage_map(param) for name, param in params.items()} + target_maps = {name: replicated_layout_map(layouts[name], replicas) for name in params} + sync = AffineWeightSync(source_maps, target_maps, dtype) + arena = WeightArena({name: layouts[name].shard_shapes[tp_rank] for name in params}, dtype, engine.device) + + for x, y in random_dataloader(engine, 1, HIDDEN, engine.device, dtype): + engine.backward(engine(x, y)) + engine.step() + engine.empty_partition_cache() + before = {name: zero_param_storage_shard(param).data_ptr() for name, param in params.items()} + engine.offload_states(pin_memory=get_accelerator().device_name() != 'cpu') + engine.reload_states() + moved = any(zero_param_storage_shard(param).data_ptr() != before[name] for name, param in params.items()) + if get_accelerator().device_name() != 'cpu': + # Otherwise this test would not cover a move. + assert moved + + sync.sync({name: zero_param_storage_shard(param) for name, param in params.items()}, dict(arena.items())) + for name, param in params.items(): + assert torch.equal(arena[name], layouts[name].extract(_full_param(param), tp_rank)), name + engine.destroy() + + def test_a_graph_captured_before_a_sync_reads_the_synced_weights(self): + if get_accelerator().device_name() != 'cuda': + pytest.skip('requires CUDA graphs') + shape = (6, 8) + layout = contiguous_split_map(shape, [3, 3], 0) + rank = dist.get_rank() + device = get_accelerator().current_device_name() + sync = AffineWeightSync({'w': replicated_layout_map(replicated_map(shape, 1), [[0]])}, + {'w': replicated_layout_map(layout, [[2, 3]])}, torch.float32) + tp_rank = _tp_rank([[2, 3]], rank) + arena = None + if tp_rank is not None: + arena = WeightArena({'w': layout.shard_shapes[tp_rank]}, torch.float32, device) + x = torch.randn(4, shape[1], device=device) + if arena is not None: + # Warm up outside the graph, as capture requires. + y = x @ arena['w'].t() + graph = get_accelerator().create_graph() + with get_accelerator().capture_to_graph(graph): + y = x @ arena['w'].t() + + for version in range(2): + full = torch.randn(shape, device=device) + version + targets = dict(arena.items()) if arena is not None else {} + sync.sync({'w': full}, targets) + # Every rank needs the source values to check its shard. + dist.broadcast(full, 0) + if arena is not None: + get_accelerator().replay_graph(graph) + assert torch.equal(y, x @ layout.extract(full, tp_rank).t()) + + +def test_arena_views_are_aligned_disjoint_and_zeroed(): + shapes = {'a': (3, 5), 'b': (7, ), 'c': (2, 2, 3)} + # Several arenas alive at once, at different addresses: an allocation may happen to be aligned. + arenas = [WeightArena(shapes, torch.float16) for _ in range(8)] + for arena in arenas: + assert torch.all(arena.buffer == 0) + for index, (name, view) in enumerate(arena.items()): + assert view.shape == shapes[name] + assert view.data_ptr() % 256 == 0, name + view.fill_(index + 1) + # A view that overlapped another would have overwritten part of it. + for index, (name, view) in enumerate(arena.items()): + assert torch.all(view == index + 1), name + + +def test_replicated_layout_map_gives_each_replica_its_layout_shard(): + full = torch.arange(60, dtype=torch.float32).view(6, 10) + layout = contiguous_split_map((6, 10), [4, 3, 3], 1) + replicas = [[5, 0, 3], [1, 4, 2]] + target = replicated_layout_map(layout, replicas) + for ranks in replicas: + for tp_rank, rank in enumerate(ranks): + shard = target.extract(full, rank).view(layout.shard_shapes[tp_rank]) + assert torch.equal(shard, layout.extract(full, tp_rank)) + + +@pytest.mark.parametrize('replicas', [[[0, 1], [2]], [[0, 1], [1, 2]], []]) +def test_replicated_layout_map_refuses_replicas_that_do_not_fit_the_layout(replicas): + with pytest.raises(ValueError): + replicated_layout_map(contiguous_split_map((4, 4), [2, 2], 0), replicas)