From 20169fb99d43178336a5cf9961a433d2fd380a30 Mon Sep 17 00:00:00 2001 From: "Jin, Youzhi" Date: Mon, 9 Nov 2026 13:51:03 +0000 Subject: [PATCH 1/3] Check strided shard destinations without pairwise comparisons ParamAffineMap._validate_destinations compared every box-shaped piece of a rank with every other one, and validate() runs whenever a ZeRO, AutoEP or block-gather map is built. Sweep the boxes with boxes_disjoint first; only a rank that fails the sweep takes the pairwise walk, which names the shared address, so error messages are unchanged. Validating 2048 interleaved column pieces per rank drops from 6.7 s to 68 ms. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Jin, Youzhi --- deepspeed/checkpoint/affine.py | 12 ++++- .../unit/checkpoint/test_affine_shard_map.py | 44 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/deepspeed/checkpoint/affine.py b/deepspeed/checkpoint/affine.py index f2e98f353295..59395b3fe4d5 100644 --- a/deepspeed/checkpoint/affine.py +++ b/deepspeed/checkpoint/affine.py @@ -20,6 +20,7 @@ import torch +from .affine_geometry import LogicalBox, boxes_disjoint from .affine_geometry import (dot as _dot, product as _product, row_major_range_boxes as _row_major_range_boxes, row_major_strides as _row_major_strides, strided_address as _strided_address) @@ -292,8 +293,8 @@ def _validate_destinations(self, rank, pieces, shard_shape): cursor = stop # A column split's pieces have no interval, so the check above skips them. Check them as - # boxes against every other box-shaped piece of the rank instead. Ranks whose pieces are - # all dense are fully covered above and skip this pairwise pass. + # boxes of the shard grid instead. Ranks whose pieces are all dense are fully covered above + # and skip this pass. if not has_strided_piece: return boxes = [] @@ -313,6 +314,13 @@ def _validate_destinations(self, rank, pieces, shard_shape): # not a box of the shard's grid, so the walk in validate_coverage() checks it instead. continue boxes.append(box) + # A sweep proves most layouts disjoint without comparing every pair; the pairwise walk + # only runs to name the address that two boxes share. + if boxes_disjoint([ + LogicalBox(tuple(starts), tuple(stop - start for start, stop in zip(starts, stops))) + for starts, stops in boxes + ]): + return for index, box in enumerate(boxes): for other_box in boxes[index + 1:]: shared = _first_shared_address(box, other_box, shard_shape) diff --git a/tests/unit/checkpoint/test_affine_shard_map.py b/tests/unit/checkpoint/test_affine_shard_map.py index 42ff675630e9..af6cc9c80e22 100644 --- a/tests/unit/checkpoint/test_affine_shard_map.py +++ b/tests/unit/checkpoint/test_affine_shard_map.py @@ -18,6 +18,8 @@ process group or an accelerator. """ +import random + import pytest import torch @@ -589,6 +591,48 @@ def refuse_walk(self, base, strides): affine_map.validate() +def _random_column_block_map(rng): + """Column blocks of row bands tiling one shard, with one block sometimes shifted onto another.""" + rows, cols = rng.randint(1, 6), rng.randint(2, 8) + col_cuts = sorted(rng.sample(range(1, cols), rng.randint(1, cols - 1))) + blocks = [] + for low, high in zip([0] + col_cuts, col_cuts + [cols]): + row_cuts = sorted(rng.sample(range(1, rows), rng.randint(0, rows - 1))) if rows > 1 else [] + blocks += [[top, low, bottom - top, high - low] for top, bottom in zip([0] + row_cuts, row_cuts + [rows])] + if rng.random() < 0.5: + block = rng.choice(blocks) + block[0] = rng.randint(0, rows - block[2]) + block[1] = rng.randint(0, cols - block[3]) + rng.shuffle(blocks) + pieces = [ + AffinePiece((height, width), top * cols + left, (cols, 1), top * cols + left, (cols, 1), [0]) + for top, left, height, width in blocks + ] + return ParamAffineMap((rows, cols), {0: (rows, cols)}, {0: pieces}) + + +def _validation_outcome(affine_map): + try: + affine_map.validate() + except ValueError as error: + return str(error) + return None + + +@pytest.mark.parametrize('seed', range(200)) +def test_strided_destination_sweep_agrees_with_pairwise_walk(seed, monkeypatch): + """The sweep accepts exactly the layouts that write no address twice, and errors read as before.""" + from deepspeed.checkpoint import affine as affine_module + + affine_map = _random_column_block_map(random.Random(seed)) + written = [(piece.dest_offset + row * piece.dest_strides[0] + column) for piece in affine_map.pieces_by_rank[0] + for row in range(piece.shape[0]) for column in range(piece.shape[1])] + fast = _validation_outcome(affine_map) + assert (fast is None) == (len(set(written)) == len(written)) + monkeypatch.setattr(affine_module, 'boxes_disjoint', lambda boxes: False) + assert fast == _validation_outcome(affine_map) + + # Guards added in review. Each one covers a way a map could produce a plausible but wrong # parameter rather than failing, which is the failure mode that matters for a checkpoint. From b5d12cfd90fd1363b75b93ac6c861416fef3e582 Mon Sep 17 00:00:00 2001 From: "Jin, Youzhi" Date: Wed, 11 Nov 2026 13:41:24 +0000 Subject: [PATCH 2/3] Use affine_geometry for destination boxes and address ranges _dest_box() now returns a LogicalBox built with coordinate(), so _validate_destinations() hands boxes straight to boxes_disjoint() and names a shared address from box.intersection() and dot(). The hand-written _first_shared_address() and divmod loop are gone. address_range() and overlaps_itself() move to affine_geometry. The transfer executor reuses address_range() for its buffer-reach check and dot() for chunk offsets. Error messages and behavior are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Jin, Youzhi --- deepspeed/checkpoint/affine.py | 90 +++++-------------- deepspeed/checkpoint/affine_geometry.py | 32 ++++++- .../checkpoint/affine_transfer_executor.py | 20 ++--- 3 files changed, 61 insertions(+), 81 deletions(-) diff --git a/deepspeed/checkpoint/affine.py b/deepspeed/checkpoint/affine.py index 59395b3fe4d5..0d424de577c1 100644 --- a/deepspeed/checkpoint/affine.py +++ b/deepspeed/checkpoint/affine.py @@ -20,7 +20,7 @@ import torch -from .affine_geometry import LogicalBox, boxes_disjoint +from .affine_geometry import LogicalBox, address_range, boxes_disjoint, coordinate, overlaps_itself from .affine_geometry import (dot as _dot, product as _product, row_major_range_boxes as _row_major_range_boxes, row_major_strides as _row_major_strides, strided_address as _strided_address) @@ -166,49 +166,26 @@ def _dest_interval(piece): def _dest_box(piece, shard_shape): - """Return a piece's destination as per-axis ``(starts, stops)`` in the shard, or None. + """Return a piece's destination as a box of the shard's grid, or None. A column split writes each piece into a few columns of every shard row, so its destination is not one interval. It is still a box in the shard's own grid, because its strides are the shard's row-major strides, and a box can be checked for bounds and - overlap from its corners without walking any elements. + overlap from its corners without walking any elements. A view that wraps past the end of + a shard row is not a box of that grid; the walk in validate_coverage() checks it instead. """ if piece.numel == 0 or len(piece.shape) != len(shard_shape) or len(piece.dest_strides) != len(shard_shape): return None shard_strides = _row_major_strides(shard_shape) if any(size > 1 and got != want for size, got, want in zip(piece.shape, piece.dest_strides, shard_strides)): return None - - starts = [] - remainder = piece.dest_offset - for stride in shard_strides: - start, remainder = divmod(remainder, stride) - starts.append(start) - stops = [start + size for start, size in zip(starts, piece.shape)] - return starts, stops - - -def _dest_address_range(piece): - """Return the lowest and highest shard address a piece writes.""" - low = piece.dest_offset - high = piece.dest_offset - for size, stride in zip(piece.shape, piece.dest_strides): - reach = (size - 1) * stride - if reach < 0: - low += reach - else: - high += reach - return low, high - - -def _first_shared_address(box, other_box, shard_shape): - """Return the first shard address two boxes both write, or None when they are disjoint.""" - corner = [] - for start, stop, other_start, other_stop in zip(box[0], box[1], other_box[0], other_box[1]): - if max(start, other_start) >= min(stop, other_stop): - return None - corner.append(max(start, other_start)) - return sum(position * stride for position, stride in zip(corner, _row_major_strides(shard_shape))) + origin = coordinate(piece.dest_offset, shard_strides, shard_shape) + if origin is None: + return None + box = LogicalBox(origin, piece.shape) + if any(high > dim for high, dim in zip(box.upper(), shard_shape)): + return None + return box class ParamAffineMap: @@ -262,7 +239,7 @@ def _validate_rank(self, rank): f'account for {held}.') for piece in pieces: for side, strides in (('source', piece.source_strides), ('destination', piece.dest_strides)): - if _overlaps_itself(piece.shape, strides): + if overlaps_itself(piece.shape, strides): raise ValueError(f'Rank {rank} piece {piece} reaches one {side} address twice within itself, ' 'so one of its elements would silently overwrite another.') self._validate_destinations(rank, pieces, self.shard_shapes[rank]) @@ -300,33 +277,24 @@ def _validate_destinations(self, rank, pieces, shard_shape): boxes = [] for piece in pieces: if piece.numel and _dest_interval(piece) is None: - low, high = _dest_address_range(piece) + low, high = address_range(piece.dest_offset, piece.shape, piece.dest_strides) if low < 0 or high >= shard_numel: raise ValueError(f'Rank {rank} piece {piece} writes shard addresses [{low}, {high}], ' f'outside the shard of {shard_numel} elements.') box = _dest_box(piece, shard_shape) - if box is None: - continue - starts, stops = box - fits = all(start >= 0 and stop <= dim for start, stop, dim in zip(starts, stops, shard_shape)) - if not fits: - # A view may wrap past the end of a shard row while staying inside the shard. It is - # not a box of the shard's grid, so the walk in validate_coverage() checks it instead. - continue - boxes.append(box) + if box is not None: + boxes.append(box) # A sweep proves most layouts disjoint without comparing every pair; the pairwise walk # only runs to name the address that two boxes share. - if boxes_disjoint([ - LogicalBox(tuple(starts), tuple(stop - start for start, stop in zip(starts, stops))) - for starts, stops in boxes - ]): + if boxes_disjoint(boxes): return + shard_strides = _row_major_strides(shard_shape) for index, box in enumerate(boxes): for other_box in boxes[index + 1:]: - shared = _first_shared_address(box, other_box, shard_shape) - if shared is not None: - raise ValueError(f'Rank {rank} writes shard address {shared} twice, so one piece would ' - 'silently overwrite another.') + shared = box.intersection(other_box) + if not shared.is_empty: + raise ValueError(f'Rank {rank} writes shard address {_dot(shared.origin, shard_strides)} ' + 'twice, so one piece would silently overwrite another.') def validate_coverage(self): """Full check: each shard address is written once, the shards cover the parameter, @@ -520,22 +488,6 @@ def _scaled(tensor, scale, scale_power): _UNWRITTEN, _SINGLE_HOLDER, _REPLICA = 0, 1, 2 -def _overlaps_itself(shape, strides): - """Whether two positions of a view with this shape and these strides share an address. - - Taking the axes that move in order of stride, the view is free of overlap when each stride - clears every address the axes before it reach. That holds for every layout the builders here - produce. A view that interleaves its axes some other way is refused even where it would not - collide, which keeps this to a few comparisons per piece instead of a walk over its elements. - """ - reach = 0 - for stride, size in sorted((stride, size) for size, stride in zip(shape, strides) if size > 1): - if stride <= reach: - return True - reach += stride * (size - 1) - return False - - def _place(placed, piece, rank): """Mark the shard addresses ``piece`` occupies in ``placed``, refusing one already taken. diff --git a/deepspeed/checkpoint/affine_geometry.py b/deepspeed/checkpoint/affine_geometry.py index 1ef764f8e64b..d02562ab1bd7 100644 --- a/deepspeed/checkpoint/affine_geometry.py +++ b/deepspeed/checkpoint/affine_geometry.py @@ -10,7 +10,8 @@ from typing import List, Optional, Sequence, Tuple __all__ = [ - 'LogicalBox', 'coordinate', 'dot', 'product', 'row_major_range_boxes', 'row_major_strides', 'strided_address' + 'LogicalBox', 'address_range', 'coordinate', 'dot', 'overlaps_itself', 'product', 'row_major_range_boxes', + 'row_major_strides', 'strided_address' ] @@ -46,6 +47,35 @@ def dot(index, strides): return sum(position * stride for position, stride in zip(index, strides)) +def address_range(offset: int, shape: Sequence[int], strides: Sequence[int]) -> Tuple[int, int]: + """The lowest and highest address a strided view reaches, both inclusive.""" + low = offset + high = offset + for size, stride in zip(shape, strides): + reach = (size - 1) * stride + if reach < 0: + low += reach + else: + high += reach + return low, high + + +def overlaps_itself(shape: Sequence[int], strides: Sequence[int]) -> bool: + """Whether two positions of a view with this shape and these strides share an address. + + Taking the axes that move in order of stride, the view is free of overlap when each stride + clears every address the axes before it reach. That holds for every layout the builders + produce. A view that interleaves its axes some other way is refused even where it would not + collide, which keeps this to a few comparisons per view instead of a walk over its elements. + """ + reach = 0 + for stride, size in sorted((stride, size) for size, stride in zip(shape, strides) if size > 1): + if stride <= reach: + return True + reach += stride * (size - 1) + return False + + def strided_address(shape, offset, strides, index): address = offset for size, stride in reversed(list(zip(shape, strides))): diff --git a/deepspeed/checkpoint/affine_transfer_executor.py b/deepspeed/checkpoint/affine_transfer_executor.py index a9d769aae731..2ec10ad02124 100644 --- a/deepspeed/checkpoint/affine_transfer_executor.py +++ b/deepspeed/checkpoint/affine_transfer_executor.py @@ -51,6 +51,7 @@ import torch from deepspeed.accelerator import get_accelerator +from .affine_geometry import address_range, dot from .affine_transfer import TransferPlan, TransferSegment __all__ = [ @@ -137,16 +138,13 @@ def _chunks(segment: TransferSegment, max_elements: int): origins = [range(0, extent, step) for extent, step in zip(segment.shape, tile)] for origin in product(*origins): shape = tuple(min(step, extent - low) for step, extent, low in zip(tile, segment.shape, origin)) - yield TransferSegment( - source_rank=segment.source_rank, - source_offset=segment.source_offset + sum(low * stride - for low, stride in zip(origin, segment.source_strides)), - source_strides=segment.source_strides, - target_rank=segment.target_rank, - target_offset=segment.target_offset + sum(low * stride - for low, stride in zip(origin, segment.target_strides)), - target_strides=segment.target_strides, - shape=shape) + yield TransferSegment(source_rank=segment.source_rank, + source_offset=segment.source_offset + dot(origin, segment.source_strides), + source_strides=segment.source_strides, + target_rank=segment.target_rank, + target_offset=segment.target_offset + dot(origin, segment.target_strides), + target_strides=segment.target_strides, + shape=shape) def build_operations(plan: TransferPlan, endpoints: TransferEndpoints, budget: StagingBudget, @@ -236,7 +234,7 @@ def _validate_buffers(plan: TransferPlan, source_buffers: Mapping[int, torch.Ten continue if location not in buffers: raise AffineTransferExecutionError(f'{which} location {location} is owned here but holds no buffer') - reach = offset + sum((size - 1) * stride for size, stride in zip(segment.shape, strides)) + _, reach = address_range(offset, segment.shape, strides) if reach >= buffers[location].numel(): raise AffineTransferExecutionError( f'{which} of {segment!r} reaches element {reach} of a {buffers[location].numel()}-element buffer') From b06a4c83d422bbd20d9621db3cfbec13a216b2be Mon Sep 17 00:00:00 2001 From: "Jin, Youzhi" Date: Thu, 12 Nov 2026 14:23:59 +0000 Subject: [PATCH 3/3] Assert the refused address instead of forcing the pairwise walk The randomized destination test monkeypatched affine.boxes_disjoint to compare the sweep with the pairwise walk, which ties it to that helper. Check the contract instead: a refusal names an address the map writes twice. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Jin, Youzhi --- .../unit/checkpoint/test_affine_shard_map.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/unit/checkpoint/test_affine_shard_map.py b/tests/unit/checkpoint/test_affine_shard_map.py index af6cc9c80e22..d8d1b3803465 100644 --- a/tests/unit/checkpoint/test_affine_shard_map.py +++ b/tests/unit/checkpoint/test_affine_shard_map.py @@ -19,6 +19,7 @@ """ import random +import re import pytest import torch @@ -620,17 +621,20 @@ def _validation_outcome(affine_map): @pytest.mark.parametrize('seed', range(200)) -def test_strided_destination_sweep_agrees_with_pairwise_walk(seed, monkeypatch): - """The sweep accepts exactly the layouts that write no address twice, and errors read as before.""" - from deepspeed.checkpoint import affine as affine_module - +def test_strided_destination_check_refuses_exactly_double_writes(seed): + """validate() accepts exactly the layouts that write no address twice, and a refusal names + an address that is written twice.""" affine_map = _random_column_block_map(random.Random(seed)) written = [(piece.dest_offset + row * piece.dest_strides[0] + column) for piece in affine_map.pieces_by_rank[0] for row in range(piece.shape[0]) for column in range(piece.shape[1])] - fast = _validation_outcome(affine_map) - assert (fast is None) == (len(set(written)) == len(written)) - monkeypatch.setattr(affine_module, 'boxes_disjoint', lambda boxes: False) - assert fast == _validation_outcome(affine_map) + outcome = _validation_outcome(affine_map) + assert (outcome is None) == (len(set(written)) == len(written)) + if outcome is not None: + match = re.fullmatch( + r'Rank 0 writes shard address (\d+) twice, so one piece would silently overwrite ' + r'another\.', outcome) + assert match is not None, outcome + assert written.count(int(match.group(1))) > 1 # Guards added in review. Each one covers a way a map could produce a plausible but wrong