diff --git a/deepspeed/checkpoint/affine.py b/deepspeed/checkpoint/affine.py index f2e98f353295..0d424de577c1 100644 --- a/deepspeed/checkpoint/affine.py +++ b/deepspeed/checkpoint/affine.py @@ -20,6 +20,7 @@ import torch +from .affine_geometry import LogicalBox, address_range, boxes_disjoint, coordinate, overlaps_itself from .affine_geometry import (dot as _dot, product as _product, row_major_range_boxes as _row_major_range_boxes, row_major_strides as _row_major_strides, strided_address as _strided_address) @@ -165,49 +166,26 @@ def _dest_interval(piece): def _dest_box(piece, shard_shape): - """Return a piece's destination as per-axis ``(starts, stops)`` in the shard, or None. + """Return a piece's destination as a box of the shard's grid, or None. A column split writes each piece into a few columns of every shard row, so its destination is not one interval. It is still a box in the shard's own grid, because its strides are the shard's row-major strides, and a box can be checked for bounds and - overlap from its corners without walking any elements. + overlap from its corners without walking any elements. A view that wraps past the end of + a shard row is not a box of that grid; the walk in validate_coverage() checks it instead. """ if piece.numel == 0 or len(piece.shape) != len(shard_shape) or len(piece.dest_strides) != len(shard_shape): return None shard_strides = _row_major_strides(shard_shape) if any(size > 1 and got != want for size, got, want in zip(piece.shape, piece.dest_strides, shard_strides)): return None - - starts = [] - remainder = piece.dest_offset - for stride in shard_strides: - start, remainder = divmod(remainder, stride) - starts.append(start) - stops = [start + size for start, size in zip(starts, piece.shape)] - return starts, stops - - -def _dest_address_range(piece): - """Return the lowest and highest shard address a piece writes.""" - low = piece.dest_offset - high = piece.dest_offset - for size, stride in zip(piece.shape, piece.dest_strides): - reach = (size - 1) * stride - if reach < 0: - low += reach - else: - high += reach - return low, high - - -def _first_shared_address(box, other_box, shard_shape): - """Return the first shard address two boxes both write, or None when they are disjoint.""" - corner = [] - for start, stop, other_start, other_stop in zip(box[0], box[1], other_box[0], other_box[1]): - if max(start, other_start) >= min(stop, other_stop): - return None - corner.append(max(start, other_start)) - return sum(position * stride for position, stride in zip(corner, _row_major_strides(shard_shape))) + origin = coordinate(piece.dest_offset, shard_strides, shard_shape) + if origin is None: + return None + box = LogicalBox(origin, piece.shape) + if any(high > dim for high, dim in zip(box.upper(), shard_shape)): + return None + return box class ParamAffineMap: @@ -261,7 +239,7 @@ def _validate_rank(self, rank): f'account for {held}.') for piece in pieces: for side, strides in (('source', piece.source_strides), ('destination', piece.dest_strides)): - if _overlaps_itself(piece.shape, strides): + if overlaps_itself(piece.shape, strides): raise ValueError(f'Rank {rank} piece {piece} reaches one {side} address twice within itself, ' 'so one of its elements would silently overwrite another.') self._validate_destinations(rank, pieces, self.shard_shapes[rank]) @@ -292,33 +270,31 @@ def _validate_destinations(self, rank, pieces, shard_shape): cursor = stop # A column split's pieces have no interval, so the check above skips them. Check them as - # boxes against every other box-shaped piece of the rank instead. Ranks whose pieces are - # all dense are fully covered above and skip this pairwise pass. + # boxes of the shard grid instead. Ranks whose pieces are all dense are fully covered above + # and skip this pass. if not has_strided_piece: return boxes = [] for piece in pieces: if piece.numel and _dest_interval(piece) is None: - low, high = _dest_address_range(piece) + low, high = address_range(piece.dest_offset, piece.shape, piece.dest_strides) if low < 0 or high >= shard_numel: raise ValueError(f'Rank {rank} piece {piece} writes shard addresses [{low}, {high}], ' f'outside the shard of {shard_numel} elements.') box = _dest_box(piece, shard_shape) - if box is None: - continue - starts, stops = box - fits = all(start >= 0 and stop <= dim for start, stop, dim in zip(starts, stops, shard_shape)) - if not fits: - # A view may wrap past the end of a shard row while staying inside the shard. It is - # not a box of the shard's grid, so the walk in validate_coverage() checks it instead. - continue - boxes.append(box) + if box is not None: + boxes.append(box) + # A sweep proves most layouts disjoint without comparing every pair; the pairwise walk + # only runs to name the address that two boxes share. + if boxes_disjoint(boxes): + return + shard_strides = _row_major_strides(shard_shape) for index, box in enumerate(boxes): for other_box in boxes[index + 1:]: - shared = _first_shared_address(box, other_box, shard_shape) - if shared is not None: - raise ValueError(f'Rank {rank} writes shard address {shared} twice, so one piece would ' - 'silently overwrite another.') + shared = box.intersection(other_box) + if not shared.is_empty: + raise ValueError(f'Rank {rank} writes shard address {_dot(shared.origin, shard_strides)} ' + 'twice, so one piece would silently overwrite another.') def validate_coverage(self): """Full check: each shard address is written once, the shards cover the parameter, @@ -512,22 +488,6 @@ def _scaled(tensor, scale, scale_power): _UNWRITTEN, _SINGLE_HOLDER, _REPLICA = 0, 1, 2 -def _overlaps_itself(shape, strides): - """Whether two positions of a view with this shape and these strides share an address. - - Taking the axes that move in order of stride, the view is free of overlap when each stride - clears every address the axes before it reach. That holds for every layout the builders here - produce. A view that interleaves its axes some other way is refused even where it would not - collide, which keeps this to a few comparisons per piece instead of a walk over its elements. - """ - reach = 0 - for stride, size in sorted((stride, size) for size, stride in zip(shape, strides) if size > 1): - if stride <= reach: - return True - reach += stride * (size - 1) - return False - - def _place(placed, piece, rank): """Mark the shard addresses ``piece`` occupies in ``placed``, refusing one already taken. diff --git a/deepspeed/checkpoint/affine_geometry.py b/deepspeed/checkpoint/affine_geometry.py index 1ef764f8e64b..d02562ab1bd7 100644 --- a/deepspeed/checkpoint/affine_geometry.py +++ b/deepspeed/checkpoint/affine_geometry.py @@ -10,7 +10,8 @@ from typing import List, Optional, Sequence, Tuple __all__ = [ - 'LogicalBox', 'coordinate', 'dot', 'product', 'row_major_range_boxes', 'row_major_strides', 'strided_address' + 'LogicalBox', 'address_range', 'coordinate', 'dot', 'overlaps_itself', 'product', 'row_major_range_boxes', + 'row_major_strides', 'strided_address' ] @@ -46,6 +47,35 @@ def dot(index, strides): return sum(position * stride for position, stride in zip(index, strides)) +def address_range(offset: int, shape: Sequence[int], strides: Sequence[int]) -> Tuple[int, int]: + """The lowest and highest address a strided view reaches, both inclusive.""" + low = offset + high = offset + for size, stride in zip(shape, strides): + reach = (size - 1) * stride + if reach < 0: + low += reach + else: + high += reach + return low, high + + +def overlaps_itself(shape: Sequence[int], strides: Sequence[int]) -> bool: + """Whether two positions of a view with this shape and these strides share an address. + + Taking the axes that move in order of stride, the view is free of overlap when each stride + clears every address the axes before it reach. That holds for every layout the builders + produce. A view that interleaves its axes some other way is refused even where it would not + collide, which keeps this to a few comparisons per view instead of a walk over its elements. + """ + reach = 0 + for stride, size in sorted((stride, size) for size, stride in zip(shape, strides) if size > 1): + if stride <= reach: + return True + reach += stride * (size - 1) + return False + + def strided_address(shape, offset, strides, index): address = offset for size, stride in reversed(list(zip(shape, strides))): diff --git a/deepspeed/checkpoint/affine_transfer_executor.py b/deepspeed/checkpoint/affine_transfer_executor.py index a9d769aae731..2ec10ad02124 100644 --- a/deepspeed/checkpoint/affine_transfer_executor.py +++ b/deepspeed/checkpoint/affine_transfer_executor.py @@ -51,6 +51,7 @@ import torch from deepspeed.accelerator import get_accelerator +from .affine_geometry import address_range, dot from .affine_transfer import TransferPlan, TransferSegment __all__ = [ @@ -137,16 +138,13 @@ def _chunks(segment: TransferSegment, max_elements: int): origins = [range(0, extent, step) for extent, step in zip(segment.shape, tile)] for origin in product(*origins): shape = tuple(min(step, extent - low) for step, extent, low in zip(tile, segment.shape, origin)) - yield TransferSegment( - source_rank=segment.source_rank, - source_offset=segment.source_offset + sum(low * stride - for low, stride in zip(origin, segment.source_strides)), - source_strides=segment.source_strides, - target_rank=segment.target_rank, - target_offset=segment.target_offset + sum(low * stride - for low, stride in zip(origin, segment.target_strides)), - target_strides=segment.target_strides, - shape=shape) + yield TransferSegment(source_rank=segment.source_rank, + source_offset=segment.source_offset + dot(origin, segment.source_strides), + source_strides=segment.source_strides, + target_rank=segment.target_rank, + target_offset=segment.target_offset + dot(origin, segment.target_strides), + target_strides=segment.target_strides, + shape=shape) def build_operations(plan: TransferPlan, endpoints: TransferEndpoints, budget: StagingBudget, @@ -236,7 +234,7 @@ def _validate_buffers(plan: TransferPlan, source_buffers: Mapping[int, torch.Ten continue if location not in buffers: raise AffineTransferExecutionError(f'{which} location {location} is owned here but holds no buffer') - reach = offset + sum((size - 1) * stride for size, stride in zip(segment.shape, strides)) + _, reach = address_range(offset, segment.shape, strides) if reach >= buffers[location].numel(): raise AffineTransferExecutionError( f'{which} of {segment!r} reaches element {reach} of a {buffers[location].numel()}-element buffer') diff --git a/tests/unit/checkpoint/test_affine_shard_map.py b/tests/unit/checkpoint/test_affine_shard_map.py index 42ff675630e9..d8d1b3803465 100644 --- a/tests/unit/checkpoint/test_affine_shard_map.py +++ b/tests/unit/checkpoint/test_affine_shard_map.py @@ -18,6 +18,9 @@ process group or an accelerator. """ +import random +import re + import pytest import torch @@ -589,6 +592,51 @@ def refuse_walk(self, base, strides): affine_map.validate() +def _random_column_block_map(rng): + """Column blocks of row bands tiling one shard, with one block sometimes shifted onto another.""" + rows, cols = rng.randint(1, 6), rng.randint(2, 8) + col_cuts = sorted(rng.sample(range(1, cols), rng.randint(1, cols - 1))) + blocks = [] + for low, high in zip([0] + col_cuts, col_cuts + [cols]): + row_cuts = sorted(rng.sample(range(1, rows), rng.randint(0, rows - 1))) if rows > 1 else [] + blocks += [[top, low, bottom - top, high - low] for top, bottom in zip([0] + row_cuts, row_cuts + [rows])] + if rng.random() < 0.5: + block = rng.choice(blocks) + block[0] = rng.randint(0, rows - block[2]) + block[1] = rng.randint(0, cols - block[3]) + rng.shuffle(blocks) + pieces = [ + AffinePiece((height, width), top * cols + left, (cols, 1), top * cols + left, (cols, 1), [0]) + for top, left, height, width in blocks + ] + return ParamAffineMap((rows, cols), {0: (rows, cols)}, {0: pieces}) + + +def _validation_outcome(affine_map): + try: + affine_map.validate() + except ValueError as error: + return str(error) + return None + + +@pytest.mark.parametrize('seed', range(200)) +def test_strided_destination_check_refuses_exactly_double_writes(seed): + """validate() accepts exactly the layouts that write no address twice, and a refusal names + an address that is written twice.""" + affine_map = _random_column_block_map(random.Random(seed)) + written = [(piece.dest_offset + row * piece.dest_strides[0] + column) for piece in affine_map.pieces_by_rank[0] + for row in range(piece.shape[0]) for column in range(piece.shape[1])] + outcome = _validation_outcome(affine_map) + assert (outcome is None) == (len(set(written)) == len(written)) + if outcome is not None: + match = re.fullmatch( + r'Rank 0 writes shard address (\d+) twice, so one piece would silently overwrite ' + r'another\.', outcome) + assert match is not None, outcome + assert written.count(int(match.group(1))) > 1 + + # Guards added in review. Each one covers a way a map could produce a plausible but wrong # parameter rather than failing, which is the failure mode that matters for a checkpoint.