Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 26 additions & 66 deletions deepspeed/checkpoint/affine.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import torch

from .affine_geometry import LogicalBox, address_range, boxes_disjoint, coordinate, overlaps_itself
from .affine_geometry import (dot as _dot, product as _product, row_major_range_boxes as _row_major_range_boxes,
row_major_strides as _row_major_strides, strided_address as _strided_address)

Expand Down Expand Up @@ -165,49 +166,26 @@ def _dest_interval(piece):


def _dest_box(piece, shard_shape):
"""Return a piece's destination as per-axis ``(starts, stops)`` in the shard, or None.
"""Return a piece's destination as a box of the shard's grid, or None.

A column split writes each piece into a few columns of every shard row, so its
destination is not one interval. It is still a box in the shard's own grid, because its
strides are the shard's row-major strides, and a box can be checked for bounds and
overlap from its corners without walking any elements.
overlap from its corners without walking any elements. A view that wraps past the end of
a shard row is not a box of that grid; the walk in validate_coverage() checks it instead.
"""
if piece.numel == 0 or len(piece.shape) != len(shard_shape) or len(piece.dest_strides) != len(shard_shape):
return None
shard_strides = _row_major_strides(shard_shape)
if any(size > 1 and got != want for size, got, want in zip(piece.shape, piece.dest_strides, shard_strides)):
return None

starts = []
remainder = piece.dest_offset
for stride in shard_strides:
start, remainder = divmod(remainder, stride)
starts.append(start)
stops = [start + size for start, size in zip(starts, piece.shape)]
return starts, stops


def _dest_address_range(piece):
"""Return the lowest and highest shard address a piece writes."""
low = piece.dest_offset
high = piece.dest_offset
for size, stride in zip(piece.shape, piece.dest_strides):
reach = (size - 1) * stride
if reach < 0:
low += reach
else:
high += reach
return low, high


def _first_shared_address(box, other_box, shard_shape):
"""Return the first shard address two boxes both write, or None when they are disjoint."""
corner = []
for start, stop, other_start, other_stop in zip(box[0], box[1], other_box[0], other_box[1]):
if max(start, other_start) >= min(stop, other_stop):
return None
corner.append(max(start, other_start))
return sum(position * stride for position, stride in zip(corner, _row_major_strides(shard_shape)))
origin = coordinate(piece.dest_offset, shard_strides, shard_shape)
if origin is None:
return None
box = LogicalBox(origin, piece.shape)
if any(high > dim for high, dim in zip(box.upper(), shard_shape)):
return None
return box


class ParamAffineMap:
Expand Down Expand Up @@ -261,7 +239,7 @@ def _validate_rank(self, rank):
f'account for {held}.')
for piece in pieces:
for side, strides in (('source', piece.source_strides), ('destination', piece.dest_strides)):
if _overlaps_itself(piece.shape, strides):
if overlaps_itself(piece.shape, strides):
raise ValueError(f'Rank {rank} piece {piece} reaches one {side} address twice within itself, '
'so one of its elements would silently overwrite another.')
self._validate_destinations(rank, pieces, self.shard_shapes[rank])
Expand Down Expand Up @@ -292,33 +270,31 @@ def _validate_destinations(self, rank, pieces, shard_shape):
cursor = stop

# A column split's pieces have no interval, so the check above skips them. Check them as
# boxes against every other box-shaped piece of the rank instead. Ranks whose pieces are
# all dense are fully covered above and skip this pairwise pass.
# boxes of the shard grid instead. Ranks whose pieces are all dense are fully covered above
# and skip this pass.
if not has_strided_piece:
return
boxes = []
for piece in pieces:
if piece.numel and _dest_interval(piece) is None:
low, high = _dest_address_range(piece)
low, high = address_range(piece.dest_offset, piece.shape, piece.dest_strides)
if low < 0 or high >= shard_numel:
raise ValueError(f'Rank {rank} piece {piece} writes shard addresses [{low}, {high}], '
f'outside the shard of {shard_numel} elements.')
box = _dest_box(piece, shard_shape)
if box is None:
continue
starts, stops = box
fits = all(start >= 0 and stop <= dim for start, stop, dim in zip(starts, stops, shard_shape))
if not fits:
# A view may wrap past the end of a shard row while staying inside the shard. It is
# not a box of the shard's grid, so the walk in validate_coverage() checks it instead.
continue
boxes.append(box)
if box is not None:
boxes.append(box)
# A sweep proves most layouts disjoint without comparing every pair; the pairwise walk
# only runs to name the address that two boxes share.
if boxes_disjoint(boxes):
return
shard_strides = _row_major_strides(shard_shape)
for index, box in enumerate(boxes):
for other_box in boxes[index + 1:]:
shared = _first_shared_address(box, other_box, shard_shape)
if shared is not None:
raise ValueError(f'Rank {rank} writes shard address {shared} twice, so one piece would '
'silently overwrite another.')
shared = box.intersection(other_box)
if not shared.is_empty:
raise ValueError(f'Rank {rank} writes shard address {_dot(shared.origin, shard_strides)} '
'twice, so one piece would silently overwrite another.')

def validate_coverage(self):
"""Full check: each shard address is written once, the shards cover the parameter,
Expand Down Expand Up @@ -512,22 +488,6 @@ def _scaled(tensor, scale, scale_power):
_UNWRITTEN, _SINGLE_HOLDER, _REPLICA = 0, 1, 2


def _overlaps_itself(shape, strides):
"""Whether two positions of a view with this shape and these strides share an address.

Taking the axes that move in order of stride, the view is free of overlap when each stride
clears every address the axes before it reach. That holds for every layout the builders here
produce. A view that interleaves its axes some other way is refused even where it would not
collide, which keeps this to a few comparisons per piece instead of a walk over its elements.
"""
reach = 0
for stride, size in sorted((stride, size) for size, stride in zip(shape, strides) if size > 1):
if stride <= reach:
return True
reach += stride * (size - 1)
return False


def _place(placed, piece, rank):
"""Mark the shard addresses ``piece`` occupies in ``placed``, refusing one already taken.

Expand Down
32 changes: 31 additions & 1 deletion deepspeed/checkpoint/affine_geometry.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
from typing import List, Optional, Sequence, Tuple

__all__ = [
'LogicalBox', 'coordinate', 'dot', 'product', 'row_major_range_boxes', 'row_major_strides', 'strided_address'
'LogicalBox', 'address_range', 'coordinate', 'dot', 'overlaps_itself', 'product', 'row_major_range_boxes',
'row_major_strides', 'strided_address'
]


Expand Down Expand Up @@ -46,6 +47,35 @@ def dot(index, strides):
return sum(position * stride for position, stride in zip(index, strides))


def address_range(offset: int, shape: Sequence[int], strides: Sequence[int]) -> Tuple[int, int]:
"""The lowest and highest address a strided view reaches, both inclusive."""
low = offset
high = offset
for size, stride in zip(shape, strides):
reach = (size - 1) * stride
if reach < 0:
low += reach
else:
high += reach
return low, high


def overlaps_itself(shape: Sequence[int], strides: Sequence[int]) -> bool:
"""Whether two positions of a view with this shape and these strides share an address.

Taking the axes that move in order of stride, the view is free of overlap when each stride
clears every address the axes before it reach. That holds for every layout the builders
produce. A view that interleaves its axes some other way is refused even where it would not
collide, which keeps this to a few comparisons per view instead of a walk over its elements.
"""
reach = 0
for stride, size in sorted((stride, size) for size, stride in zip(shape, strides) if size > 1):
if stride <= reach:
return True
reach += stride * (size - 1)
return False


def strided_address(shape, offset, strides, index):
address = offset
for size, stride in reversed(list(zip(shape, strides))):
Expand Down
20 changes: 9 additions & 11 deletions deepspeed/checkpoint/affine_transfer_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import torch
from deepspeed.accelerator import get_accelerator

from .affine_geometry import address_range, dot
from .affine_transfer import TransferPlan, TransferSegment

__all__ = [
Expand Down Expand Up @@ -137,16 +138,13 @@ def _chunks(segment: TransferSegment, max_elements: int):
origins = [range(0, extent, step) for extent, step in zip(segment.shape, tile)]
for origin in product(*origins):
shape = tuple(min(step, extent - low) for step, extent, low in zip(tile, segment.shape, origin))
yield TransferSegment(
source_rank=segment.source_rank,
source_offset=segment.source_offset + sum(low * stride
for low, stride in zip(origin, segment.source_strides)),
source_strides=segment.source_strides,
target_rank=segment.target_rank,
target_offset=segment.target_offset + sum(low * stride
for low, stride in zip(origin, segment.target_strides)),
target_strides=segment.target_strides,
shape=shape)
yield TransferSegment(source_rank=segment.source_rank,
source_offset=segment.source_offset + dot(origin, segment.source_strides),
source_strides=segment.source_strides,
target_rank=segment.target_rank,
target_offset=segment.target_offset + dot(origin, segment.target_strides),
target_strides=segment.target_strides,
shape=shape)


def build_operations(plan: TransferPlan, endpoints: TransferEndpoints, budget: StagingBudget,
Expand Down Expand Up @@ -236,7 +234,7 @@ def _validate_buffers(plan: TransferPlan, source_buffers: Mapping[int, torch.Ten
continue
if location not in buffers:
raise AffineTransferExecutionError(f'{which} location {location} is owned here but holds no buffer')
reach = offset + sum((size - 1) * stride for size, stride in zip(segment.shape, strides))
_, reach = address_range(offset, segment.shape, strides)
if reach >= buffers[location].numel():
raise AffineTransferExecutionError(
f'{which} of {segment!r} reaches element {reach} of a {buffers[location].numel()}-element buffer')
Expand Down
48 changes: 48 additions & 0 deletions tests/unit/checkpoint/test_affine_shard_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
process group or an accelerator.
"""

import random
import re

import pytest
import torch

Expand Down Expand Up @@ -589,6 +592,51 @@ def refuse_walk(self, base, strides):
affine_map.validate()


def _random_column_block_map(rng):
"""Column blocks of row bands tiling one shard, with one block sometimes shifted onto another."""
rows, cols = rng.randint(1, 6), rng.randint(2, 8)
col_cuts = sorted(rng.sample(range(1, cols), rng.randint(1, cols - 1)))
blocks = []
for low, high in zip([0] + col_cuts, col_cuts + [cols]):
row_cuts = sorted(rng.sample(range(1, rows), rng.randint(0, rows - 1))) if rows > 1 else []
blocks += [[top, low, bottom - top, high - low] for top, bottom in zip([0] + row_cuts, row_cuts + [rows])]
if rng.random() < 0.5:
block = rng.choice(blocks)
block[0] = rng.randint(0, rows - block[2])
block[1] = rng.randint(0, cols - block[3])
rng.shuffle(blocks)
pieces = [
AffinePiece((height, width), top * cols + left, (cols, 1), top * cols + left, (cols, 1), [0])
for top, left, height, width in blocks
]
return ParamAffineMap((rows, cols), {0: (rows, cols)}, {0: pieces})


def _validation_outcome(affine_map):
try:
affine_map.validate()
except ValueError as error:
return str(error)
return None


@pytest.mark.parametrize('seed', range(200))
def test_strided_destination_check_refuses_exactly_double_writes(seed):
"""validate() accepts exactly the layouts that write no address twice, and a refusal names
an address that is written twice."""
affine_map = _random_column_block_map(random.Random(seed))
written = [(piece.dest_offset + row * piece.dest_strides[0] + column) for piece in affine_map.pieces_by_rank[0]
for row in range(piece.shape[0]) for column in range(piece.shape[1])]
outcome = _validation_outcome(affine_map)
assert (outcome is None) == (len(set(written)) == len(written))
if outcome is not None:
match = re.fullmatch(
r'Rank 0 writes shard address (\d+) twice, so one piece would silently overwrite '
r'another\.', outcome)
assert match is not None, outcome
assert written.count(int(match.group(1))) > 1


# Guards added in review. Each one covers a way a map could produce a plausible but wrong
# parameter rather than failing, which is the failure mode that matters for a checkpoint.

Expand Down
Loading