Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions ffn/inference/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ def queue_put(self, x):

return self._input_queue.put(x)

def put_exit(self) -> None:
self._input_queue.put('exit')

def is_shutting_down(self) -> bool:
return self.exit_request.is_set()

def queue_get(self, **kwargs):
return self._input_queue.get(**kwargs)

Expand Down Expand Up @@ -103,6 +109,9 @@ def finish(self):
"""Deregisters the client."""
raise NotImplementedError()

def is_shutting_down(self) -> bool:
return self._interface.is_shutting_down()

def predict(self, seed: np.ndarray, image: np.ndarray,
fetches: Sequence[str]) -> dict[str, np.ndarray]:
raise NotImplementedError()
Expand All @@ -127,9 +136,15 @@ def start(self) -> int:
def finish(self):
if self._client_id is None:
return
client_id = self._client_id
self._client_id = None
with self._interface.lock:
del self._interface.outputs[self._client_id]
self._interface.queue_put(-1 - self._client_id)
self._interface.outputs.pop(client_id, None)
try:
self._interface.queue_put(-1 - client_id)
except TerminationException:
# Executor is already shutting down; client output queue was removed.
pass

def predict(self, seed: np.ndarray, image: np.ndarray,
fetches: Sequence[str]) -> dict[str, np.ndarray]:
Expand Down Expand Up @@ -257,8 +272,8 @@ def stop_server(self):
if self.th_executor is None:
return
logging.info('Requesting executor shutdown.')
self._interface.queue_put('exit')
self._interface.exit_request.set()
self._interface.put_exit()
self.th_executor.join()
self.th_executor = None
logging.info('Executor shutdown complete.')
Expand Down
151 changes: 111 additions & 40 deletions ffn/inference/inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,30 @@ def visualize_state(seed_logits, pos: Tuple3i, movement_policy, dynimage):
dynimage.UpdateFromPIL(vis)


def _as_object_array(value):
arr = np.empty((), dtype=object)
arr[()] = value
return arr


def _unwrap_object_array(value):
if (
isinstance(value, np.ndarray)
and value.dtype == object
and value.ndim == 0
):
return value.item()
return value


# TODO(mjanusz): Add support for sparse inference.
class Canvas:
"""Tracks state of the inference progress and results within a subvolume."""

# Saving compressed NPZ files involves temporary files in the local
# filesystem. In order to avoid running out of disk/RAM, we serialize
# these operations.
io_lock = threading.Lock()
io_lock = threading.RLock()

def __init__(
self,
Expand Down Expand Up @@ -279,6 +295,22 @@ def _deregister_client(self):
self._exec_client.finish()
self._exec_client_id = None

def _is_shutting_down(self) -> bool:
return (
isinstance(self._exec_client, executor.ExecutorClient)
and self._exec_client.is_shutting_down()
)

def _on_sigterm_checkpoint(
self, partial_segment_iters: int, in_progress_seed: bool
) -> None:
if self.checkpoint_path:
self.save_checkpoint(
self.checkpoint_path,
partial_segment_iters,
in_progress_seed=in_progress_seed,
)

def __del__(self):
# Note that the presence of this method will cause a memory leak in
# case the Canvas object is part of a reference cycle. Use weakref.proxy
Expand Down Expand Up @@ -508,7 +540,11 @@ def segment_at(
self.counters['skip_restriced_pos'].Increment()
continue

pred = self.update_at(pos)
try:
pred = self.update_at(pos)
except executor.TerminationException:
self._on_sigterm_checkpoint(num_iters, in_progress_seed=True)
raise
self._min_pos = np.minimum(self._min_pos, pos)
self._max_pos = np.maximum(self._max_pos, pos)
num_iters += 1
Expand Down Expand Up @@ -557,6 +593,12 @@ def segment_all(self, seed_policy=seed.PolicyPeaks, partial_segment_iters=0):
mbd = np.array([mbd.z, mbd.y, mbd.x])

for pos in TimedIter(self.seed_policy, self.counters, 'seed-policy'):
if self._is_shutting_down():
self._on_sigterm_checkpoint(
partial_segment_iters, in_progress_seed=True
)
raise executor.TerminationException()

# When starting a new segment the move_threshold on the probability
# should be ignored when determining if the position is valid.
if not (
Expand All @@ -568,7 +610,9 @@ def segment_all(self, seed_policy=seed.PolicyPeaks, partial_segment_iters=0):
continue

if not partial_segment_iters:
self._maybe_save_checkpoint(partial_segment_iters=0)
self._maybe_save_checkpoint(
partial_segment_iters=0, in_progress_seed=True
)

# Too close to an existing segment?
low = np.array(pos) - mbd
Expand Down Expand Up @@ -754,20 +798,24 @@ def restore_checkpoint(self, path: str) -> int:
segmented_voxels = np.sum(self.segmentation != 0)
self.counters['voxels-segmented'].Set(segmented_voxels)
self._max_id = int(np.max(self.segmentation))
if 'max_id' in data:
self._max_id = max(self._max_id, int(data['max_id']))
self._min_pos = data['min_pos']
self._max_pos = data['max_pos']

self.movement_policy.restore_state(data['movement_policy'])
self.movement_policy.restore_state(
_unwrap_object_array(data['movement_policy'])
)

# When restoring the state of a previously unused Canvas, the seed
# policy will not be defined. We just save the seed policy state here
# for future use in .segment_all().
self._seed_policy_state = data['seed_policy_state']
self._seed_policy_state = _unwrap_object_array(data['seed_policy_state'])

self.counters.loads(data['counters'].item())

if 'partial_segment_iters' in data:
partial_segment_iters = data['partial_segment_iters']
partial_segment_iters = int(data['partial_segment_iters'])
else:
partial_segment_iters = 0

Expand All @@ -777,50 +825,69 @@ def restore_checkpoint(self, path: str) -> int:
self.log_info('Inference checkpoint restored.')
return partial_segment_iters

def save_checkpoint(self, path: str, partial_segment_iters: int):
def save_checkpoint(
self,
path: str | None,
partial_segment_iters: int,
in_progress_seed: bool | None = None,
):
"""Saves an inference checkpoint to `path`.

Args:
path: path at which the checkpoint should be saved
partial_segment_iters: if this checkpoint is for an in-flight segment,
number of inference iterations already executed; 0 otherwise
in_progress_seed: whether the current seed from `seed_policy` has already
been consumed for an unfinished segment; defaults to
`partial_segment_iters > 0` when omitted.
"""
if not path:
return
if in_progress_seed is None:
in_progress_seed = partial_segment_iters > 0

self.log_info('Saving inference checkpoint to %s.', path)
with timer_counter(self.counters, 'save_checkpoint'):
gfile.makedirs(os.path.dirname(path))
with storage.atomic_file(path) as fd:
seed_policy_state = None
if self.seed_policy is not None:
seed_policy_state = self.seed_policy.get_state(
partial_segment_iters > 0
)
with Canvas.io_lock:
with timer_counter(self.counters, 'save_checkpoint'):
gfile.makedirs(os.path.dirname(path))
with storage.atomic_file(path) as fd:
seed_policy_state = None
if self.seed_policy is not None:
seed_policy_state = self.seed_policy.get_state(
previous=in_progress_seed
)

aux = {}
if self.keep_probability_maps:
aux['seg_qprob'] = self.seg_prob

np.savez_compressed(
fd,
movement_policy=np.asarray(
self.movement_policy.get_state(), dtype=object
),
segmentation=self.segmentation,
seed=self.seed,
origins=self.origins, # pyrefly: ignore[bad-argument-type]
overlaps=self.overlaps, # pyrefly: ignore[bad-argument-type]
min_pos=self._min_pos,
max_pos=self._max_pos,
history=np.array(self.history),
history_deleted=np.array(self.history_deleted),
seed_policy_state=np.asarray(seed_policy_state, dtype=object),
counters=self.counters.dumps(),
partial_segment_iters=partial_segment_iters,
hosts=self._hosts,
**aux,
)
aux = {}
if self.keep_probability_maps:
aux['seg_qprob'] = self.seg_prob

np.savez_compressed(
fd,
movement_policy=_as_object_array(
self.movement_policy.get_state()
),
segmentation=self.segmentation,
seed=self.seed,
origins=self.origins, # pyrefly: ignore[bad-argument-type]
overlaps=self.overlaps, # pyrefly: ignore[bad-argument-type]
min_pos=self._min_pos,
max_pos=self._max_pos,
max_id=int(self._max_id),
history=np.array(self.history),
history_deleted=np.array(self.history_deleted),
seed_policy_state=_as_object_array(seed_policy_state),
counters=self.counters.dumps(),
partial_segment_iters=int(partial_segment_iters),
hosts=self._hosts,
**aux,
)
self.log_info('Inference checkpoint saved.')

def _maybe_save_checkpoint(self, partial_segment_iters=0):
def _maybe_save_checkpoint(
self,
partial_segment_iters: int = 0,
in_progress_seed: bool | None = None,
):
"""Attempts to save a checkpoint.

A checkpoint is only saved if the canvas is configured to keep checkpoints
Expand All @@ -829,6 +896,8 @@ def _maybe_save_checkpoint(self, partial_segment_iters=0):
Args:
partial_segment_iters: if this checkpoint is for an in-flight segment,
number of inference iterations already executed; 0 otherwise
in_progress_seed: whether the current seed from `seed_policy` has already
been consumed for an unfinished segment.
"""
if self.checkpoint_path is None or self.checkpoint_interval_sec <= 0:
return
Expand All @@ -838,6 +907,8 @@ def _maybe_save_checkpoint(self, partial_segment_iters=0):

with Canvas.io_lock:
self.save_checkpoint(
self.checkpoint_path, partial_segment_iters=partial_segment_iters
self.checkpoint_path,
partial_segment_iters=partial_segment_iters,
in_progress_seed=in_progress_seed,
)
self.checkpoint_last = time.time()
2 changes: 1 addition & 1 deletion ffn/inference/inference_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def IncrementBy(self, x, export=True):
self._update()

if self._parent is not None:
self._parent.IncrementBy(x)
self._parent.IncrementBy(x, export=export)

def Set(self, x, export=True):
"""Sets the counter value to 'x'.
Expand Down
27 changes: 19 additions & 8 deletions ffn/inference/movement.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,32 +101,42 @@ def __init__(self, canvas, deltas=(4, 8, 8), score_threshold=0.9):
self.done_rounded_coords = set()
self.score_threshold = score_threshold
self._start_pos = None
self._in_flight = None
super(FaceMaxMovementPolicy, self).__init__(canvas, deque([]), deltas)

def reset_state(self, start_pos):
self.scored_coords = deque([])
self.done_rounded_coords = set()
self._start_pos = start_pos
self._in_flight = None

def get_state(self):
return [(self.scored_coords, self.done_rounded_coords, self._start_pos)]
scored_coords = deque(self.scored_coords)
if self._in_flight is not None:
scored_coords.appendleft(self._in_flight)
return [(scored_coords, set(self.done_rounded_coords), self._start_pos)]

def restore_state(self, state):
self.scored_coords, self.done_rounded_coords, self._start_pos = state[0]
scored_coords, done_rounded_coords, start_pos = state[0]
self.scored_coords = deque(scored_coords)
self.done_rounded_coords = set(done_rounded_coords)
self._start_pos = (
tuple(int(x) for x in start_pos) if start_pos is not None else None
)
self._in_flight = None

def __next__(self):
"""Pops positions from queue until a valid one is found and returns it."""
self._in_flight = None
while self.scored_coords:
_, coord = self.scored_coords.popleft()
score, coord = self.scored_coords.popleft()
coord = tuple(coord)
if self.quantize_pos(coord) in self.done_rounded_coords:
continue
if self.canvas.is_valid_pos(coord):
break
else: # Else goes with while, not with if!
raise StopIteration()

return tuple(coord)
self._in_flight = (score, coord)
return coord
raise StopIteration()

def quantize_pos(self, pos):
"""Quantizes the positions symmetrically to a grid downsampled by deltas."""
Expand All @@ -140,6 +150,7 @@ def quantize_pos(self, pos):

def update(self, prob_map, position):
"""Adds movements to queue for the cuboid face maxima of ``prob_map``."""
self._in_flight = None
qpos = self.quantize_pos(position)
self.done_rounded_coords.add(qpos)

Expand Down
15 changes: 9 additions & 6 deletions ffn/inference/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,7 @@ def run(self, corner: Tuple3i, subvol_size: Tuple3i, reset_counters=True):

assert alignment is not None

partial_segment_iters = 0
if gfile.exists(cpoint_path):
partial_segment_iters = canvas.restore_checkpoint(cpoint_path)

Expand All @@ -528,12 +529,14 @@ def run(self, corner: Tuple3i, subvol_size: Tuple3i, reset_counters=True):
# keep a record of canvases executing inference. this is used to save
# checkpoints in case termination is requested.
self.canvases[corner] = canvas
canvas.segment_all(
seed_policy=self.get_seed_policy(corner, subvol_size),
partial_segment_iters=partial_segment_iters, # pyrefly: ignore[unbound-name]
)
self.save_segmentation(canvas, alignment, seg_path, prob_path)
del self.canvases[corner]
try:
canvas.segment_all(
seed_policy=self.get_seed_policy(corner, subvol_size),
partial_segment_iters=partial_segment_iters,
)
self.save_segmentation(canvas, alignment, seg_path, prob_path)
finally:
self.canvases.pop(corner, None)

# Attempt to remove the checkpoint file now that we no longer need it.
try:
Expand Down
Loading
Loading