diff --git a/ffn/inference/executor.py b/ffn/inference/executor.py index ca1b620..a5b02a1 100644 --- a/ffn/inference/executor.py +++ b/ffn/inference/executor.py @@ -68,6 +68,12 @@ def queue_put(self, x): return self._input_queue.put(x) + def put_exit(self) -> None: + self._input_queue.put('exit') + + def is_shutting_down(self) -> bool: + return self.exit_request.is_set() + def queue_get(self, **kwargs): return self._input_queue.get(**kwargs) @@ -103,6 +109,9 @@ def finish(self): """Deregisters the client.""" raise NotImplementedError() + def is_shutting_down(self) -> bool: + return self._interface.is_shutting_down() + def predict(self, seed: np.ndarray, image: np.ndarray, fetches: Sequence[str]) -> dict[str, np.ndarray]: raise NotImplementedError() @@ -127,9 +136,15 @@ def start(self) -> int: def finish(self): if self._client_id is None: return + client_id = self._client_id + self._client_id = None with self._interface.lock: - del self._interface.outputs[self._client_id] - self._interface.queue_put(-1 - self._client_id) + self._interface.outputs.pop(client_id, None) + try: + self._interface.queue_put(-1 - client_id) + except TerminationException: + # Executor is already shutting down; client output queue was removed. + pass def predict(self, seed: np.ndarray, image: np.ndarray, fetches: Sequence[str]) -> dict[str, np.ndarray]: @@ -257,8 +272,8 @@ def stop_server(self): if self.th_executor is None: return logging.info('Requesting executor shutdown.') - self._interface.queue_put('exit') self._interface.exit_request.set() + self._interface.put_exit() self.th_executor.join() self.th_executor = None logging.info('Executor shutdown complete.') diff --git a/ffn/inference/inference.py b/ffn/inference/inference.py index e32cab6..9e30a60 100644 --- a/ffn/inference/inference.py +++ b/ffn/inference/inference.py @@ -125,6 +125,22 @@ def visualize_state(seed_logits, pos: Tuple3i, movement_policy, dynimage): dynimage.UpdateFromPIL(vis) +def _as_object_array(value): + arr = np.empty((), dtype=object) + arr[()] = value + return arr + + +def _unwrap_object_array(value): + if ( + isinstance(value, np.ndarray) + and value.dtype == object + and value.ndim == 0 + ): + return value.item() + return value + + # TODO(mjanusz): Add support for sparse inference. class Canvas: """Tracks state of the inference progress and results within a subvolume.""" @@ -132,7 +148,7 @@ class Canvas: # Saving compressed NPZ files involves temporary files in the local # filesystem. In order to avoid running out of disk/RAM, we serialize # these operations. - io_lock = threading.Lock() + io_lock = threading.RLock() def __init__( self, @@ -279,6 +295,22 @@ def _deregister_client(self): self._exec_client.finish() self._exec_client_id = None + def _is_shutting_down(self) -> bool: + return ( + isinstance(self._exec_client, executor.ExecutorClient) + and self._exec_client.is_shutting_down() + ) + + def _on_sigterm_checkpoint( + self, partial_segment_iters: int, in_progress_seed: bool + ) -> None: + if self.checkpoint_path: + self.save_checkpoint( + self.checkpoint_path, + partial_segment_iters, + in_progress_seed=in_progress_seed, + ) + def __del__(self): # Note that the presence of this method will cause a memory leak in # case the Canvas object is part of a reference cycle. Use weakref.proxy @@ -508,7 +540,11 @@ def segment_at( self.counters['skip_restriced_pos'].Increment() continue - pred = self.update_at(pos) + try: + pred = self.update_at(pos) + except executor.TerminationException: + self._on_sigterm_checkpoint(num_iters, in_progress_seed=True) + raise self._min_pos = np.minimum(self._min_pos, pos) self._max_pos = np.maximum(self._max_pos, pos) num_iters += 1 @@ -557,6 +593,12 @@ def segment_all(self, seed_policy=seed.PolicyPeaks, partial_segment_iters=0): mbd = np.array([mbd.z, mbd.y, mbd.x]) for pos in TimedIter(self.seed_policy, self.counters, 'seed-policy'): + if self._is_shutting_down(): + self._on_sigterm_checkpoint( + partial_segment_iters, in_progress_seed=True + ) + raise executor.TerminationException() + # When starting a new segment the move_threshold on the probability # should be ignored when determining if the position is valid. if not ( @@ -568,7 +610,9 @@ def segment_all(self, seed_policy=seed.PolicyPeaks, partial_segment_iters=0): continue if not partial_segment_iters: - self._maybe_save_checkpoint(partial_segment_iters=0) + self._maybe_save_checkpoint( + partial_segment_iters=0, in_progress_seed=True + ) # Too close to an existing segment? low = np.array(pos) - mbd @@ -754,20 +798,24 @@ def restore_checkpoint(self, path: str) -> int: segmented_voxels = np.sum(self.segmentation != 0) self.counters['voxels-segmented'].Set(segmented_voxels) self._max_id = int(np.max(self.segmentation)) + if 'max_id' in data: + self._max_id = max(self._max_id, int(data['max_id'])) self._min_pos = data['min_pos'] self._max_pos = data['max_pos'] - self.movement_policy.restore_state(data['movement_policy']) + self.movement_policy.restore_state( + _unwrap_object_array(data['movement_policy']) + ) # When restoring the state of a previously unused Canvas, the seed # policy will not be defined. We just save the seed policy state here # for future use in .segment_all(). - self._seed_policy_state = data['seed_policy_state'] + self._seed_policy_state = _unwrap_object_array(data['seed_policy_state']) self.counters.loads(data['counters'].item()) if 'partial_segment_iters' in data: - partial_segment_iters = data['partial_segment_iters'] + partial_segment_iters = int(data['partial_segment_iters']) else: partial_segment_iters = 0 @@ -777,50 +825,69 @@ def restore_checkpoint(self, path: str) -> int: self.log_info('Inference checkpoint restored.') return partial_segment_iters - def save_checkpoint(self, path: str, partial_segment_iters: int): + def save_checkpoint( + self, + path: str | None, + partial_segment_iters: int, + in_progress_seed: bool | None = None, + ): """Saves an inference checkpoint to `path`. Args: path: path at which the checkpoint should be saved partial_segment_iters: if this checkpoint is for an in-flight segment, number of inference iterations already executed; 0 otherwise + in_progress_seed: whether the current seed from `seed_policy` has already + been consumed for an unfinished segment; defaults to + `partial_segment_iters > 0` when omitted. """ + if not path: + return + if in_progress_seed is None: + in_progress_seed = partial_segment_iters > 0 + self.log_info('Saving inference checkpoint to %s.', path) - with timer_counter(self.counters, 'save_checkpoint'): - gfile.makedirs(os.path.dirname(path)) - with storage.atomic_file(path) as fd: - seed_policy_state = None - if self.seed_policy is not None: - seed_policy_state = self.seed_policy.get_state( - partial_segment_iters > 0 - ) + with Canvas.io_lock: + with timer_counter(self.counters, 'save_checkpoint'): + gfile.makedirs(os.path.dirname(path)) + with storage.atomic_file(path) as fd: + seed_policy_state = None + if self.seed_policy is not None: + seed_policy_state = self.seed_policy.get_state( + previous=in_progress_seed + ) - aux = {} - if self.keep_probability_maps: - aux['seg_qprob'] = self.seg_prob - - np.savez_compressed( - fd, - movement_policy=np.asarray( - self.movement_policy.get_state(), dtype=object - ), - segmentation=self.segmentation, - seed=self.seed, - origins=self.origins, # pyrefly: ignore[bad-argument-type] - overlaps=self.overlaps, # pyrefly: ignore[bad-argument-type] - min_pos=self._min_pos, - max_pos=self._max_pos, - history=np.array(self.history), - history_deleted=np.array(self.history_deleted), - seed_policy_state=np.asarray(seed_policy_state, dtype=object), - counters=self.counters.dumps(), - partial_segment_iters=partial_segment_iters, - hosts=self._hosts, - **aux, - ) + aux = {} + if self.keep_probability_maps: + aux['seg_qprob'] = self.seg_prob + + np.savez_compressed( + fd, + movement_policy=_as_object_array( + self.movement_policy.get_state() + ), + segmentation=self.segmentation, + seed=self.seed, + origins=self.origins, # pyrefly: ignore[bad-argument-type] + overlaps=self.overlaps, # pyrefly: ignore[bad-argument-type] + min_pos=self._min_pos, + max_pos=self._max_pos, + max_id=int(self._max_id), + history=np.array(self.history), + history_deleted=np.array(self.history_deleted), + seed_policy_state=_as_object_array(seed_policy_state), + counters=self.counters.dumps(), + partial_segment_iters=int(partial_segment_iters), + hosts=self._hosts, + **aux, + ) self.log_info('Inference checkpoint saved.') - def _maybe_save_checkpoint(self, partial_segment_iters=0): + def _maybe_save_checkpoint( + self, + partial_segment_iters: int = 0, + in_progress_seed: bool | None = None, + ): """Attempts to save a checkpoint. A checkpoint is only saved if the canvas is configured to keep checkpoints @@ -829,6 +896,8 @@ def _maybe_save_checkpoint(self, partial_segment_iters=0): Args: partial_segment_iters: if this checkpoint is for an in-flight segment, number of inference iterations already executed; 0 otherwise + in_progress_seed: whether the current seed from `seed_policy` has already + been consumed for an unfinished segment. """ if self.checkpoint_path is None or self.checkpoint_interval_sec <= 0: return @@ -838,6 +907,8 @@ def _maybe_save_checkpoint(self, partial_segment_iters=0): with Canvas.io_lock: self.save_checkpoint( - self.checkpoint_path, partial_segment_iters=partial_segment_iters + self.checkpoint_path, + partial_segment_iters=partial_segment_iters, + in_progress_seed=in_progress_seed, ) self.checkpoint_last = time.time() diff --git a/ffn/inference/inference_utils.py b/ffn/inference/inference_utils.py index 8c7378f..9bb32c1 100644 --- a/ffn/inference/inference_utils.py +++ b/ffn/inference/inference_utils.py @@ -62,7 +62,7 @@ def IncrementBy(self, x, export=True): self._update() if self._parent is not None: - self._parent.IncrementBy(x) + self._parent.IncrementBy(x, export=export) def Set(self, x, export=True): """Sets the counter value to 'x'. diff --git a/ffn/inference/movement.py b/ffn/inference/movement.py index ba523ca..c8c999f 100644 --- a/ffn/inference/movement.py +++ b/ffn/inference/movement.py @@ -101,32 +101,42 @@ def __init__(self, canvas, deltas=(4, 8, 8), score_threshold=0.9): self.done_rounded_coords = set() self.score_threshold = score_threshold self._start_pos = None + self._in_flight = None super(FaceMaxMovementPolicy, self).__init__(canvas, deque([]), deltas) def reset_state(self, start_pos): self.scored_coords = deque([]) self.done_rounded_coords = set() self._start_pos = start_pos + self._in_flight = None def get_state(self): - return [(self.scored_coords, self.done_rounded_coords, self._start_pos)] + scored_coords = deque(self.scored_coords) + if self._in_flight is not None: + scored_coords.appendleft(self._in_flight) + return [(scored_coords, set(self.done_rounded_coords), self._start_pos)] def restore_state(self, state): - self.scored_coords, self.done_rounded_coords, self._start_pos = state[0] + scored_coords, done_rounded_coords, start_pos = state[0] + self.scored_coords = deque(scored_coords) + self.done_rounded_coords = set(done_rounded_coords) + self._start_pos = ( + tuple(int(x) for x in start_pos) if start_pos is not None else None + ) + self._in_flight = None def __next__(self): """Pops positions from queue until a valid one is found and returns it.""" + self._in_flight = None while self.scored_coords: - _, coord = self.scored_coords.popleft() + score, coord = self.scored_coords.popleft() coord = tuple(coord) if self.quantize_pos(coord) in self.done_rounded_coords: continue if self.canvas.is_valid_pos(coord): - break - else: # Else goes with while, not with if! - raise StopIteration() - - return tuple(coord) + self._in_flight = (score, coord) + return coord + raise StopIteration() def quantize_pos(self, pos): """Quantizes the positions symmetrically to a grid downsampled by deltas.""" @@ -140,6 +150,7 @@ def quantize_pos(self, pos): def update(self, prob_map, position): """Adds movements to queue for the cuboid face maxima of ``prob_map``.""" + self._in_flight = None qpos = self.quantize_pos(position) self.done_rounded_coords.add(qpos) diff --git a/ffn/inference/runner.py b/ffn/inference/runner.py index ac10d7b..68e284a 100644 --- a/ffn/inference/runner.py +++ b/ffn/inference/runner.py @@ -515,6 +515,7 @@ def run(self, corner: Tuple3i, subvol_size: Tuple3i, reset_counters=True): assert alignment is not None + partial_segment_iters = 0 if gfile.exists(cpoint_path): partial_segment_iters = canvas.restore_checkpoint(cpoint_path) @@ -528,12 +529,14 @@ def run(self, corner: Tuple3i, subvol_size: Tuple3i, reset_counters=True): # keep a record of canvases executing inference. this is used to save # checkpoints in case termination is requested. self.canvases[corner] = canvas - canvas.segment_all( - seed_policy=self.get_seed_policy(corner, subvol_size), - partial_segment_iters=partial_segment_iters, # pyrefly: ignore[unbound-name] - ) - self.save_segmentation(canvas, alignment, seg_path, prob_path) - del self.canvases[corner] + try: + canvas.segment_all( + seed_policy=self.get_seed_policy(corner, subvol_size), + partial_segment_iters=partial_segment_iters, + ) + self.save_segmentation(canvas, alignment, seg_path, prob_path) + finally: + self.canvases.pop(corner, None) # Attempt to remove the checkpoint file now that we no longer need it. try: diff --git a/ffn/inference/seed.py b/ffn/inference/seed.py index 8476a6e..8671782 100644 --- a/ffn/inference/seed.py +++ b/ffn/inference/seed.py @@ -18,7 +18,6 @@ points at which the FFN will attempt to create a segment. """ -import itertools import threading from typing import Any, Sequence import weakref @@ -110,7 +109,8 @@ def get_state(self, previous=False): return self.coords, self.idx def set_state(self, state): - self.coords, self.idx = state + self.coords, idx = state + self.idx = int(idx) # This is done as an optimization to avoid generating seeds that are # immediately going to be rejected. Keep this in sync with @@ -635,12 +635,18 @@ def __init__(self, canvas, policies: Sequence[tuple[str, dict[str, Any]]], del kwargs super().__init__(canvas) self._policies = [] + self._policy_idx = 0 for seed_policy, seed_policy_kwargs in policies: policy_cls = globals()[seed_policy] self._policies.append(policy_cls(canvas, **seed_policy_kwargs)) - def init_coords(self): - self.coords = np.array(list(itertools.chain(*self._policies))) + def __next__(self): + while self._policy_idx < len(self._policies): + try: + return next(self._policies[self._policy_idx]) + except StopIteration: + self._policy_idx += 1 + raise StopIteration() def get_state(self, previous=False): """Returns a pickleable state for this seeding policy. @@ -649,11 +655,20 @@ def get_state(self, previous=False): previous: if True, indicates that a state for the already consumed seed, and so an in-progress segment, is being requested """ - states = [] - for policy in self._policies: - states.append(policy.get_state(previous=previous)) - return states + states = [ + policy.get_state(previous=(previous and i == self._policy_idx)) + for i, policy in enumerate(self._policies) + ] + return self._policy_idx, states def set_state(self, state): - for s, policy in zip(state, self._policies): + if isinstance(state, tuple) and len(state) == 2 and isinstance( + state[0], (int, np.integer) + ): + policy_idx, states = state + self._policy_idx = int(policy_idx) + else: + self._policy_idx = 0 + states = state + for s, policy in zip(states, self._policies): policy.set_state(s)