diff --git a/ffn/jax/input_pipeline.py b/ffn/jax/input_pipeline.py index e68bea3..c8d2832 100644 --- a/ffn/jax/input_pipeline.py +++ b/ffn/jax/input_pipeline.py @@ -15,6 +15,7 @@ """Input pipeline for FFN models.""" +from collections import abc from concurrent import futures import functools import threading @@ -44,6 +45,8 @@ def _check_thread_success(future: futures.Future[Any], err_msg: str = ''): + if future.cancelled(): + return e = future.exception() if e is None: @@ -216,6 +219,11 @@ def __next__(self): seeds, patches, labels, weights = super().__next__() return {'seed': seeds, 'label': labels, 'patch': patches, 'weight': weights} + def close(self) -> None: + """Releases any background resources held by the iterator.""" + if isinstance(self._batch_generator, abc.Generator): + self._batch_generator.close() + class MixingBatchExampleIter(BatchDictExampleIter): """Like BatchDictExampleIter but with more examples in parallel. @@ -259,6 +267,7 @@ def __init__( self._batch_size = batch_size self._info = model_info self._jmp_policy = jmp_policy + self._closed = threading.Event() # Loading of individual training examples. self._generators = [ @@ -284,19 +293,30 @@ def _generate_batch(self): """Returns a batch of training examples.""" seeds, patches, labels, weights, batch_ids = [], [], [], [], [] - while len(batch_ids) < self._batch_size: + while not self._closed.is_set() and len(batch_ids) < self._batch_size: with self._fs_lock: - for f in futures.as_completed(self._fs): - self._fs.remove(f) - i, (seed, patch, label, weight) = f.result() - seeds.append(seed) - patches.append(patch) - labels.append(label) - weights.append(weight) - batch_ids.append(i) - - if len(batch_ids) == self._batch_size: - break + if self._closed.is_set(): + return None + try: + for f in futures.as_completed(self._fs, timeout=0.1): + self._fs.remove(f) + if self._closed.is_set() or f.cancelled(): + return None + i, (seed, patch, label, weight) = f.result() + seeds.append(seed) + patches.append(patch) + labels.append(label) + weights.append(weight) + batch_ids.append(i) + + if len(batch_ids) == self._batch_size: + break + except futures.TimeoutError: + pass + if len(batch_ids) < self._batch_size and not self._closed.is_set(): + self._closed.wait(0.001) + if self._closed.is_set(): + return None batched_seeds = np.concatenate(seeds) batched_weights = np.concatenate(weights) @@ -317,13 +337,25 @@ def _generate_batch(self): ) def __next__(self): + if self._closed.is_set(): + raise StopIteration # The time reported here indicates how long the training script had to # wait to get a new batch of examples. It should ideally be ~0. with utils.report_time('MixingBatchExampleIter'): f = next(futures.as_completed(self._batch_fs)) self._batch_fs.remove(f) - self._batch_fs.add(self._batch_tpe.submit(self._generate_batch)) + if f.cancelled() or self._closed.is_set(): + raise StopIteration + res = f.result() + if res is None: + raise StopIteration + if not self._closed.is_set(): + try: + self._batch_fs.add(self._batch_tpe.submit(self._generate_batch)) + except RuntimeError: + if not self._closed.is_set(): + raise ( self._current_idx, @@ -332,7 +364,7 @@ def __next__(self): batched_weights, batched_labels, batched_patches, - ) = f.result() + ) = res self._eval_tracker.track_weights(batched_weights) return { 'seed': batched_seeds, @@ -375,22 +407,42 @@ def _update( ] = batched_seeds[i, ...] # pyrefly: ignore[bad-index] with self._fs_lock: + if self._closed.is_set(): + return for gen_idx in current: gen = self._generators[gen_idx] self._fs.add( self._tpe.submit(lambda gen=gen, i=gen_idx: (i, next(gen))) # pyrefly: ignore[bad-argument-type] ) + if self._closed.is_set(): + return # Distribute data asynchronously. - update_future = self._seed_update_tpe.submit( - _update, self._seeds, batched_seeds, self._current_idx - ) + try: + update_future = self._seed_update_tpe.submit( + _update, self._seeds, batched_seeds, self._current_idx + ) + except RuntimeError: + if self._closed.is_set(): + return + raise update_future.add_done_callback( functools.partial( _check_thread_success, err_msg='Error while updating seeds.' ) ) + def close(self) -> None: + """Shuts down background prefetch and seed-update thread pools.""" + if self._closed.is_set(): + return + self._closed.set() + self._seed_update_tpe.shutdown(wait=False, cancel_futures=True) + with self._fs_lock: + self._tpe.shutdown(wait=False, cancel_futures=True) + self._fs.clear() + self._batch_tpe.shutdown(wait=False, cancel_futures=True) + class UnbatchIter: """Fetches batches from a tf.data iterator and returns elements one by one. diff --git a/ffn/jax/main.py b/ffn/jax/main.py index 6e3a798..75d9810 100644 --- a/ffn/jax/main.py +++ b/ffn/jax/main.py @@ -33,6 +33,12 @@ def main(argv: Sequence[str]) -> None: if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') + # Disable space-to-batch XLA transform which can crash on VLP small 3D FoVs. + if 'xla_tpu_run_space_to_batch' in flags.FLAGS: + flags.FLAGS.xla_tpu_run_space_to_batch = False + if 'xla_tpu_run_space_to_batch_on_new_platforms' in flags.FLAGS: + flags.FLAGS.xla_tpu_run_space_to_batch_on_new_platforms = False + training.prep_training() train.train_and_evaluate(FLAGS.config, FLAGS.workdir, FLAGS.service_address) diff --git a/ffn/jax/train.py b/ffn/jax/train.py index 8eef5ea..997a59d 100644 --- a/ffn/jax/train.py +++ b/ffn/jax/train.py @@ -15,6 +15,7 @@ """Training script for FFN models.""" import collections +import contextlib import functools as ft import os import random @@ -247,7 +248,7 @@ def loss_fn(params): ema_params=new_ema_params, ) - lr = schedule(state.opt_state.count) # pyrefly: ignore[missing-attribute] + lr = schedule(state.step) metrics_update = TrainMetrics.single_from_model_output( loss=loss, learning_rate=lr ) @@ -375,48 +376,56 @@ def _make_ckpt_args( return ocp.args.Composite(**args) -def _restore_initial_state( - init_from_cpoint: str, +def _restore_tree_with_shapes( + template: Any, raw_tree: Any, ckpt_path: epath.PathLike +) -> Any: + """Restores `raw_tree` into `template` structure and validates leaf shapes.""" + restored = flax.serialization.from_state_dict(template, raw_tree) + for new_leaf, tmpl_leaf in zip( + jax.tree.leaves(restored), jax.tree.leaves(template), strict=True + ): + if np.shape(new_leaf) != np.shape(tmpl_leaf): + raise ValueError( + f'Restored parameter shape {np.shape(new_leaf)} does not match ' + f'model shape {np.shape(tmpl_leaf)} for {ckpt_path!r}.' + ) + return restored + + +def restore_train_state_for_finetune( + init_from_cpoint: epath.PathLike, state: TrainState, optimizer: optax.GradientTransformation, config: ml_collections.ConfigDict, - reset_train_state: bool, ) -> TrainState: - """Restores `TrainState` (or weights when resetting state) from a checkpoint.""" + """Restores model weights while resetting step and optimizer state.""" init_path = epath.Path(init_from_cpoint) train_state_path = init_path / 'train_state' target_path = train_state_path if train_state_path.exists() else init_path handler = ocp.StandardCheckpointHandler() - if not reset_train_state: - return handler.restore(target_path, args=ocp.args.StandardRestore(state)) + try: + raw_state = handler.restore(target_path) + finally: + handler.close() - raw_state = handler.restore(target_path) raw_params = ( raw_state['params'] if 'params' in raw_state else raw_state['optimizer']['target'] ) - params = flax.serialization.from_state_dict(state.params, raw_params) - for new_leaf, tmpl_leaf in zip( - jax.tree.leaves(params), jax.tree.leaves(state.params) - ): - if np.shape(new_leaf) != np.shape(tmpl_leaf): - raise ValueError( - f'Restored parameter shape {np.shape(new_leaf)} does not match ' - f'model shape {np.shape(tmpl_leaf)} for {init_from_cpoint!r}.' - ) + params = _restore_tree_with_shapes(state.params, raw_params, init_from_cpoint) batch_stats = state.batch_stats if batch_stats is not None and raw_state.get('batch_stats') is not None: - batch_stats = flax.serialization.from_state_dict( - batch_stats, raw_state['batch_stats'] + batch_stats = _restore_tree_with_shapes( + batch_stats, raw_state['batch_stats'], init_from_cpoint ) ema_params = None if config.get('ema_decay', 0.0) > 0.0: raw_ema = raw_state.get('ema_params') ema_params = ( - flax.serialization.from_state_dict(state.params, raw_ema) + _restore_tree_with_shapes(state.params, raw_ema, init_from_cpoint) if raw_ema is not None else params ) @@ -430,6 +439,28 @@ def _restore_initial_state( ) +def _restore_initial_state( + init_from_cpoint: epath.PathLike, + state: TrainState, + optimizer: optax.GradientTransformation, + config: ml_collections.ConfigDict, + reset_train_state: bool, +) -> TrainState: + """Restores `TrainState` (or weights when resetting state) from a checkpoint.""" + if reset_train_state: + return restore_train_state_for_finetune( + init_from_cpoint, state, optimizer, config + ) + init_path = epath.Path(init_from_cpoint) + train_state_path = init_path / 'train_state' + target_path = train_state_path if train_state_path.exists() else init_path + handler = ocp.StandardCheckpointHandler() + try: + return handler.restore(target_path, args=ocp.args.StandardRestore(state)) + finally: + handler.close() + + def train_and_evaluate( config: ml_collections.ConfigDict, workdir: str, @@ -614,6 +645,8 @@ def train_and_evaluate( num_total_examples // topo_info.global_batch_size * (len(fov_shifts) + 1) ) num_train_steps = steps_per_epoch * config.num_epochs + if config.get('num_train_steps', -1) > 0: + num_train_steps = config.num_train_steps logging.info( 'num_train_steps=%d, steps_per_epoch=%d', num_train_steps, steps_per_epoch ) @@ -726,7 +759,12 @@ def _reshape(x): }, ) - with metric_writers.ensure_flushes(writer): + step = initial_step - 1 + with ( + contextlib.closing(checkpoint_manager), + contextlib.closing(batch_iter), + metric_writers.ensure_flushes(writer), + ): # Record a summary scalar to indicate the specific steps at which # training restarts occurred. writer.write_scalars(initial_step, {'start': 1}) @@ -818,8 +856,9 @@ def _reshape(x): train_metrics = None eval_tracker.reset() - checkpoint_manager.wait_until_finished() + checkpoint_manager.wait_until_finished() logging.info('Finished training at step %d.', step) + logging.flush() if shutdown_request: # Allow time for other workers to finish checkpoint saving. Soon after diff --git a/ffn/jax/train_test.py b/ffn/jax/train_test.py new file mode 100644 index 0000000..45d5842 --- /dev/null +++ b/ffn/jax/train_test.py @@ -0,0 +1,16 @@ +# Copyright 2026 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for FFN JAX fine-tuning state restoration and iterator cleanup."""