Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 69 additions & 17 deletions ffn/jax/input_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

"""Input pipeline for FFN models."""

from collections import abc
from concurrent import futures
import functools
import threading
Expand Down Expand Up @@ -44,6 +45,8 @@


def _check_thread_success(future: futures.Future[Any], err_msg: str = ''):
if future.cancelled():
return
e = future.exception()

if e is None:
Expand Down Expand Up @@ -216,6 +219,11 @@ def __next__(self):
seeds, patches, labels, weights = super().__next__()
return {'seed': seeds, 'label': labels, 'patch': patches, 'weight': weights}

def close(self) -> None:
"""Releases any background resources held by the iterator."""
if isinstance(self._batch_generator, abc.Generator):
self._batch_generator.close()


class MixingBatchExampleIter(BatchDictExampleIter):
"""Like BatchDictExampleIter but with more examples in parallel.
Expand Down Expand Up @@ -259,6 +267,7 @@ def __init__(
self._batch_size = batch_size
self._info = model_info
self._jmp_policy = jmp_policy
self._closed = threading.Event()

# Loading of individual training examples.
self._generators = [
Expand All @@ -284,19 +293,30 @@ def _generate_batch(self):
"""Returns a batch of training examples."""
seeds, patches, labels, weights, batch_ids = [], [], [], [], []

while len(batch_ids) < self._batch_size:
while not self._closed.is_set() and len(batch_ids) < self._batch_size:
with self._fs_lock:
for f in futures.as_completed(self._fs):
self._fs.remove(f)
i, (seed, patch, label, weight) = f.result()
seeds.append(seed)
patches.append(patch)
labels.append(label)
weights.append(weight)
batch_ids.append(i)

if len(batch_ids) == self._batch_size:
break
if self._closed.is_set():
return None
try:
for f in futures.as_completed(self._fs, timeout=0.1):
self._fs.remove(f)
if self._closed.is_set() or f.cancelled():
return None
i, (seed, patch, label, weight) = f.result()
seeds.append(seed)
patches.append(patch)
labels.append(label)
weights.append(weight)
batch_ids.append(i)

if len(batch_ids) == self._batch_size:
break
except futures.TimeoutError:
pass
if len(batch_ids) < self._batch_size and not self._closed.is_set():
self._closed.wait(0.001)
if self._closed.is_set():
return None

batched_seeds = np.concatenate(seeds)
batched_weights = np.concatenate(weights)
Expand All @@ -317,13 +337,25 @@ def _generate_batch(self):
)

def __next__(self):
if self._closed.is_set():
raise StopIteration
# The time reported here indicates how long the training script had to
# wait to get a new batch of examples. It should ideally be ~0.
with utils.report_time('MixingBatchExampleIter'):
f = next(futures.as_completed(self._batch_fs))

self._batch_fs.remove(f)
self._batch_fs.add(self._batch_tpe.submit(self._generate_batch))
if f.cancelled() or self._closed.is_set():
raise StopIteration
res = f.result()
if res is None:
raise StopIteration
if not self._closed.is_set():
try:
self._batch_fs.add(self._batch_tpe.submit(self._generate_batch))
except RuntimeError:
if not self._closed.is_set():
raise

(
self._current_idx,
Expand All @@ -332,7 +364,7 @@ def __next__(self):
batched_weights,
batched_labels,
batched_patches,
) = f.result()
) = res
self._eval_tracker.track_weights(batched_weights)
return {
'seed': batched_seeds,
Expand Down Expand Up @@ -375,22 +407,42 @@ def _update(
] = batched_seeds[i, ...] # pyrefly: ignore[bad-index]

with self._fs_lock:
if self._closed.is_set():
return
for gen_idx in current:
gen = self._generators[gen_idx]
self._fs.add(
self._tpe.submit(lambda gen=gen, i=gen_idx: (i, next(gen))) # pyrefly: ignore[bad-argument-type]
)

if self._closed.is_set():
return
# Distribute data asynchronously.
update_future = self._seed_update_tpe.submit(
_update, self._seeds, batched_seeds, self._current_idx
)
try:
update_future = self._seed_update_tpe.submit(
_update, self._seeds, batched_seeds, self._current_idx
)
except RuntimeError:
if self._closed.is_set():
return
raise
update_future.add_done_callback(
functools.partial(
_check_thread_success, err_msg='Error while updating seeds.'
)
)

def close(self) -> None:
"""Shuts down background prefetch and seed-update thread pools."""
if self._closed.is_set():
return
self._closed.set()
self._seed_update_tpe.shutdown(wait=False, cancel_futures=True)
with self._fs_lock:
self._tpe.shutdown(wait=False, cancel_futures=True)
self._fs.clear()
self._batch_tpe.shutdown(wait=False, cancel_futures=True)


class UnbatchIter:
"""Fetches batches from a tf.data iterator and returns elements one by one.
Expand Down
6 changes: 6 additions & 0 deletions ffn/jax/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ def main(argv: Sequence[str]) -> None:
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')

# Disable space-to-batch XLA transform which can crash on VLP small 3D FoVs.
if 'xla_tpu_run_space_to_batch' in flags.FLAGS:
flags.FLAGS.xla_tpu_run_space_to_batch = False
if 'xla_tpu_run_space_to_batch_on_new_platforms' in flags.FLAGS:
flags.FLAGS.xla_tpu_run_space_to_batch_on_new_platforms = False

training.prep_training()
train.train_and_evaluate(FLAGS.config, FLAGS.workdir, FLAGS.service_address)

Expand Down
83 changes: 61 additions & 22 deletions ffn/jax/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"""Training script for FFN models."""

import collections
import contextlib
import functools as ft
import os
import random
Expand Down Expand Up @@ -247,7 +248,7 @@ def loss_fn(params):
ema_params=new_ema_params,
)

lr = schedule(state.opt_state.count) # pyrefly: ignore[missing-attribute]
lr = schedule(state.step)
metrics_update = TrainMetrics.single_from_model_output(
loss=loss, learning_rate=lr
)
Expand Down Expand Up @@ -375,48 +376,56 @@ def _make_ckpt_args(
return ocp.args.Composite(**args)


def _restore_initial_state(
init_from_cpoint: str,
def _restore_tree_with_shapes(
template: Any, raw_tree: Any, ckpt_path: epath.PathLike
) -> Any:
"""Restores `raw_tree` into `template` structure and validates leaf shapes."""
restored = flax.serialization.from_state_dict(template, raw_tree)
for new_leaf, tmpl_leaf in zip(
jax.tree.leaves(restored), jax.tree.leaves(template), strict=True
):
if np.shape(new_leaf) != np.shape(tmpl_leaf):
raise ValueError(
f'Restored parameter shape {np.shape(new_leaf)} does not match '
f'model shape {np.shape(tmpl_leaf)} for {ckpt_path!r}.'
)
return restored


def restore_train_state_for_finetune(
init_from_cpoint: epath.PathLike,
state: TrainState,
optimizer: optax.GradientTransformation,
config: ml_collections.ConfigDict,
reset_train_state: bool,
) -> TrainState:
"""Restores `TrainState` (or weights when resetting state) from a checkpoint."""
"""Restores model weights while resetting step and optimizer state."""
init_path = epath.Path(init_from_cpoint)
train_state_path = init_path / 'train_state'
target_path = train_state_path if train_state_path.exists() else init_path
handler = ocp.StandardCheckpointHandler()
if not reset_train_state:
return handler.restore(target_path, args=ocp.args.StandardRestore(state))
try:
raw_state = handler.restore(target_path)
finally:
handler.close()

raw_state = handler.restore(target_path)
raw_params = (
raw_state['params']
if 'params' in raw_state
else raw_state['optimizer']['target']
)
params = flax.serialization.from_state_dict(state.params, raw_params)
for new_leaf, tmpl_leaf in zip(
jax.tree.leaves(params), jax.tree.leaves(state.params)
):
if np.shape(new_leaf) != np.shape(tmpl_leaf):
raise ValueError(
f'Restored parameter shape {np.shape(new_leaf)} does not match '
f'model shape {np.shape(tmpl_leaf)} for {init_from_cpoint!r}.'
)
params = _restore_tree_with_shapes(state.params, raw_params, init_from_cpoint)

batch_stats = state.batch_stats
if batch_stats is not None and raw_state.get('batch_stats') is not None:
batch_stats = flax.serialization.from_state_dict(
batch_stats, raw_state['batch_stats']
batch_stats = _restore_tree_with_shapes(
batch_stats, raw_state['batch_stats'], init_from_cpoint
)

ema_params = None
if config.get('ema_decay', 0.0) > 0.0:
raw_ema = raw_state.get('ema_params')
ema_params = (
flax.serialization.from_state_dict(state.params, raw_ema)
_restore_tree_with_shapes(state.params, raw_ema, init_from_cpoint)
if raw_ema is not None
else params
)
Expand All @@ -430,6 +439,28 @@ def _restore_initial_state(
)


def _restore_initial_state(
init_from_cpoint: epath.PathLike,
state: TrainState,
optimizer: optax.GradientTransformation,
config: ml_collections.ConfigDict,
reset_train_state: bool,
) -> TrainState:
"""Restores `TrainState` (or weights when resetting state) from a checkpoint."""
if reset_train_state:
return restore_train_state_for_finetune(
init_from_cpoint, state, optimizer, config
)
init_path = epath.Path(init_from_cpoint)
train_state_path = init_path / 'train_state'
target_path = train_state_path if train_state_path.exists() else init_path
handler = ocp.StandardCheckpointHandler()
try:
return handler.restore(target_path, args=ocp.args.StandardRestore(state))
finally:
handler.close()


def train_and_evaluate(
config: ml_collections.ConfigDict,
workdir: str,
Expand Down Expand Up @@ -614,6 +645,8 @@ def train_and_evaluate(
num_total_examples // topo_info.global_batch_size * (len(fov_shifts) + 1)
)
num_train_steps = steps_per_epoch * config.num_epochs
if config.get('num_train_steps', -1) > 0:
num_train_steps = config.num_train_steps
logging.info(
'num_train_steps=%d, steps_per_epoch=%d', num_train_steps, steps_per_epoch
)
Expand Down Expand Up @@ -726,7 +759,12 @@ def _reshape(x):
},
)

with metric_writers.ensure_flushes(writer):
step = initial_step - 1
with (
contextlib.closing(checkpoint_manager),
contextlib.closing(batch_iter),
metric_writers.ensure_flushes(writer),
):
# Record a summary scalar to indicate the specific steps at which
# training restarts occurred.
writer.write_scalars(initial_step, {'start': 1})
Expand Down Expand Up @@ -818,8 +856,9 @@ def _reshape(x):
train_metrics = None
eval_tracker.reset()

checkpoint_manager.wait_until_finished()
checkpoint_manager.wait_until_finished()
logging.info('Finished training at step %d.', step)
logging.flush()

if shutdown_request:
# Allow time for other workers to finish checkpoint saving. Soon after
Expand Down
16 changes: 16 additions & 0 deletions ffn/jax/train_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Copyright 2026 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Tests for FFN JAX fine-tuning state restoration and iterator cleanup."""
Loading