Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,8 @@ with h5py.File(path) as f:
- Names of `NXdetector` groups are the names of the groups in the NeXus files of the runs.
A name that is not a bank of the instrument is an error, since a misspelled name would otherwise mask nothing.
- The name of a dataset in `masks` is the name of the mask in the reduced data.
Datasets with the same name in several banks form one mask.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copilot decided this should go. I also fail to see how they could form just one mask, since detector banks are loaded separately in the nexus workflow. It would imply that the mask has a name and then lists the detector banks inside it, which leads to the previous data structure.

I think we are still checking that masks with the same name in different detector banks should have the same dims.

The data of each bank gets a mask of that name, made from the dataset of that bank.
A bank without a dataset of that name has no pixels masked by it.
Masks are associated with their detector group; a detector without a dataset does
not get that mask.
- A pixel is masked if its value is not zero.
HDF5 has no boolean type.
h5py stores numpy booleans as an enumeration based on `int8`, which other HDF5 libraries may not read as booleans.
Expand All @@ -65,7 +64,7 @@ with h5py.File(path) as f:
`write_pixel_masks` writes all three, and workflows do not read them.

Workflows take a list of filenames and use the masks of all files.
Two files with a mask of the same name are an error.
Two files with a mask of the same name for the same detector are an error.
A file can hold all masks of an instrument, or masks can be split into files, for example by validity period: one file for broken tubes and one for the beamstop of an experiment.
Workflows do not check validity dates.
Selecting the files that are valid for a run is left to the caller of the workflow, which sets the list of filenames.
Expand Down
55 changes: 36 additions & 19 deletions packages/essreduce/src/ess/reduce/masking.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@
"""Files with pixel masks to apply. Use an empty sequence to apply none."""

PixelMasks = NewType('PixelMasks', dict[str, sc.Variable | sc.DataGroup])
"""Pixel masks of the whole instrument, by mask name.
"""Pixel masks of the whole instrument.

Each value is either a variable of masked detector IDs, or a data group of boolean masks
by detector. :py:func:`read_pixel_masks` provides them from files, and they can also be
set directly. :py:func:`to_detector_masks` selects the masks for one detector.
Each value is either a variable of masked detector IDs, named by the mask, or a data
group of boolean masks, named by detector and then mask. :py:func:`read_pixel_masks`
provides them from files, and they can also be set directly.
:py:func:`to_detector_masks` selects the masks for one detector.
"""

_HDF5_SUFFIXES = ('.h5', '.hdf', '.hdf5', '.nxs')
Expand Down Expand Up @@ -55,10 +56,8 @@ def read_pixel_masks(filenames: PixelMaskFilenames) -> PixelMasks:
beamstop integer array

Detector names are the names of the NeXus detector groups, and mask names are
the names of the datasets. Datasets with the same name in several detectors form
one mask. The data of each detector gets a mask of that name, made from the
dataset of that detector. A detector without a dataset of that name has no
pixels masked by it. A pixel is masked if its value is not zero. The dimensions
the names of the datasets. The data of each detector gets the masks in its
``masks`` collection. A pixel is masked if its value is not zero. The dimensions
of a dataset are given by HDF5 dimension labels. They are named after the
logical dimensions of the detector, and may be a subset of them, in any order.
Other entries of the file, such as the instrument name or a validity date, are
Expand All @@ -75,15 +74,27 @@ def read_pixel_masks(filenames: PixelMaskFilenames) -> PixelMasks:
Raises
------
ValueError
If a file has an unsupported suffix or content, or two files give the same mask
name. One mask would replace the other otherwise.
If a file has an unsupported suffix or content, or two files define the same
mask for a detector, or same-named masks in different detectors have different
dimension labels. One mask would replace the other otherwise.
"""
masks = {}
for filename in map(Path, filenames):
for name, content in _read_mask_file(filename).items():
if name in masks:
if name not in masks:
masks[name] = content
elif isinstance(content, sc.DataGroup) and isinstance(
previous := masks[name], sc.DataGroup
):
duplicates = set(previous) & set(content)
if duplicates:
raise ValueError(
f"Several mask files give detector masks {sorted(duplicates)} "
f"for detector '{name}'."
)
masks[name] = sc.DataGroup({**dict(previous), **dict(content)})
else:
raise ValueError(f"Several mask files give the mask name '{name}'.")
masks[name] = content
return PixelMasks(masks)


Expand Down Expand Up @@ -126,12 +137,15 @@ def to_detector_masks(
if isinstance(content, sc.Variable):
masks[name] = _mask_from_ids(content, detector.coords[id_coord], name)
continue
if known_banks is not None and name not in known_banks:
raise ValueError(f"Masks have unknown banks: {[name]}.")
if bank is None:
raise ValueError(f"Mask '{name}' is given per bank, but no bank is known.")
if known_banks is not None and (unknown := set(content) - set(known_banks)):
raise ValueError(f"Mask '{name}' has unknown banks: {sorted(unknown)}.")
if bank in content:
masks[name] = _align_bank_mask(content[bank], detector.sizes, name)
raise ValueError("Masks are given per bank, but no bank is known.")
if name == bank:
for mask_name, mask in content.items():
if mask_name in masks:
raise ValueError(f"Several masks have the name '{mask_name}'.")
masks[mask_name] = _align_bank_mask(mask, detector.sizes, mask_name)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This entire function looks fishy. name is a bank name, so why does the first branch blindly assign that to masks? And why does the last branch check the name? And why does this loop at all instead of calling pixel_masks.get(bank)? The bank is None case doesn't seem to be supported by PixelMasks.

@nvaytet nvaytet Nov 16, 2026 •

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As I understand it, this function takes in a list of masks for all detectors, and tries to extract the masks for the requested detector.

First branch blindly assigns to masks

The isinstance(content, sc.Variable): case is for the ISIS xml masks case, where the loaded masks look like:

{'mask_new_July2022.xml': <scipp.Variable> (detector_id: 249022)      int32        <no unit>  [1, 2, ..., 458751, 458752]}

I believe this is also why we can't just pixel_masks.get(bank). In the case above, if we try to load some xml masks for Loki, it would fail because bank which is loki_detector_0 would not be in the dict of masks.
Instead, we have to lookup the detector ids and determine if they are in the requested bank or not.

So the asumption is that if the value is a Variable, it must be a mask from an xml file (which only contains det ids, has no concept of bank names).
I guess this is quite a bold assumption, but maybe ok? I fail to see how you could save masks such as

{'bad_tubes': sc.zeros(sizes={'layer':4, 'tube':32}),
 'bad_straws': sc.zeros(sizes={'layer':4, 'tube':32, 'straw':7}),}

that would be for multiple detector banks, as each bank usually has its own number of layers, tubes, straws. And you don't want to have a mechanism to identify the right shape, as it can be ambiguous.

You can of course save a mask in a scipp hdf file, that would hold detector ids, like the xml mask, but then we would just treat it as the xml case above.

The bank is None case

I don't understand fully this case. For isis xml masks, this part of the code is not used.
For sans (Loki), the bank name is always specified.
Powder diffraction is the only use case I found where bank is None, but couldn't really wrap my head around how the pixel_masks were defined (they are just empty dicts in the docs examples).

if known_banks is not None and name not in known_banks:

I guess that's to protect against bank names that are misspelled, so that we raise an error instead of sliently dropping the masks?

@jl-wynen jl-wynen Nov 17, 2026 •

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That seems too complicated. I would normalise when loading instead of having a complex extraction logic because we might look at the masks dict manually or using other tools as well. So I'd rather have a consistent in-memory structure.

@nvaytet nvaytet Nov 17, 2026 •

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also thought that would be better, but I think it would mean you need to know the detector name to be able to load the masks, meaning that you can't just load one nexus file with masks all in one go, you just load the part for the detector that you're interested in.
Which in turn makes it more difficult to just load the masks and inspect them?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this only a concern for XML mask files where we don't have a bank name in the file? I see two options for adjusting the representation:

  1. Use a catch-all bank name like "__all__". But that would mean checking for it in the extraction function.
  2. Use a dedicated type that allows us to check whether we have this case or the ESS case of one entry per bank. E.g., two separate classes or a class with an attribute (returning an enum) to distinguish the cases.

I just don't like having to inspect the contents of a dict and guessing that if you see a variable, the dict key is a mask name, otherwise it is a bank name.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with Jan-Lukas: the problem is that the top-level key of PixelMasks is a mask name for detector-ID masks but a bank name for per-bank masks. The file layout (<bank>/masks/<mask_name>) is fine, and I'd keep the bank-first writer input from #874, since it matches the file. What I'd suggest is that the two kinds of masks get separate fields instead of sharing one dict:

@dataclass(frozen=True)
class PixelMasks:
    by_bank: dict[str, dict[str, sc.Variable]] = field(default_factory=dict)
    """Boolean masks by bank name, then mask name."""
    by_detector_id: dict[str, sc.Variable] = field(default_factory=dict)
    """Masked detector IDs by mask name (ISIS XML, scipp HDF5). Apply to any bank."""

to_detector_masks then needs no type checks and no loop over banks:

if known_banks is not None and (unknown := pixel_masks.by_bank.keys() - set(known_banks)):
    raise ValueError(...)
if pixel_masks.by_bank and bank is None:
    raise ValueError(...)
id_masks = {name: _mask_from_ids(...) for name, ids in pixel_masks.by_detector_id.items()}
bank_masks = {name: _align_bank_mask(...) for name, m in pixel_masks.by_bank.get(bank, {}).items()}
# raise if id_masks and bank_masks share a name
return {**id_masks, **bank_masks}

write_pixel_masks takes a by_bank dict, so read_pixel_masks([path]).by_bank returns exactly what was written. Merging several files raises only if two files give the same mask name for the same bank.

On your questions:

  • I'd avoid a separate provider for XML masks. It would make reading the mask files depend on the run, while now we read them once and select the bank afterwards. The separate by_detector_id field already keeps the two kinds apart.
  • Detector-ID masks are not only XML: scipp HDF5 files with ID lists are supported too, and the powder workflow currently uses only ID masks.
  • With either layout, reading does not need the bank name. read_pixel_masks loads everything, and to_detector_masks picks the bank.
  • bank is None is the powder workflow, which does not pass NeXusDetectorName and therefore supports only detector-ID masks. The mask providers should move from esssans and essdiffraction into the generic workflow so every instrument gets per-bank masks. I opened [essreduce] Apply pixel mask files in the generic NeXus workflow #893 for this.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the powder workflow currently uses only ID masks

Is that the workflow for ESS instruments or SNS powder? Do we expect to continue using detector id masks? I guess we do if we want other facilities to also use Scipp?
I just want to make sure we are not adding complication because of other facilities, while ESS instruments could all use logical masks.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We already have the code to load/apply. So the question is whether making PixelMasks a dataclass is a complication, or kind of nice anyway, since it is more self-documenting.

Right now I would say we are not going to use ID-based masks, but with low confidence. Is it worth the effort stripping the support out now, or just keep things as they are (with the proposed PixelMasks)?

@SimonHeybrock SimonHeybrock Nov 20, 2026 •

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I'd want to tweak PixelMasks a bit more, I don't really like the by_bank field as it is. Should the format support future additions such as allowing to specify angle ranges (without adding those fields now, but at least ensuring it can be done)?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I always thought masking should also support callables that can apply a masking rule, but I don't know of a good way to store a callable in a nexus file!

Is it worth the effort stripping the support out now

I don't think we should close a door now if we are not sure about it.

Is the broad conclusion here that we keep the support for reading xml detector ids, scipp-h5 detector ids, and masks with logical dims, and continue to do the extraction work in to_detector_masks?

I have not figured out what the reader would look like with the proposer PixelMasks dataclass.

But it would be good to have a decision as this is blocking #834

return masks


Expand Down Expand Up @@ -248,11 +262,14 @@ def _read_nexus_masks(entry: snx.Group, filename: Path) -> dict[str, sc.DataGrou
for detector_name, detector in entry['instrument'][snx.NXdetector].items():
if 'masks' not in detector:
continue
detector_masks = {}
for name, field in detector['masks'].items():
masks.setdefault(name, {})[detector_name] = _read_bank_mask(field, filename)
detector_masks[name] = _read_bank_mask(field, filename)
if detector_masks:
masks[detector_name] = sc.DataGroup(detector_masks)
if not masks:
raise ValueError(f"Mask file '{filename}' has no detector with masks.")
return {name: sc.DataGroup(banks) for name, banks in masks.items()}
return masks


def _read_bank_mask(field: snx.Field | snx.Group, filename: Path) -> sc.Variable:
Expand Down
60 changes: 44 additions & 16 deletions packages/essreduce/tests/masking_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ def test_read_raises_if_hdf5_does_not_hold_integers(tmp_path: Path) -> None:
_read(path)


def test_read_bank_masks_names_masks_after_datasets(tmp_path: Path) -> None:
def test_read_bank_masks_by_detector_and_mask_name(tmp_path: Path) -> None:
broken = sc.array(dims=['tube'], values=[True, False])
beamstop = sc.array(dims=['straw'], values=[False, True, True])
write_pixel_masks(
Expand All @@ -102,8 +102,7 @@ def test_read_bank_masks_names_masks_after_datasets(tmp_path: Path) -> None:
instrument='TEST',
)
masks = _read(tmp_path / 'a.nxs')
assert sc.identical(masks['broken'], sc.DataGroup(bank_0=broken))
assert sc.identical(masks['beamstop'], sc.DataGroup(bank_0=beamstop))
assert sc.identical(masks['bank_0'], sc.DataGroup(broken=broken, beamstop=beamstop))


def test_write_uses_detector_name_and_mask_name(tmp_path: Path) -> None:
Expand All @@ -115,20 +114,24 @@ def test_write_uses_detector_name_and_mask_name(tmp_path: Path) -> None:
assert file['entry/instrument/loki_detector_0/masks/bad_tubes'][
()
].tolist() == [1]
assert sc.identical(_read(path)['bad_tubes'], sc.DataGroup(loki_detector_0=mask))
assert sc.identical(_read(path)['loki_detector_0'], sc.DataGroup(bad_tubes=mask))


def test_read_bank_masks_keeps_dimensions_of_all_banks(tmp_path: Path) -> None:
def test_read_same_named_masks_can_have_different_shapes_and_order(
tmp_path: Path,
) -> None:
banks = {
'bank_0': sc.array(dims=['layer', 'tube'], values=[[True, False]]),
'bank_1': sc.array(dims=['straw'], values=[False, True, False]),
'bank_1': sc.array(dims=['tube', 'layer'], values=[[True], [False]]),
}
write_pixel_masks(
tmp_path / 'a.nxs',
{bank: {'m': mask} for bank, mask in banks.items()},
instrument='TEST',
)
assert sc.identical(_read(tmp_path / 'a.nxs')['m'], sc.DataGroup(banks))
loaded = _read(tmp_path / 'a.nxs')
assert sc.identical(loaded['bank_0'], sc.DataGroup(m=banks['bank_0']))
assert sc.identical(loaded['bank_1'], sc.DataGroup(m=banks['bank_1']))


def test_read_bank_masks_omits_banks_without_dataset_of_mask(tmp_path: Path) -> None:
Expand All @@ -139,8 +142,8 @@ def test_read_bank_masks_omits_banks_without_dataset_of_mask(tmp_path: Path) ->
instrument='TEST',
)
masks = _read(tmp_path / 'a.nxs')
assert list(masks['broken']) == ['bank_0']
assert list(masks['beamstop']) == ['bank_1']
assert list(masks['bank_0']) == ['broken']
assert list(masks['bank_1']) == ['beamstop']


def test_read_ignores_other_entries_of_the_file(tmp_path: Path) -> None:
Expand All @@ -153,7 +156,7 @@ def test_read_ignores_other_entries_of_the_file(tmp_path: Path) -> None:
with snx.File(path, 'r+') as file:
file['entry'].create_field('valid_from', '2026-11-01T00:00:00Z')
file['entry/instrument/bank_0'].create_field('pixel_mask', np.array([1]))
assert list(_read(path)) == ['m']
assert list(_read(path)) == ['bank_0']


def test_read_skips_detector_without_masks(tmp_path: Path) -> None:
Expand All @@ -165,7 +168,7 @@ def test_read_skips_detector_without_masks(tmp_path: Path) -> None:
)
with snx.File(path, 'r+') as file:
file['entry/instrument'].create_class('bank_1', snx.NXdetector)
assert list(_read(path)['m']) == ['bank_0']
assert list(_read(path)['bank_0']) == ['m']


def test_read_combines_masks_of_several_files(tmp_path: Path) -> None:
Expand All @@ -177,21 +180,46 @@ def test_read_combines_masks_of_several_files(tmp_path: Path) -> None:
tmp_path / 'b.nxs', {'bank_0': {'beamstop': mask}}, instrument='TEST'
)
masks = _read(tmp_path / 'a.nxs', tmp_path / 'b.nxs')
assert list(masks) == ['broken', 'beamstop']
assert list(masks['bank_0']) == ['broken', 'beamstop']


def test_read_raises_if_files_have_mask_with_same_name(tmp_path: Path) -> None:
def test_read_raises_if_files_have_mask_for_same_detector_and_name(
tmp_path: Path,
) -> None:
mask = sc.array(dims=['tube'], values=[True])
write_pixel_masks(
tmp_path / 'a.nxs', {'bank_0': {'broken': mask}}, instrument='TEST'
)
write_pixel_masks(
tmp_path / 'b.nxs', {'bank_1': {'broken': mask}}, instrument='TEST'
tmp_path / 'b.nxs', {'bank_0': {'broken': mask}}, instrument='TEST'
)
with pytest.raises(ValueError, match='broken'):
_read(tmp_path / 'a.nxs', tmp_path / 'b.nxs')


def test_read_round_trips_masks_by_detector(tmp_path: Path) -> None:
masks = {
f'loki_detector_{i}': {
'bad_tubes': sc.array(
dims=['layer', 'tube'],
values=np.arange(4 * 32).reshape(4, 32) % 9 == i,
),
'bad_straws': sc.array(
dims=['layer', 'tube', 'straw'],
values=np.arange(4 * 32 * 7).reshape(4, 32, 7) % 9 == i,
),
}
for i in range(9)
}
path = tmp_path / 'masks.hdf'
write_pixel_masks(path, masks, instrument='LOKI')

loaded = _read(path)
assert list(loaded) == list(masks)
for detector, detector_masks in masks.items():
assert sc.identical(loaded[detector], sc.DataGroup(detector_masks))


def test_write_stores_masks_as_uint8(tmp_path: Path) -> None:
path = tmp_path / 'a.nxs'
write_pixel_masks(
Expand Down Expand Up @@ -220,7 +248,7 @@ def test_read_masks_pixels_with_nonzero_value(tmp_path: Path, dtype: str) -> Non
values = np.array([0, 1, 1], dtype=dtype)
if dtype != 'bool':
values[2] = 2
loaded = _read(_write_raw_mask(tmp_path / 'a.nxs', values))['m']['bank_0']
loaded = _read(_write_raw_mask(tmp_path / 'a.nxs', values))['bank_0']['m']
assert loaded.dims == ('tube',)
assert loaded.values.tolist() == [False, True, True]

Expand Down Expand Up @@ -305,7 +333,7 @@ def bank() -> sc.DataArray:


def _per_bank(**banks: sc.Variable) -> PixelMasks:
return PixelMasks({'m': sc.DataGroup(banks)})
return PixelMasks({bank: sc.DataGroup(m=mask) for bank, mask in banks.items()})


def test_to_detector_masks_selects_bank_and_allows_fewer_dims(
Expand Down
Loading