From 90db02c937d9cf4cbe7e3269b26af0349d4c0635 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 19 Nov 2026 11:44:33 +0000 Subject: [PATCH 1/3] The table of a batch as a frame: dtypes, cells, rows.json table.py gains the frame form of ADR 0012 next to RowTable, with no caller yet: - dtypes(model): the dtype of the column of each parameter. - typed(frame, model): the frame as a batch holds it; refuses a key that is not text, empty or given twice, and a cell not of its column's dtype, naming each. - as_request / cell_of: the one place where a cell (text for a file or an output) and a value of a request are converted. - data_of / frame_of: the plain data of rows.json. pandas' to_json rounds floats and integers above 2**53, so the records are Python values and each column is read back with pd.array. ADR 0012 and 0013 are accepted. ADR 0012 now states how rows.json is written and read, takes a row with to_dict('index') (a row as one Series turns Int64 into float), converts files and outputs with as_request before fill, and names rows_of; ADR 0013 names rows_of. Co-Authored-By: Claude Fable 5.2 --- ...person-makes-the-batches-of-a-workspace.md | 2 +- docs/developer/adr/0012-a-batch-is-a-table.md | 30 +- ...013-follow-adds-the-rows-not-considered.md | 6 +- packages/essapps/src/ess/apps/batch/table.py | 354 +++++++++++++++++- packages/essapps/tests/batch/table_test.py | 200 +++++++++- .../essinstruments/tests/loki/batch_test.py | 24 +- 6 files changed, 591 insertions(+), 25 deletions(-) diff --git a/docs/developer/adr/0011-a-person-makes-the-batches-of-a-workspace.md b/docs/developer/adr/0011-a-person-makes-the-batches-of-a-workspace.md index 4c1207f5..3d049afd 100644 --- a/docs/developer/adr/0011-a-person-makes-the-batches-of-a-workspace.md +++ b/docs/developer/adr/0011-a-person-makes-the-batches-of-a-workspace.md @@ -1,6 +1,6 @@ # ADR 0011: A person makes the batches of a workspace, each with a scope over the rows of a source -- Status: accepted. Several workflows on one batch are described below and not decided. Amended 2026-11-18: the header holds the spec and the columns. Amended 2026-11-19: the manager keeps which batch follows; "Start the next batch here" leaves Follow where it is. [ADR 0012](0012-a-batch-is-a-table.md), proposed, replaces what is said here of the spec and the columns in the header and of `edits.json`. [ADR 0013](0013-follow-adds-the-rows-not-considered.md), proposed, replaces what is said here of the scope, of the source in the header, and of what a step of Follow reduces. +- Status: accepted. Several workflows on one batch are described below and not decided. Amended 2026-11-18: the header holds the spec and the columns. Amended 2026-11-19: the manager keeps which batch follows; "Start the next batch here" leaves Follow where it is. [ADR 0012](0012-a-batch-is-a-table.md) replaces what is said here of the spec and the columns in the header and of `edits.json`. [ADR 0013](0013-follow-adds-the-rows-not-considered.md) replaces what is said here of the scope, of the source in the header, and of what a step of Follow reduces. - Deciders: Simon - Date: 2026-11-18 diff --git a/docs/developer/adr/0012-a-batch-is-a-table.md b/docs/developer/adr/0012-a-batch-is-a-table.md index 2eee80a3..b71928fa 100644 --- a/docs/developer/adr/0012-a-batch-is-a-table.md +++ b/docs/developer/adr/0012-a-batch-is-a-table.md @@ -1,6 +1,6 @@ # ADR 0012: A batch is a template and a table -- Status: proposed (#236) +- Status: accepted (#236) - Deciders: Simon - Date: 2026-11-19 @@ -55,21 +55,22 @@ This decides what a batch is when it is made once and then changed by a person. - **The key** of a row is text, unique in the batch, and the index of the frame. It joins a row to its results: the records under the label of the batch with that key, as today. - **A column named as a parameter of the spec is a parameter column**, and the parameter columns are the blanks of the template. Any other column, such as `run` or `name`, is a note for the person. Nothing else tells them apart. -- **A cell holds one value, or is empty.** The value is the plain form of a parameter's value, as a request holds it before it is resolved. A file and an output are text: `run:60339`, and `r17.centre` for the output `centre` of record `r17`. +- **A cell holds one value, or is empty.** The value is plain data: a number, text, a truth value, a list, or a group. A file and an output are text: `run:60339`, and `r17.centre` for the output `centre` of record `r17`. A request takes neither as text, so the cells of a row are converted in one place before `fill` (`table.as_request`), and a value of a request back to a cell in one place (`table.cell_of`). - **A column has the type of its parameter**, so a wrong value is refused where it enters the table: | The parameter takes | dtype | A cell | |---|---|---| -| a file, an output, text | `string` | `'run:60339'` | +| a file, an output, a file or an output, text | `string` | `'run:60339'` | | a number | `Float64`, `Int64` | `2.0` | | a truth value | `boolean` | `True` | -| a list, a group, or more than one of the above | `object` | `['run:1', 'run:2']` | +| a list, a group, a value or an output, anything else | `object` | `['run:1', 'run:2']` | ### A row fills the template ```python def request(key): - cells = rows.loc[key, list(template.blanks)].dropna().to_dict() + row = rows[list(template.blanks)].loc[[key]].to_dict('index')[key] + cells = as_request(spec.params, {c: v for c, v in row.items() if v is not None}) for parameter in template.blanks - cells.keys(): # the empty cells if has_default(parameter): cells[parameter] = default(parameter) @@ -78,6 +79,8 @@ def request(key): return template.fill(params=cells) # refuses "a blank, not filled": the row is incomplete ``` +`to_dict('index')` gives each cell as a Python value of its column's type, and an empty cell as `None`. `rows.loc[key, blanks]` would not: the row is a `Series` of one dtype, so with only numbers among the blanks an `Int64` cell becomes a float, which a parameter of whole numbers refuses. + - **A parameter is a setting or a column, never both.** These are the rules of a template, unchanged: `fill` refuses a value for a parameter that the template fixes, and naming a parameter as a blank drops its value. So every value that the table contributes is in the table, and no cell depends on a setting. - **The settings are one dict**, parameter to value, with no keys of rows: `template.params`. - **An empty cell gives no value.** The parameter then takes its default, or `None` if it takes `None`, and the row is incomplete otherwise. A cell holds no word for `None`: in a table, empty is `None`. @@ -113,7 +116,7 @@ Removing a row drops it. Its records stay under the label, as every record does, The first rows are an argument of making the batch, as its first settings are; the parameter columns of the rows are its blanks. Where they come from is the business of whoever makes it: ```python -frame = table_of(source.rows(files)) # rows from the files: code of the application +frame = rows_of(client, source) # rows from the files: code of the application manager.create('loki-iofq', title='Afternoon', rows=frame[frame.run >= 60339]) manager.create('loki-iofq', title='Water', rows=pd.read_csv('runs.csv', index_col='key', dtype='string')) manager.create('loki-iofq', title='By hand') # no rows yet @@ -136,10 +139,19 @@ A cell that names a file that is not found is marked when the table is shown, an ``` definitions/labels/20261118-130204/header.json title, description definitions/labels/20261118-130204/settings.json the template, as a settings file that a person saves: spec, params, blanks -definitions/labels/20261118-130204/rows.json the table: the plain data of frame.to_json(orient='table') +definitions/labels/20261118-130204/rows.json the table: {"schema": ..., "data": [{"key": "60339", "sample": "run:60339", ...}, ...]} +``` + +`rows.json` has the shape of `frame.to_json(orient='table')`, but pandas neither writes nor reads its records: `to_json` rounds floats (`0.1 + 0.2` reads back `0.3`), and an `Int64` above 2**53 loses its last digit. So: + +```python +data = {'schema': build_table_schema(frame, version=False), # pandas: the dtype of each column + 'data': [{'key': key, **row} for key, row in frame.to_dict('index').items()]} +text = pydantic_core.to_json(data) # the shortest text of each float that reads back equal +column = pd.array([r.get(name) for r in data['data']], dtype=field['extDtype']) # each column, read back ``` -pandas writes and reads the table with the types of its columns. Checked on a table with `string`, `Float64`, `Int64` and `object` columns, empty cells, a list and a dict in a cell, and the key `'0342'`: the frame read back equals the frame written, with an empty cell of an `object` column read as `NaN`. +`table.data_of` and `table.frame_of` are the two halves. Checked on a table with `string`, `Float64`, `Int64`, `boolean` and `object` columns, empty cells, a list and a dict in a cell, `0.1 + 0.2`, `2**53 + 1`, a table without rows, and the key `'0342'`: the frame read back equals the frame written, dtypes included, with an empty cell of an `object` column read as `None`. ### "Start the next batch here" splits the table @@ -191,7 +203,7 @@ The batch holds its rows as a `DataFrame`, so pandas becomes a dependency of `es |---|---| | a row, a key | a row of the frame, its index | | the parameter columns | `[c for c in rows.columns if c in parameters]`, equal to `template.blanks` | -| the request of a row | `template.fill(params=rows.loc[key, blanks].dropna().to_dict())` | +| the request of a row | `template.fill(params=as_request(spec.params, rows[blanks].loc[[key]].to_dict('index')[key]))`, its empty cells left out | | set, empty, add, remove | `.loc[key, column] = value`, `= pd.NA`, `.loc[key, column] = value` at a new key, `.drop(key)` | | the results | `records.sort_values('created').drop_duplicates('member', keep='last')` | | the state | `rows.join(results)`, a left join on the key | diff --git a/docs/developer/adr/0013-follow-adds-the-rows-not-considered.md b/docs/developer/adr/0013-follow-adds-the-rows-not-considered.md index 0e3fab04..fd14a394 100644 --- a/docs/developer/adr/0013-follow-adds-the-rows-not-considered.md +++ b/docs/developer/adr/0013-follow-adds-the-rows-not-considered.md @@ -1,6 +1,6 @@ # ADR 0013: Follow adds the rows that a batch has not considered -- Status: proposed (#236) +- Status: accepted (#236) - Deciders: Simon - Date: 2026-11-19 @@ -26,8 +26,8 @@ To do the third, a batch has to know who wrote each cell. To do the first withou **A batch made from a source keeps the name of the source and the keys it has considered. A step of Follow adds the rows of the source whose keys are not considered, once the source has filled them, and never changes a row that the table has.** ```python -def step(batch, source, files, considered): - given = table_of(source.rows(files)) # every row the source makes from the files now +def step(batch, source, considered): + given = rows_of(batch.client, source) # every row the source makes from the files now table = batch.frame() filled = [c for c in source.fills if c in table.columns] is_new = ~given.index.isin(considered) & ~given.index.isin(table.index) diff --git a/packages/essapps/src/ess/apps/batch/table.py b/packages/essapps/src/ess/apps/batch/table.py index e7e43f50..87fce0e3 100644 --- a/packages/essapps/src/ess/apps/batch/table.py +++ b/packages/essapps/src/ess/apps/batch/table.py @@ -1,26 +1,94 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2026 Scipp contributors (https://github.com/scipp) """ -The rows of a batch, as an immutable value. - -The table holds two layers: what the last scan gave, and what was set by hand -since, a person's edit or a program's. A row's cells are the scan's with the -edits on top, worked out when asked for. So a new scan replaces only its own -layer, and no merge is needed: each rule below follows from which layer an -operation writes. The table keeps every row of the scan and shows those in its -scope, so a change of scope needs no new scan. Cells are opaque; this module -knows nothing of specs, files, or widgets. +The rows of a batch: as an immutable value of two layers (:class:`RowTable`), +and as a frame (:data:`KEY` and the functions after it). + +A :class:`RowTable` holds two layers: what the last scan gave, and what was set +by hand since, a person's edit or a program's. A row's cells are the scan's +with the edits on top, worked out when asked for. So a new scan replaces only +its own layer, and no merge is needed: each rule below follows from which layer +an operation writes. The table keeps every row of the scan and shows those in +its scope, so a change of scope needs no new scan. The cells of a +:class:`RowTable` are opaque; it knows nothing of specs, files, or widgets. + +The frame form +-------------- + +A table is one ``DataFrame``: the key of each row as index, named +:data:`KEY`, a column per parameter of the spec that the rows give, and notes, +any other column:: + + key run name sample sample_transmission + 60339 60339 D2O buffer run:60339 run:60305 + water by hand run:60012 + +A key is text, not empty, and unique. A column of a parameter has the dtype +that :func:`dtypes` reads from the type of the parameter, with +``field_types.unwrapped`` and ``field_types.split``:: + + the parameter takes dtype + a file, an output, a file or an output string + text, a Literal of text, a time string + a number Float64 + a whole number Int64 + a truth value boolean + a list, a group, a value or an output, anything else object + +``| None`` does not change the dtype: empty is ``None``. :func:`typed` casts a +frame to these dtypes, and refuses what pandas refuses and a value that is not +text in a ``string`` column, which pandas would turn into text, ``60339.0`` +into ``'60339.0'``. Notes take the dtypes of ``convert_dtypes()``. So a table +has one kind of empty per dtype: ``pd.NA``, and ``None`` in an ``object`` +column. + +A cell is plain data. A file and an output are text, which a request does not +take, so :func:`as_request` converts the cells of a row before +``Template.fill``, and :func:`cell_of` converts a value of a request back:: + + # at a parameter that takes a file + 'run:60339' -> FileRef(file='run:60339') + 60339 -> FileRef(file='run:60339') # a run; in a template, or a list + # at a parameter that takes an output + 'r17.centre' -> OutputRef(record='r17', output='centre') + 'r18.slices[2]' -> OutputRef(record='r18', output='slices', key='2') + # at a parameter that takes a file or an output + 'run:1' -> FileRef(file='run:1') # text with a kind, [a-z]+: + 'r17.centre' -> OutputRef(record='r17', output='centre') + # at a parameter that takes a list of these: each element + ['run:1', 2] -> [FileRef(file='run:1'), FileRef(file='run:2')] + # anything else, a file inside a group too: the cell + {'bins': 4} -> {'bins': 4} + +``rows.json`` holds :func:`data_of` of the frame, and :func:`frame_of` reads it +back equal, dtypes included. Its shape is that of ``to_json(orient='table')``: +a table schema, by ``build_table_schema``, and one record per row, of Python +values. pandas' own ``to_json`` and ``read_json`` round floats, and integers +above 2**53, so they neither write nor read the records. A JSON writer of +Python values keeps each float and integer exact, and :func:`frame_of` builds +each column with ``pd.array(values, dtype=extDtype)``. +A row is read as ``frame.to_dict('index')``, which gives Python values of each +column's type, never a NumPy scalar or a ``Float64`` in place of an ``Int64``. """ from __future__ import annotations +import re from collections import Counter from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field, replace from functools import cached_property -from typing import Any +from typing import Any, Literal, get_args, get_origin + +import pandas as pd +from pandas.io.json import build_table_schema +from pydantic import BaseModel +from pydantic.fields import FieldInfo + +from ess.spec import FileRef, OutputRef, field_types from ..errors import FieldError, SubmitError +from ..files import file class _Empty: @@ -290,3 +358,269 @@ def reset_each(self, keys: Iterable[str], *columns: str) -> RowTable: edits[key] = kept removed = self.removed if columns else self.removed - keys return replace(self, edits=edits, removed=removed) + + +KEY = 'key' +"""The name of the index of a table: the key of each row.""" + +_WORDS = { + 'string': 'text', + 'Float64': 'a number', + 'Int64': 'a whole number', + 'boolean': 'true or false', +} +"""What a cell of each dtype but ``object`` holds, as a message says it.""" + +_KIND = re.compile(r'[a-z]+:') +_OUTPUT = re.compile( + r'(?P[^.\[\]]+)\.(?P[^.\[\]]+)(?:\[(?P.*)\])?' +) + + +@dataclass(frozen=True) +class _Parameter: + """What a parameter takes, as a cell gives it.""" + + many: bool # a list of what the other fields say + inline: Any # the part that a request writes, None if only an output + file: bool + output: bool + + +def _parameter(info: FieldInfo) -> _Parameter: + metadata = field_types.field_metadata(info) + tp, _, metadata = field_types.unwrapped(info.annotation, metadata) + many = get_origin(tp) is list + if many: + tp, _, metadata = field_types.unwrapped(get_args(tp)[0]) + inline, inline_metadata, referenced = field_types.split(tp, *metadata) + is_file = ( + inline is not None + and field_types.file_marker(inline, *inline_metadata) is not None + ) + return _Parameter(many, inline, is_file, bool(referenced)) + + +def _dtype(parameter: _Parameter) -> str: + tp = parameter.inline + if parameter.many: + return 'object' + if parameter.file or tp is None: + return 'string' + if parameter.output: + return 'object' + if tp is bool: + return 'boolean' + if tp is int: + return 'Int64' + if tp is float: + return 'Float64' + if ( + tp is str + or tp in field_types.TIMES + or (get_origin(tp) is Literal and all(type(a) is str for a in get_args(tp))) + ): + return 'string' + return 'object' + + +def dtypes(model: type[BaseModel]) -> dict[str, str]: + """The dtype of the column of each parameter of ``model``, by name.""" + return {n: _dtype(_parameter(i)) for n, i in model.model_fields.items()} + + +def _empty(value: Any) -> bool: + """Whether a cell is empty: ``None``, ``NaN`` or ``pd.NA``, not a list.""" + return value is None or (pd.api.types.is_scalar(value) and bool(pd.isna(value))) + + +def _keys(index: pd.Index) -> pd.Index: + """``index`` as a table's index; raises ``ValueError`` for a key refused.""" + if pd.api.types.is_numeric_dtype(index) and len(index): + raise ValueError( + f'a key is text, not a number such as {index[0]}; make the keys ' + 'text with frame.index = frame.index.astype(str)' + ) + problems = [ + f'the key {k!r} is not text' + for k in index + if not _empty(k) and not isinstance(k, str) + ] + if any(_empty(k) or k == '' for k in index): + problems.append('a key is empty') + problems += [ + f'the key {k!r} is given twice' for k, n in Counter(index).items() if n > 1 + ] + if problems: + raise ValueError('; '.join(problems)) + return pd.Index(index, name=KEY, dtype='string') + + +def _objects(column: pd.Series) -> pd.Series: + """``column`` as an ``object`` column, ``None`` in each empty cell.""" + return pd.Series( + [None if _empty(v) else v for v in column], index=column.index, dtype=object + ) + + +def _fits(value: Any, dtype: str) -> bool: + """Whether a cell that is not empty fits a column of ``dtype``.""" + if dtype == 'string': + return isinstance(value, str) + try: + pd.array([value], dtype=dtype) + except (TypeError, ValueError): + return False + return True + + +def _cast(column: pd.Series, dtype: str) -> tuple[pd.Series, list[str]]: + """``column`` cast to ``dtype``, and the cells refused, each as a message.""" + if dtype == 'object': + return _objects(column), [] + if dtype != 'string' or all(_empty(v) or isinstance(v, str) for v in column): + try: + return column.astype(dtype), [] + except (TypeError, ValueError): + pass + refused = [ + f'row {k!r}, column {column.name!r}: {v!r} is not {_WORDS[dtype]}' + for k, v in zip(column.index, column.tolist(), strict=True) + if not _empty(v) and not _fits(v, dtype) + ] + return column, refused or [ + f'column {column.name!r}: not {_WORDS[dtype]} in each cell' + ] + + +def typed(frame: pd.DataFrame, model: type[BaseModel]) -> pd.DataFrame: + """ + ``frame`` as a batch holds it: the index of text named :data:`KEY`, each + column of a parameter of ``model`` cast to its dtype (see :func:`dtypes`), + and each note to the dtype of ``convert_dtypes()``. An empty cell of an + ``object`` column is ``None``. + + Raises ``ValueError``, naming each key, column or cell refused: a key + that is not text, empty, or given twice; a column given twice; a cell + that is not of the dtype of its column. + """ + index = _keys(frame.index) + problems = [ + f'the column {c!r} is given twice' + for c, n in Counter(frame.columns).items() + if n > 1 + ] + if problems: + raise ValueError('; '.join(problems)) + types = dtypes(model) + columns = {} + for name, column in frame.set_axis(index).items(): + if name in types: + columns[name], refused = _cast(column, types[name]) + problems += refused + else: + note = column.convert_dtypes() + columns[name] = _objects(note) if note.dtype == object else note + if problems: + raise ValueError('; '.join(problems)) + return pd.DataFrame(columns, index=index, columns=frame.columns) + + +def _request_value(parameter: _Parameter, cell: Any) -> Any: + if parameter.file and isinstance(cell, int) and not isinstance(cell, bool): + return file(run=cell) + if not isinstance(cell, str): + return cell + if parameter.file and (not parameter.output or _KIND.match(cell)): + return FileRef(file=cell) + if parameter.output and (found := _OUTPUT.fullmatch(cell)): + return OutputRef(**found.groupdict()) + return cell + + +def as_request(model: type[BaseModel], cells: Mapping[str, Any]) -> dict[str, Any]: + """ + ``cells``, by the name of a parameter of ``model``, as a request takes + them: at a parameter that takes a file, text is the file of that name and + a whole number a run; at one that takes an output, text such as + ``'r17.centre'`` or ``'r18.slices[2]'`` is that output; at one that takes + either, text with a kind, such as ``run:``, is a file, and other text an + output. In a list, each element is converted so. Any other cell is + returned as it is, for the request to refuse if it does not fit. + """ + fields = model.model_fields + converted = {} + for name, cell in cells.items(): + if name not in fields: + converted[name] = cell + continue + parameter = _parameter(fields[name]) + if parameter.many and isinstance(cell, list): + converted[name] = [_request_value(parameter, c) for c in cell] + else: + converted[name] = _request_value(parameter, cell) + return converted + + +def cell_of(value: Any) -> Any: + """ + The cell of ``value`` of a request, the inverse of :func:`as_request`: + a file is its name, such as ``'run:1'``, an output its text, such as + ``'r17.centre'``, and a list holds the cell of each element. + """ + if isinstance(value, list): + return [cell_of(v) for v in value] + if isinstance(value, FileRef | OutputRef): + return str(value) + return value + + +def data_of(frame: pd.DataFrame) -> dict[str, Any]: + """ + The plain data of ``frame``, a table as :func:`typed` gives it, as + ``rows.json`` holds it: ``{'schema': ..., 'data': [{'key': ..., ...}]}``, + with ``None`` in each empty cell. Written as JSON by a writer of Python + values, it keeps each value exact. + """ + records = [ + {KEY: key, **{c: None if _empty(v) else v for c, v in row.items()}} + for key, row in frame.to_dict('index').items() + ] + return {'schema': build_table_schema(frame, version=False), 'data': records} + + +def frame_of(data: Any) -> pd.DataFrame: + """ + The table of the plain data of :func:`data_of`, equal to the frame it was + made of. Raises ``ValueError``, naming the entry, for data that are not + such a table. + """ + try: + fields = { + f['name']: f.get('extDtype', 'object') for f in data['schema']['fields'] + } + records = list(data['data']) + except (KeyError, TypeError, AttributeError): + raise ValueError( + "not a table: {'schema': {'fields': [...]}, 'data': [...]}" + ) from None + if fields.pop(KEY, None) is None: + raise ValueError(f'schema: no field {KEY!r}') + for i, record in enumerate(records): + if not isinstance(record, dict) or KEY not in record: + raise ValueError(f'data[{i}]: not an object with a {KEY!r}') + if unknown := sorted(record.keys() - fields.keys() - {KEY}): + raise ValueError(f'data[{i}]: no field {", ".join(unknown)} in the schema') + index = _keys(pd.Index([r[KEY] for r in records], dtype=object)) + columns = {} + for name, dtype in fields.items(): + values = [r.get(name) for r in records] + try: + columns[name] = ( + pd.Series(values, index=index, dtype=object) + if dtype == 'object' + else pd.Series(pd.array(values, dtype=dtype), index=index) + ) + except (TypeError, ValueError) as error: + raise ValueError(f'column {name!r}: {error}') from None + return pd.DataFrame(columns, index=index, columns=list(fields)) diff --git a/packages/essapps/tests/batch/table_test.py b/packages/essapps/tests/batch/table_test.py index db242b05..1a8737f2 100644 --- a/packages/essapps/tests/batch/table_test.py +++ b/packages/essapps/tests/batch/table_test.py @@ -1,15 +1,31 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2026 Scipp contributors (https://github.com/scipp) -"""The rules of a table of rows, on plain values.""" +"""The rules of a table of rows, on plain values; the table as a frame.""" import re from dataclasses import replace from datetime import UTC, datetime +from typing import Any, Literal +import pandas as pd import pytest +from pydantic import BaseModel +from pydantic_core import from_json, to_json from ess.apps import SubmitError from ess.apps.batch import EMPTY, Row, RowTable, Scope +from ess.apps.batch.table import ( + KEY, + as_request, + cell_of, + data_of, + dtypes, + frame_of, + typed, +) +from ess.spec import FileRef, OutputRef, Referenced, RunFile + +from . import toys def cells(table: RowTable) -> dict[str, dict[str, object]]: @@ -378,3 +394,185 @@ def test_a_row_added_by_hand_with_the_key_of_a_row_outside_the_scope_is_refused( ) -> None: with pytest.raises(SubmitError, match='outside the scope'): scoped.add(Row('1')) + + +# The table as a frame + + +class Kinds(BaseModel): + count: int = 1 + flag: bool = False + word: str = '' + choice: Literal['a', 'b'] = 'a' + level: Literal[1, 2] = 1 + when: datetime | None = None + scale: float | None = None + either: int | str = 0 + centre: Referenced[toys.Offset] | None = None + source: RunFile | Referenced[toys.Offset] | None = None + + +def test_a_column_has_the_dtype_of_its_parameter() -> None: + assert dtypes(Kinds) == { + 'count': 'Int64', + 'flag': 'boolean', + 'word': 'string', + 'choice': 'string', + 'level': 'object', + 'when': 'string', + 'scale': 'Float64', + 'either': 'object', + 'centre': 'string', + 'source': 'string', + } + + +def test_the_columns_of_the_toy_have_the_dtypes_of_their_parameters() -> None: + assert dtypes(toys.Params) == { + 'binning': 'object', + 'sample': 'string', + 'transmission': 'string', + 'thickness': 'Float64', + 'offset': 'object', + } + assert dtypes(toys.SeriesParams) == {'runs': 'object'} + + +def test_typed_names_the_index_and_casts_parameters_and_notes() -> None: + frame = pd.DataFrame( + { + 'sample': ['run:1', None], + 'thickness': [2, None], + 'binning': [{'bins': 3}, float('nan')], + 'run': [1.0, None], + 'name': ['water', None], + }, + index=['0342', 'b'], + ) + + table = typed(frame, toys.Params) + + assert table.index.name == KEY + assert list(table.index) == ['0342', 'b'] + assert table.index.dtype == 'string' + assert table.dtypes.astype(str).to_dict() == { + 'sample': 'string', + 'thickness': 'Float64', + 'binning': 'object', + 'run': 'Int64', + 'name': 'string', + } + assert table.loc['b', 'binning'] is None + + +@pytest.mark.parametrize( + ('index', 'columns', 'message'), + [ + (['a', 'a'], {}, "the key 'a' is given twice"), + (['a', ''], {}, 'a key is empty'), + ([60339], {}, r'not a number such as 60339; .*frame.index.astype\(str\)'), + (['a', 'b'], {'scale': [1.0, 'thick']}, "row 'b', column 'scale': 'thick'"), + (['a', 'b'], {'word': ['run:1', 60339.0]}, "row 'b', column 'word': 60339.0"), + (['a'], {'count': [2.5]}, "row 'a', column 'count': 2.5 is not a whole"), + (['a'], {'flag': ['yes']}, "row 'a', column 'flag': 'yes' is not true"), + ], +) +def test_typed_refuses_a_key_or_a_cell_and_names_it( + index: list[Any], columns: dict[str, list[Any]], message: str +) -> None: + with pytest.raises(ValueError, match=message): + typed(pd.DataFrame(columns, index=index), Kinds) + + +@pytest.fixture +def exact() -> pd.DataFrame: + key = pd.Index(['a', 'b', '0342'], name=KEY, dtype='string') + return pd.DataFrame( + { + 'x': pd.array([0.1 + 0.2, None, 6.02214076e23], dtype='Float64'), + 'n': pd.array([2**53 + 1, None, 1], dtype='Int64'), + 'f': pd.array([True, None, False], dtype='boolean'), + 's': pd.array(['run:1', None, 'r17.centre'], dtype='string'), + 'o': pd.Series([['run:1'], None, {'v': 0.1 + 0.2}], index=key), + }, + index=key, + ) + + +def test_the_data_of_a_table_read_back_as_json_give_the_table_exactly( + exact: pd.DataFrame, +) -> None: + back = frame_of(from_json(to_json(data_of(exact)))) + + pd.testing.assert_frame_equal(back, exact) + assert back.loc['a', 'x'] == 0.1 + 0.2 + assert back.loc['a', 'n'] == 2**53 + 1 + + +def test_the_data_of_a_table_without_rows_give_its_columns_and_dtypes( + exact: pd.DataFrame, +) -> None: + empty = exact.iloc[:0] + + pd.testing.assert_frame_equal(frame_of(data_of(empty)), empty) + + +@pytest.mark.parametrize( + ('change', 'message'), + [ + (lambda d: d.pop('schema'), 'not a table'), + (lambda d: d['data'][1].pop(KEY), "data\\[1\\]: not an object with a 'key'"), + (lambda d: d['data'][2].update(y=1), 'data\\[2\\]: no field y'), + (lambda d: d['data'][0].update(n=2.5), "column 'n'"), + (lambda d: d['data'][1].update(key='a'), "the key 'a' is given twice"), + ], +) # fmt: skip +def test_data_that_are_not_a_table_are_refused_by_entry( + exact: pd.DataFrame, change: Any, message: str +) -> None: + data = from_json(to_json(data_of(exact))) + change(data) + + with pytest.raises(ValueError, match=message): + frame_of(data) + + +class Either(BaseModel): + source: RunFile | Referenced[toys.Offset] + centre: Referenced[toys.Offset] + + +def output(record: str, name: str, key: str | None = None) -> OutputRef: + return OutputRef(record=record, output=name, key=key) + + +@pytest.mark.parametrize( + ('model', 'cells', 'request_values'), + [ + (toys.Params, {'sample': 'run:60339'}, {'sample': FileRef(file='run:60339')}), + (toys.Params, {'sample': 'path:a.nxs'}, {'sample': FileRef(file='path:a.nxs')}), + (toys.Params, {'offset': 'r17.offset'}, {'offset': output('r17', 'offset')}), + (toys.Params, {'binning': {'bins': 3}}, {'binning': {'bins': 3}}), + (Either, {'centre': 'r18.slices[2]'}, {'centre': output('r18', 'slices', '2')}), + (Either, {'source': 'run:1'}, {'source': FileRef(file='run:1')}), + (Either, {'source': 'r17.offset'}, {'source': output('r17', 'offset')}), + ( + toys.SeriesParams, + {'runs': ['run:1', 'run:2']}, + {'runs': [FileRef(file='run:1'), FileRef(file='run:2')]}, + ), + ], +) +def test_a_cell_is_converted_to_what_a_request_takes_and_back( + model: type[BaseModel], cells: dict[str, Any], request_values: dict[str, Any] +) -> None: + converted = as_request(model, cells) + + assert converted == request_values + assert {name: cell_of(v) for name, v in converted.items()} == cells + + +def test_a_whole_number_at_a_file_is_a_run() -> None: + assert as_request(toys.SeriesParams, {'runs': [60339, 'run:2']}) == { + 'runs': [FileRef(file='run:60339'), FileRef(file='run:2')] + } diff --git a/packages/essinstruments/tests/loki/batch_test.py b/packages/essinstruments/tests/loki/batch_test.py index 9bf34e0a..14f3a604 100644 --- a/packages/essinstruments/tests/loki/batch_test.py +++ b/packages/essinstruments/tests/loki/batch_test.py @@ -8,10 +8,16 @@ import pytest from ess.instruments.loki.batch import iofq_batch, pair_by_sample from ess.instruments.loki.bindings import BINDINGS -from ess.instruments.loki.specs import CALIBRATED_DETECTOR, MASKS, BeamCentre +from ess.instruments.loki.specs import ( + CALIBRATED_DETECTOR, + MASKS, + BeamCentre, + IofQParams, +) from ess.apps import Client, FolderFiles, file, local from ess.apps.batch import FileInfo, left_to_reduce, ready +from ess.apps.batch.table import dtypes from ess.spec import FileRef @@ -19,6 +25,22 @@ def info(run: int, **metadata: object) -> FileInfo: return FileInfo(FileRef(file=f'uuid:{run}'), {'run': run, **metadata}) +def test_a_column_of_iofq_has_the_dtype_of_its_parameter() -> None: + assert dtypes(IofQParams) == { + 'sample': 'string', + 'sample_transmission': 'string', + 'can': 'string', + 'can_transmission': 'string', + 'empty_beam': 'string', + 'direct_beam': 'string', + 'masks': 'string', + 'beam_centre': 'object', + 'wavelength': 'object', + 'q': 'object', + 'corrections': 'object', + } + + def test_a_sample_run_pairs_with_the_last_transmission_run_of_its_sample() -> None: rows = pair_by_sample( [ From af310bb93a23ee091072cb62d1f3d8f583c3e52b Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 19 Nov 2026 11:58:35 +0000 Subject: [PATCH 2/3] Close the edges of the frame form of a table From review of the frame form: - A cell must be of the kind of its column, with no conversion by pandas: string takes text, Float64 an int or a float, Int64 an int within 64 bits, boolean a bool, object plain data. A float NaN is empty. So text 'nan' cannot become a value that rows.json empties, and True is not a number. - typed refuses a column named 'key' (data_of would overwrite the keys with it), a column not named by text, and a note whose dtype is not one of the five, which rows.json would change silently. - frame_of builds each column with the same rule as typed, refuses a field given twice, and names row and column in its errors. - as_request leaves empty text at a file for the request to refuse. - cell_of gives the JSON form of anything that is not a reference or a list, such as a group or a time. - dtypes refuses a parameter that takes text or an output, since a cell cannot tell the two apart; an enum of text and a date are text. ADR 0012 says a time is text, that no column is named 'key', and that its read_csv example holds for tables of files and text only. Co-Authored-By: Claude Fable 5.2 --- docs/developer/adr/0012-a-batch-is-a-table.md | 10 +- packages/essapps/src/ess/apps/batch/table.py | 259 +++++++++++------- packages/essapps/tests/batch/table_test.py | 110 +++++++- 3 files changed, 260 insertions(+), 119 deletions(-) diff --git a/docs/developer/adr/0012-a-batch-is-a-table.md b/docs/developer/adr/0012-a-batch-is-a-table.md index b71928fa..2d36fee2 100644 --- a/docs/developer/adr/0012-a-batch-is-a-table.md +++ b/docs/developer/adr/0012-a-batch-is-a-table.md @@ -53,18 +53,20 @@ This decides what a batch is when it is made once and then changed by a person. ### Keys, columns, cells -- **The key** of a row is text, unique in the batch, and the index of the frame. It joins a row to its results: the records under the label of the batch with that key, as today. +- **The key** of a row is text, unique in the batch, and the index of the frame, named `key`; no column has that name. It joins a row to its results: the records under the label of the batch with that key, as today. - **A column named as a parameter of the spec is a parameter column**, and the parameter columns are the blanks of the template. Any other column, such as `run` or `name`, is a note for the person. Nothing else tells them apart. - **A cell holds one value, or is empty.** The value is plain data: a number, text, a truth value, a list, or a group. A file and an output are text: `run:60339`, and `r17.centre` for the output `centre` of record `r17`. A request takes neither as text, so the cells of a row are converted in one place before `fill` (`table.as_request`), and a value of a request back to a cell in one place (`table.cell_of`). - **A column has the type of its parameter**, so a wrong value is refused where it enters the table: | The parameter takes | dtype | A cell | |---|---|---| -| a file, an output, a file or an output, text | `string` | `'run:60339'` | +| a file, an output, a file or an output; text, a `Literal` or an enum of text, a time, a date | `string` | `'run:60339'` | | a number | `Float64`, `Int64` | `2.0` | | a truth value | `boolean` | `True` | | a list, a group, a value or an output, anything else | `object` | `['run:1', 'run:2']` | +A cell must be of the kind of its column, and nothing converts it: `'2'` and `True` are not numbers. + ### A row fills the template ```python @@ -118,7 +120,7 @@ The first rows are an argument of making the batch, as its first settings are; t ```python frame = rows_of(client, source) # rows from the files: code of the application manager.create('loki-iofq', title='Afternoon', rows=frame[frame.run >= 60339]) -manager.create('loki-iofq', title='Water', rows=pd.read_csv('runs.csv', index_col='key', dtype='string')) +manager.create('loki-iofq', title='Water', rows=pd.read_csv('runs.csv', index_col='key', dtype='string')) # files and text only manager.create('loki-iofq', title='By hand') # no rows yet ``` @@ -151,7 +153,7 @@ text = pydantic_core.to_json(data) column = pd.array([r.get(name) for r in data['data']], dtype=field['extDtype']) # each column, read back ``` -`table.data_of` and `table.frame_of` are the two halves. Checked on a table with `string`, `Float64`, `Int64`, `boolean` and `object` columns, empty cells, a list and a dict in a cell, `0.1 + 0.2`, `2**53 + 1`, a table without rows, and the key `'0342'`: the frame read back equals the frame written, dtypes included, with an empty cell of an `object` column read as `None`. +`table.data_of` and `table.frame_of` are the two halves. Checked on a table with `string`, `Float64`, `Int64`, `boolean` and `object` columns, empty cells, a list and a dict in a cell, `0.1 + 0.2`, `2**53 + 1`, a table without rows, and the key `'0342'`: the frame read back equals the frame written, dtypes included, with an empty cell of an `object` column read as `None`. It holds for every table, since a note too is text, a number, a truth value, or plain data. ### "Start the next batch here" splits the table diff --git a/packages/essapps/src/ess/apps/batch/table.py b/packages/essapps/src/ess/apps/batch/table.py index 87fce0e3..a51088d1 100644 --- a/packages/essapps/src/ess/apps/batch/table.py +++ b/packages/essapps/src/ess/apps/batch/table.py @@ -23,24 +23,29 @@ 60339 60339 D2O buffer run:60339 run:60305 water by hand run:60012 -A key is text, not empty, and unique. A column of a parameter has the dtype -that :func:`dtypes` reads from the type of the parameter, with -``field_types.unwrapped`` and ``field_types.split``:: +A key is text, not empty, and unique, and no column is named :data:`KEY`. +A column of a parameter has the dtype that :func:`dtypes` reads from the type +of the parameter, with ``field_types.unwrapped`` and ``field_types.split``:: the parameter takes dtype a file, an output, a file or an output string - text, a Literal of text, a time string + text, a Literal or an enum of text, a time, a date string a number Float64 a whole number Int64 a truth value boolean a list, a group, a value or an output, anything else object + (a Literal of numbers, for one) -``| None`` does not change the dtype: empty is ``None``. :func:`typed` casts a -frame to these dtypes, and refuses what pandas refuses and a value that is not -text in a ``string`` column, which pandas would turn into text, ``60339.0`` -into ``'60339.0'``. Notes take the dtypes of ``convert_dtypes()``. So a table -has one kind of empty per dtype: ``pd.NA``, and ``None`` in an ``object`` -column. +``| None`` does not change the dtype. A parameter that takes text or an output +is refused: a cell cannot tell its text from an output. + +A cell must be of the kind of its column, with no conversion by pandas: +``string`` takes text, ``Float64`` an ``int`` or a ``float``, ``Int64`` an +``int``, ``boolean`` a ``bool``, and ``object`` plain data, the values of JSON +as Python gives them. A note takes the dtype of ``convert_dtypes()``, which +must be one of these five. An empty cell is ``None``, ``NaN`` or ``pd.NA`` as +given; in the table it is ``pd.NA``, or ``None`` in an ``object`` column, and +in a row read with ``to_dict('index')`` and in ``rows.json`` it is ``None``. A cell is plain data. A file and an output are text, which a request does not take, so :func:`as_request` converts the cells of a row before @@ -60,6 +65,9 @@ # anything else, a file inside a group too: the cell {'bins': 4} -> {'bins': 4} +:func:`cell_of` gives the JSON form of anything else, such as a group, a time, +or a file inside a group (``{'@ref': 'file', 'file': 'run:1'}``). + ``rows.json`` holds :func:`data_of` of the frame, and :func:`frame_of` reads it back equal, dtypes included. Its shape is that of ``to_json(orient='table')``: a table schema, by ``build_table_schema``, and one record per row, of Python @@ -77,6 +85,8 @@ from collections import Counter from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field, replace +from datetime import date +from enum import Enum from functools import cached_property from typing import Any, Literal, get_args, get_origin @@ -84,6 +94,7 @@ from pandas.io.json import build_table_schema from pydantic import BaseModel from pydantic.fields import FieldInfo +from pydantic_core import to_jsonable_python from ess.spec import FileRef, OutputRef, field_types @@ -368,12 +379,15 @@ def reset_each(self, keys: Iterable[str], *columns: str) -> RowTable: 'Float64': 'a number', 'Int64': 'a whole number', 'boolean': 'true or false', + 'object': 'plain data', } -"""What a cell of each dtype but ``object`` holds, as a message says it.""" +"""What a cell of each dtype holds, as a message says it.""" _KIND = re.compile(r'[a-z]+:') +# What OutputRef.__str__ writes, given that a record ID (uuid4().hex) and an +# output name (a field name) contain none of '.', '[', ']', ':'. _OUTPUT = re.compile( - r'(?P[^.\[\]]+)\.(?P[^.\[\]]+)(?:\[(?P.*)\])?' + r'(?P[^.\[\]:]+)\.(?P[^.\[\]]+)(?:\[(?P.*)\])?' ) @@ -387,13 +401,26 @@ class _Parameter: output: bool -def _parameter(info: FieldInfo) -> _Parameter: +def _text(tp: Any) -> bool: + """Whether a value of ``tp`` is text in its JSON form.""" + if get_origin(tp) is Literal: + return all(type(a) is str for a in get_args(tp)) + if isinstance(tp, type) and issubclass(tp, Enum): + return all(type(m.value) is str for m in tp) + return tp is str or tp is date or tp in field_types.TIMES + + +def _parameter(name: str, info: FieldInfo) -> _Parameter: metadata = field_types.field_metadata(info) tp, _, metadata = field_types.unwrapped(info.annotation, metadata) many = get_origin(tp) is list if many: tp, _, metadata = field_types.unwrapped(get_args(tp)[0]) inline, inline_metadata, referenced = field_types.split(tp, *metadata) + if referenced and _text(inline): + raise ValueError( + f'{name}: takes text or an output, and a cell cannot tell the two apart' + ) is_file = ( inline is not None and field_types.file_marker(inline, *inline_metadata) is not None @@ -415,18 +442,15 @@ def _dtype(parameter: _Parameter) -> str: return 'Int64' if tp is float: return 'Float64' - if ( - tp is str - or tp in field_types.TIMES - or (get_origin(tp) is Literal and all(type(a) is str for a in get_args(tp))) - ): - return 'string' - return 'object' + return 'string' if _text(tp) else 'object' def dtypes(model: type[BaseModel]) -> dict[str, str]: - """The dtype of the column of each parameter of ``model``, by name.""" - return {n: _dtype(_parameter(i)) for n, i in model.model_fields.items()} + """ + The dtype of the column of each parameter of ``model``, by name. Raises + ``ValueError`` for a parameter that takes text or an output. + """ + return {n: _dtype(_parameter(n, i)) for n, i in model.model_fields.items()} def _empty(value: Any) -> bool: @@ -434,13 +458,31 @@ def _empty(value: Any) -> bool: return value is None or (pd.api.types.is_scalar(value) and bool(pd.isna(value))) +def _plain(value: Any) -> bool: + """Whether ``value`` is plain data: what JSON holds, as Python gives it.""" + if isinstance(value, list): + return all(_plain(v) for v in value) + if isinstance(value, dict): + return all(isinstance(k, str) and _plain(v) for k, v in value.items()) + return value is None or type(value) in (str, int, float, bool) + + +def _fits(value: Any, dtype: str) -> bool: + """Whether a cell that is not empty is of the kind of a column of ``dtype``.""" + if dtype == 'object': + return _plain(value) + if isinstance(value, bool): + return dtype == 'boolean' + if dtype == 'Int64': + return isinstance(value, int) and -(2**63) <= value < 2**63 + kinds = {'string': str, 'Float64': (int, float), 'boolean': bool} + return isinstance(value, kinds[dtype]) + + def _keys(index: pd.Index) -> pd.Index: """``index`` as a table's index; raises ``ValueError`` for a key refused.""" if pd.api.types.is_numeric_dtype(index) and len(index): - raise ValueError( - f'a key is text, not a number such as {index[0]}; make the keys ' - 'text with frame.index = frame.index.astype(str)' - ) + raise ValueError(f'a key is text, not a number such as {index[0]}') problems = [ f'the key {k!r} is not text' for k in index @@ -456,74 +498,73 @@ def _keys(index: pd.Index) -> pd.Index: return pd.Index(index, name=KEY, dtype='string') -def _objects(column: pd.Series) -> pd.Series: - """``column`` as an ``object`` column, ``None`` in each empty cell.""" - return pd.Series( - [None if _empty(v) else v for v in column], index=column.index, dtype=object - ) - - -def _fits(value: Any, dtype: str) -> bool: - """Whether a cell that is not empty fits a column of ``dtype``.""" - if dtype == 'string': - return isinstance(value, str) - try: - pd.array([value], dtype=dtype) - except (TypeError, ValueError): - return False - return True +def _names(names: Iterable[Any]) -> list[str]: + """The problems of the names of the columns of a table, each as a message.""" + names = list(names) + problems = [f'the column {c!r} is not text' for c in names if type(c) is not str] + problems += [ + f'the column {c!r} is given twice' for c, n in Counter(names).items() if n > 1 + ] + if KEY in names: + problems.append( + f'a column is named {KEY!r}, the name of the index: make it the ' + f'index with frame.set_index({KEY!r})' + ) + return problems def _cast(column: pd.Series, dtype: str) -> tuple[pd.Series, list[str]]: - """``column`` cast to ``dtype``, and the cells refused, each as a message.""" - if dtype == 'object': - return _objects(column), [] - if dtype != 'string' or all(_empty(v) or isinstance(v, str) for v in column): - try: - return column.astype(dtype), [] - except (TypeError, ValueError): - pass + """ + ``column`` as a column of ``dtype``, an empty cell as ``pd.NA`` or ``None``; + and each cell that is not of its kind, as a message. + """ + values = [None if _empty(v) else v for v in column.tolist()] refused = [ f'row {k!r}, column {column.name!r}: {v!r} is not {_WORDS[dtype]}' - for k, v in zip(column.index, column.tolist(), strict=True) - if not _empty(v) and not _fits(v, dtype) - ] - return column, refused or [ - f'column {column.name!r}: not {_WORDS[dtype]} in each cell' + for k, v in zip(column.index, values, strict=True) + if v is not None and not _fits(v, dtype) ] + if dtype == 'object': + return pd.Series(values, index=column.index, dtype=object), refused + if refused: + return column, refused + return pd.Series(pd.array(values, dtype=dtype), index=column.index), [] def typed(frame: pd.DataFrame, model: type[BaseModel]) -> pd.DataFrame: """ ``frame`` as a batch holds it: the index of text named :data:`KEY`, each - column of a parameter of ``model`` cast to its dtype (see :func:`dtypes`), - and each note to the dtype of ``convert_dtypes()``. An empty cell of an - ``object`` column is ``None``. + column of a parameter of ``model`` of its dtype (see :func:`dtypes`), and + each note of the dtype of ``convert_dtypes()``. - Raises ``ValueError``, naming each key, column or cell refused: a key - that is not text, empty, or given twice; a column given twice; a cell - that is not of the dtype of its column. + Raises ``ValueError``, naming each key, column or cell refused: a key that + is not text, empty, or given twice; a column named :data:`KEY`, not named + by text, or given twice; a note of another dtype than the five of a table; + a cell that is not of the kind of its column. """ index = _keys(frame.index) - problems = [ - f'the column {c!r} is given twice' - for c, n in Counter(frame.columns).items() - if n > 1 - ] - if problems: + if problems := _names(frame.columns): raise ValueError('; '.join(problems)) types = dtypes(model) columns = {} for name, column in frame.set_axis(index).items(): if name in types: - columns[name], refused = _cast(column, types[name]) - problems += refused + dtype = types[name] else: - note = column.convert_dtypes() - columns[name] = _objects(note) if note.dtype == object else note + column = column.convert_dtypes() + dtype = str(column.dtype) + if dtype not in _WORDS: + problems.append( + f'the note {name!r} is {dtype}, not text, a number, true or ' + f'false, or plain data: make it text, with ' + f"frame[{name!r}].astype('string')" + ) + continue + columns[name], refused = _cast(column, dtype) + problems += refused if problems: raise ValueError('; '.join(problems)) - return pd.DataFrame(columns, index=index, columns=frame.columns) + return pd.DataFrame(columns, index=index, columns=list(frame.columns)) def _request_value(parameter: _Parameter, cell: Any) -> Any: @@ -531,7 +572,7 @@ def _request_value(parameter: _Parameter, cell: Any) -> Any: return file(run=cell) if not isinstance(cell, str): return cell - if parameter.file and (not parameter.output or _KIND.match(cell)): + if parameter.file and cell and (not parameter.output or _KIND.match(cell)): return FileRef(file=cell) if parameter.output and (found := _OUTPUT.fullmatch(cell)): return OutputRef(**found.groupdict()) @@ -541,11 +582,11 @@ def _request_value(parameter: _Parameter, cell: Any) -> Any: def as_request(model: type[BaseModel], cells: Mapping[str, Any]) -> dict[str, Any]: """ ``cells``, by the name of a parameter of ``model``, as a request takes - them: at a parameter that takes a file, text is the file of that name and - a whole number a run; at one that takes an output, text such as - ``'r17.centre'`` or ``'r18.slices[2]'`` is that output; at one that takes - either, text with a kind, such as ``run:``, is a file, and other text an - output. In a list, each element is converted so. Any other cell is + them: at a parameter that takes a file, text that is not empty is the file + of that name and a whole number a run; at one that takes an output, text + such as ``'r17.centre'`` or ``'r18.slices[2]'`` is that output; at one that + takes either, text with a kind, such as ``run:``, is a file, and other text + an output. In a list, each element is converted so. Any other cell is returned as it is, for the request to refuse if it does not fit. """ fields = model.model_fields @@ -554,7 +595,7 @@ def as_request(model: type[BaseModel], cells: Mapping[str, Any]) -> dict[str, An if name not in fields: converted[name] = cell continue - parameter = _parameter(fields[name]) + parameter = _parameter(name, fields[name]) if parameter.many and isinstance(cell, list): converted[name] = [_request_value(parameter, c) for c in cell] else: @@ -566,13 +607,14 @@ def cell_of(value: Any) -> Any: """ The cell of ``value`` of a request, the inverse of :func:`as_request`: a file is its name, such as ``'run:1'``, an output its text, such as - ``'r17.centre'``, and a list holds the cell of each element. + ``'r17.centre'``, a list holds the cell of each element, and anything else + is its JSON form, as plain data. """ if isinstance(value, list): return [cell_of(v) for v in value] if isinstance(value, FileRef | OutputRef): return str(value) - return value + return to_jsonable_python(value) def data_of(frame: pd.DataFrame) -> dict[str, Any]: @@ -582,45 +624,50 @@ def data_of(frame: pd.DataFrame) -> dict[str, Any]: with ``None`` in each empty cell. Written as JSON by a writer of Python values, it keeps each value exact. """ - records = [ - {KEY: key, **{c: None if _empty(v) else v for c, v in row.items()}} - for key, row in frame.to_dict('index').items() - ] + if frame.index.name != KEY: + raise ValueError(f'a table has an index named {KEY!r}: see typed()') + records = [{KEY: key, **row} for key, row in frame.to_dict('index').items()] return {'schema': build_table_schema(frame, version=False), 'data': records} def frame_of(data: Any) -> pd.DataFrame: """ The table of the plain data of :func:`data_of`, equal to the frame it was - made of. Raises ``ValueError``, naming the entry, for data that are not - such a table. + made of. Raises ``ValueError``, naming the entry, row or column, for data + that are not such a table, or a cell that is not of the kind of its column. """ try: - fields = { - f['name']: f.get('extDtype', 'object') for f in data['schema']['fields'] - } - records = list(data['data']) - except (KeyError, TypeError, AttributeError): + fields = [ + (f['name'], f.get('extDtype', 'object')) for f in data['schema']['fields'] + ] + records = data['data'] + except (KeyError, TypeError): raise ValueError( - "not a table: {'schema': {'fields': [...]}, 'data': [...]}" + "not a table: {'schema': {'fields': [{'name': ...}, ...]}, 'data': [...]}" ) from None - if fields.pop(KEY, None) is None: + if not isinstance(records, list): + raise ValueError('data: not a list of rows') + names = [n for n, _ in fields if n != KEY] + if len(names) == len(fields): raise ValueError(f'schema: no field {KEY!r}') + if problems := _names(names): + raise ValueError('; '.join(problems)) + types = dict(fields) + if wrong := [ + f'{n!r} is {t}' for n, t in types.items() if t not in _WORDS and n != KEY + ]: + raise ValueError(f'schema: not a dtype of a table: {", ".join(wrong)}') for i, record in enumerate(records): if not isinstance(record, dict) or KEY not in record: raise ValueError(f'data[{i}]: not an object with a {KEY!r}') - if unknown := sorted(record.keys() - fields.keys() - {KEY}): + if unknown := sorted(record.keys() - types.keys()): raise ValueError(f'data[{i}]: no field {", ".join(unknown)} in the schema') index = _keys(pd.Index([r[KEY] for r in records], dtype=object)) columns = {} - for name, dtype in fields.items(): - values = [r.get(name) for r in records] - try: - columns[name] = ( - pd.Series(values, index=index, dtype=object) - if dtype == 'object' - else pd.Series(pd.array(values, dtype=dtype), index=index) - ) - except (TypeError, ValueError) as error: - raise ValueError(f'column {name!r}: {error}') from None - return pd.DataFrame(columns, index=index, columns=list(fields)) + for name in names: + values = pd.Series([r.get(name) for r in records], index=index, dtype=object) + columns[name], refused = _cast(values.rename(name), types[name]) + problems += refused + if problems: + raise ValueError('; '.join(problems)) + return pd.DataFrame(columns, index=index, columns=names) diff --git a/packages/essapps/tests/batch/table_test.py b/packages/essapps/tests/batch/table_test.py index 1a8737f2..4c38a29d 100644 --- a/packages/essapps/tests/batch/table_test.py +++ b/packages/essapps/tests/batch/table_test.py @@ -3,8 +3,10 @@ """The rules of a table of rows, on plain values; the table as a frame.""" import re +from collections.abc import Callable from dataclasses import replace -from datetime import UTC, datetime +from datetime import UTC, date, datetime +from enum import Enum from typing import Any, Literal import pandas as pd @@ -24,6 +26,7 @@ typed, ) from ess.spec import FileRef, OutputRef, Referenced, RunFile +from ess.spec.request_types import request_model from . import toys @@ -399,6 +402,11 @@ def test_a_row_added_by_hand_with_the_key_of_a_row_outside_the_scope_is_refused( # The table as a frame +class Colour(Enum): + RED = 'red' + BLUE = 'blue' + + class Kinds(BaseModel): count: int = 1 flag: bool = False @@ -406,6 +414,8 @@ class Kinds(BaseModel): choice: Literal['a', 'b'] = 'a' level: Literal[1, 2] = 1 when: datetime | None = None + day: date | None = None + colour: Colour = Colour.RED scale: float | None = None either: int | str = 0 centre: Referenced[toys.Offset] | None = None @@ -420,6 +430,8 @@ def test_a_column_has_the_dtype_of_its_parameter() -> None: 'choice': 'string', 'level': 'object', 'when': 'string', + 'day': 'string', + 'colour': 'string', 'scale': 'Float64', 'either': 'object', 'centre': 'string', @@ -438,6 +450,15 @@ def test_the_columns_of_the_toy_have_the_dtypes_of_their_parameters() -> None: assert dtypes(toys.SeriesParams) == {'runs': 'object'} +class Ambiguous(BaseModel): + label: str | Referenced[toys.Offset] + + +def test_a_parameter_of_text_or_an_output_is_refused_by_name() -> None: + with pytest.raises(ValueError, match='label: takes text or an output'): + dtypes(Ambiguous) + + def test_typed_names_the_index_and_casts_parameters_and_notes() -> None: frame = pd.DataFrame( { @@ -470,20 +491,56 @@ def test_typed_names_the_index_and_casts_parameters_and_notes() -> None: [ (['a', 'a'], {}, "the key 'a' is given twice"), (['a', ''], {}, 'a key is empty'), - ([60339], {}, r'not a number such as 60339; .*frame.index.astype\(str\)'), + ([60339.0], {}, '^a key is text, not a number such as 60339.0$'), (['a', 'b'], {'scale': [1.0, 'thick']}, "row 'b', column 'scale': 'thick'"), + (['a'], {'scale': ['2']}, "row 'a', column 'scale': '2' is not a number"), + (['a'], {'scale': [True]}, "row 'a', column 'scale': True is not a number"), (['a', 'b'], {'word': ['run:1', 60339.0]}, "row 'b', column 'word': 60339.0"), (['a'], {'count': [2.5]}, "row 'a', column 'count': 2.5 is not a whole"), + (['a'], {'count': [float('inf')]}, "row 'a', column 'count': inf is not a"), + (['a'], {'count': [2**63]}, f"row 'a', column 'count': {2**63} is not a"), (['a'], {'flag': ['yes']}, "row 'a', column 'flag': 'yes' is not true"), + (['a'], {'flag': [1]}, "row 'a', column 'flag': 1 is not true or false"), + (['a'], {'level': [(1, 2)]}, r"row 'a', column 'level': \(1, 2\) is not plain"), + (['a'], {'key': ['x']}, "a column is named 'key'.*set_index"), + (['a'], {0: ['x']}, 'the column 0 is not text'), + ( + ['a'], + {'when': [datetime(2026, 1, 1, tzinfo=UTC)]}, + r"column 'when': Timestamp.* is not text", + ), ], ) -def test_typed_refuses_a_key_or_a_cell_and_names_it( - index: list[Any], columns: dict[str, list[Any]], message: str +def test_typed_refuses_a_key_a_column_or_a_cell_and_names_it( + index: list[Any], columns: dict[Any, list[Any]], message: str ) -> None: with pytest.raises(ValueError, match=message): typed(pd.DataFrame(columns, index=index), Kinds) +def test_typed_refuses_a_column_given_twice() -> None: + frame = pd.DataFrame([['a', 'b']], columns=['word', 'word'], index=['r']) + + with pytest.raises(ValueError, match="the column 'word' is given twice"): + typed(frame, Kinds) + + +def test_typed_refuses_a_note_of_another_dtype_by_name() -> None: + frame = pd.DataFrame({'start': [pd.Timestamp('2026-11-19')]}, index=['a']) + + with pytest.raises(ValueError, match=r"the note 'start' is datetime64.*text"): + typed(frame, Kinds) + + +def test_a_float_nan_in_a_column_of_numbers_is_an_empty_cell() -> None: + frame = pd.DataFrame({'scale': [float('nan'), 1.0]}, index=['a', 'b']) + + table = typed(frame, Kinds) + + assert table['scale'].isna().tolist() == [True, False] + pd.testing.assert_frame_equal(frame_of(data_of(table)), table) + + @pytest.fixture def exact() -> pd.DataFrame: key = pd.Index(['a', 'b', '0342'], name=KEY, dtype='string') @@ -517,18 +574,28 @@ def test_the_data_of_a_table_without_rows_give_its_columns_and_dtypes( pd.testing.assert_frame_equal(frame_of(data_of(empty)), empty) +def test_the_data_of_a_table_without_columns_give_its_keys() -> None: + table = typed(pd.DataFrame(index=['a', 'b']), Kinds) + + pd.testing.assert_frame_equal(frame_of(data_of(table)), table) + + @pytest.mark.parametrize( ('change', 'message'), [ (lambda d: d.pop('schema'), 'not a table'), - (lambda d: d['data'][1].pop(KEY), "data\\[1\\]: not an object with a 'key'"), - (lambda d: d['data'][2].update(y=1), 'data\\[2\\]: no field y'), - (lambda d: d['data'][0].update(n=2.5), "column 'n'"), + (lambda d: d.update(data={'a': 1}), 'data: not a list of rows'), + (lambda d: d['data'][1].pop(KEY), r"data\[1\]: not an object with a 'key'"), + (lambda d: d['data'][2].update(y=1), r'data\[2\]: no field y'), + (lambda d: d['data'][0].update(n=2.5), "row 'a', column 'n': 2.5 is not"), + (lambda d: d['data'][0].update(s=1.0), "row 'a', column 's': 1.0 is not"), + (lambda d: d['data'][0].update(n=True), "row 'a', column 'n': True is not"), (lambda d: d['data'][1].update(key='a'), "the key 'a' is given twice"), + (lambda d: d['schema']['fields'].append({'name': 'n'}), "'n' is given twice"), ], -) # fmt: skip +) def test_data_that_are_not_a_table_are_refused_by_entry( - exact: pd.DataFrame, change: Any, message: str + exact: pd.DataFrame, change: Callable[[dict[str, Any]], object], message: str ) -> None: data = from_json(to_json(data_of(exact))) change(data) @@ -551,6 +618,7 @@ def output(record: str, name: str, key: str | None = None) -> OutputRef: [ (toys.Params, {'sample': 'run:60339'}, {'sample': FileRef(file='run:60339')}), (toys.Params, {'sample': 'path:a.nxs'}, {'sample': FileRef(file='path:a.nxs')}), + (toys.Params, {'sample': ''}, {'sample': ''}), (toys.Params, {'offset': 'r17.offset'}, {'offset': output('r17', 'offset')}), (toys.Params, {'binning': {'bins': 3}}, {'binning': {'bins': 3}}), (Either, {'centre': 'r18.slices[2]'}, {'centre': output('r18', 'slices', '2')}), @@ -576,3 +644,27 @@ def test_a_whole_number_at_a_file_is_a_run() -> None: assert as_request(toys.SeriesParams, {'runs': [60339, 'run:2']}) == { 'runs': [FileRef(file='run:60339'), FileRef(file='run:2')] } + + +class Masked(BaseModel): + mask: RunFile + + +class Grouped(BaseModel): + masked: Masked + when: datetime + + +def test_the_cell_of_a_group_or_a_time_is_its_json_form_which_a_request_takes() -> None: + values = { + 'masked': Masked.model_construct(mask=FileRef(file='run:1')), + 'when': datetime(2026, 11, 19, 12, tzinfo=UTC), + } + + cells = {name: cell_of(v) for name, v in values.items()} + + assert cells == { + 'masked': {'mask': {'@ref': 'file', 'file': 'run:1'}}, + 'when': '2026-11-19T12:00:00Z', + } + request_model(Grouped).model_validate(as_request(Grouped, cells)) From a2a3549c3365d93487ca502b84eb5e66a4a88ac9 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 19 Nov 2026 12:00:30 +0000 Subject: [PATCH 3/3] An Int64 column takes a float that is a whole number Plain pandas gives whole numbers with a gap as a float column (pd.DataFrame({'count': [1, None]})), so a column of whole numbers takes 1.0 and keeps it as 1. 2.5, inf and a bool stay refused. Co-Authored-By: Claude Fable 5.2 --- docs/developer/adr/0012-a-batch-is-a-table.md | 2 +- packages/essapps/src/ess/apps/batch/table.py | 19 +++++++++++++------ packages/essapps/tests/batch/table_test.py | 11 +++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/developer/adr/0012-a-batch-is-a-table.md b/docs/developer/adr/0012-a-batch-is-a-table.md index 2d36fee2..517e6e1a 100644 --- a/docs/developer/adr/0012-a-batch-is-a-table.md +++ b/docs/developer/adr/0012-a-batch-is-a-table.md @@ -65,7 +65,7 @@ This decides what a batch is when it is made once and then changed by a person. | a truth value | `boolean` | `True` | | a list, a group, a value or an output, anything else | `object` | `['run:1', 'run:2']` | -A cell must be of the kind of its column, and nothing converts it: `'2'` and `True` are not numbers. +A cell must be of the kind of its column, and nothing converts it: `'2'` and `True` are not numbers. One exception: a column of whole numbers takes a float that is a whole number (`1.0` is kept as `1`), since pandas gives whole numbers with a gap as floats. ### A row fills the template diff --git a/packages/essapps/src/ess/apps/batch/table.py b/packages/essapps/src/ess/apps/batch/table.py index a51088d1..4f9ba7bd 100644 --- a/packages/essapps/src/ess/apps/batch/table.py +++ b/packages/essapps/src/ess/apps/batch/table.py @@ -41,11 +41,13 @@ A cell must be of the kind of its column, with no conversion by pandas: ``string`` takes text, ``Float64`` an ``int`` or a ``float``, ``Int64`` an -``int``, ``boolean`` a ``bool``, and ``object`` plain data, the values of JSON -as Python gives them. A note takes the dtype of ``convert_dtypes()``, which -must be one of these five. An empty cell is ``None``, ``NaN`` or ``pd.NA`` as -given; in the table it is ``pd.NA``, or ``None`` in an ``object`` column, and -in a row read with ``to_dict('index')`` and in ``rows.json`` it is ``None``. +``int`` or a whole ``float`` (``1.0`` is kept as ``1``: pandas gives whole +numbers with a gap as floats), ``boolean`` a ``bool``, and ``object`` plain +data, the values of JSON as Python gives them. A note takes the dtype of +``convert_dtypes()``, which must be one of these five. An empty cell is +``None``, ``NaN`` or ``pd.NA`` as given; in the table it is ``pd.NA``, or +``None`` in an ``object`` column, and in a row read with ``to_dict('index')`` +and in ``rows.json`` it is ``None``. A cell is plain data. A file and an output are text, which a request does not take, so :func:`as_request` converts the cells of a row before @@ -474,7 +476,10 @@ def _fits(value: Any, dtype: str) -> bool: if isinstance(value, bool): return dtype == 'boolean' if dtype == 'Int64': - return isinstance(value, int) and -(2**63) <= value < 2**63 + whole = isinstance(value, int) or ( + isinstance(value, float) and value.is_integer() + ) + return whole and -(2**63) <= value < 2**63 kinds = {'string': str, 'Float64': (int, float), 'boolean': bool} return isinstance(value, kinds[dtype]) @@ -528,6 +533,8 @@ def _cast(column: pd.Series, dtype: str) -> tuple[pd.Series, list[str]]: return pd.Series(values, index=column.index, dtype=object), refused if refused: return column, refused + if dtype == 'Int64': # a whole float, as pandas gives a column with a gap + values = [None if v is None else int(v) for v in values] return pd.Series(pd.array(values, dtype=dtype), index=column.index), [] diff --git a/packages/essapps/tests/batch/table_test.py b/packages/essapps/tests/batch/table_test.py index 4c38a29d..844906fd 100644 --- a/packages/essapps/tests/batch/table_test.py +++ b/packages/essapps/tests/batch/table_test.py @@ -498,6 +498,7 @@ def test_typed_names_the_index_and_casts_parameters_and_notes() -> None: (['a', 'b'], {'word': ['run:1', 60339.0]}, "row 'b', column 'word': 60339.0"), (['a'], {'count': [2.5]}, "row 'a', column 'count': 2.5 is not a whole"), (['a'], {'count': [float('inf')]}, "row 'a', column 'count': inf is not a"), + (['a'], {'count': [True]}, "row 'a', column 'count': True is not a whole"), (['a'], {'count': [2**63]}, f"row 'a', column 'count': {2**63} is not a"), (['a'], {'flag': ['yes']}, "row 'a', column 'flag': 'yes' is not true"), (['a'], {'flag': [1]}, "row 'a', column 'flag': 1 is not true or false"), @@ -532,6 +533,16 @@ def test_typed_refuses_a_note_of_another_dtype_by_name() -> None: typed(frame, Kinds) +def test_a_whole_float_in_a_column_of_whole_numbers_is_kept_as_an_int() -> None: + frame = pd.DataFrame({'count': [1, None]}, index=['a', 'b']) + + table = typed(frame, Kinds) + + assert table['count'].dtype == 'Int64' + assert table['count'].tolist() == [1, pd.NA] + assert type(table.to_dict('index')['a']['count']) is int + + def test_a_float_nan_in_a_column_of_numbers_is_an_empty_cell() -> None: frame = pd.DataFrame({'scale': [float('nan'), 1.0]}, index=['a', 'b'])