diff --git a/examples/run/records/x7Ex4mpleA01.extraction.json b/examples/run/records/x7Ex4mpleA01.extraction.json index ee12f17..90c25b1 100644 --- a/examples/run/records/x7Ex4mpleA01.extraction.json +++ b/examples/run/records/x7Ex4mpleA01.extraction.json @@ -2,29 +2,1305 @@ "extraction_metadata": { "extractor_model": "example-model", "extractor_version": "0.0.0-example", - "source_text_hash": null, - "extraction_date": null, - "repaired_by": null, - "paper_sections": null + "source_text_hash": "a44901241d8884e85cfead36b324ccb4dba1b0213436ed2a8d216cb8c3803da5", + "extraction_date": "2026-10-17", + "paper_sections": [ + { + "ordinal": 0, + "title": "Reward anticipation in adolescents: an example paper", + "level": 0, + "start_char": 0, + "end_char": 699 + } + ] }, - "stable_id": null, "local_id": "x7Ex4mpleA01", - "description": null, - "hypothesis": null, - "design": null, - "groups": null, - "tasks": null, - "assessments": null, - "regions": null, - "devices": null, - "acquisitions": null, - "preprocessings": null, - "model_estimations": null, - "inference_settings": null, - "measures": null, - "analyses": null, - "result_reporting": null, - "tables": null, - "coordinate_sets": null, - "external_datasets": null + "description": { + "extraction_status": "extracted", + "value": "Twenty participants performed a monetary incentive delay task.", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Twenty participants performed a monetary incentive delay task.", + "start_char": 115, + "end_char": 177 + } + ] + } + ] + } + }, + "hypothesis": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "design": { + "description": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "allocation": { + "extraction_status": "extracted", + "value": "not_applicable", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Twenty participants performed a monetary incentive delay task.", + "start_char": 115, + "end_char": 177 + } + ] + } + ] + } + }, + "assignment_structure": { + "extraction_status": "extracted", + "value": "single_group", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Twenty participants performed a monetary incentive delay task.", + "start_char": 115, + "end_char": 177 + } + ] + } + ] + } + }, + "blinding": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "arms": [], + "timepoints": [] + }, + "groups": [ + { + "local_id": "grp_participants", + "name": { + "extraction_status": "extracted", + "value": "participants", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Twenty participants", + "start_char": 115, + "end_char": 134 + } + ] + } + ] + } + }, + "species": { + "extraction_status": "extracted", + "value": "human", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "adolescents", + "start_char": 23, + "end_char": 34 + } + ] + } + ] + } + }, + "population_characteristics": { + "extraction_status": "extracted", + "value": [ + "adolescents" + ], + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "adolescents", + "start_char": 23, + "end_char": 34 + } + ] + } + ] + } + }, + "enrolled_count": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "age_mean": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "age_standard_deviation": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "age_minimum": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "age_maximum": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "is_healthy": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + } + } + ], + "tasks": [ + { + "local_id": "tsk_monetary_incentive_delay", + "name": { + "extraction_status": "extracted", + "value": "monetary incentive delay task", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "monetary incentive delay task", + "start_char": 147, + "end_char": 176 + } + ] + } + ] + } + }, + "conditions": [ + { + "local_id": "cnd_gain", + "name": { + "extraction_status": "extracted", + "value": "Gain", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain", + "start_char": 408, + "end_char": 412 + } + ] + } + ] + } + } + }, + { + "local_id": "cnd_loss", + "name": { + "extraction_status": "extracted", + "value": "Loss", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss", + "start_char": 424, + "end_char": 428 + } + ] + } + ] + } + } + }, + { + "local_id": "cnd_neutral", + "name": { + "extraction_status": "extracted", + "value": "Neutral", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Neutral", + "start_char": 415, + "end_char": 422 + } + ] + } + ] + } + } + } + ], + "design_type": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + } + } + ], + "assessments": [], + "regions": [], + "devices": [], + "acquisitions": [], + "preprocessings": [], + "model_estimations": [ + { + "local_id": "mod_condition", + "model_family": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "model_type": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "stage": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "terms": [ + { + "local_id": "trm_condition", + "name": { + "extraction_status": "extracted", + "value": "task condition", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain > Neutral", + "start_char": 408, + "end_char": 422 + }, + { + "text": "Loss > Neutral", + "start_char": 424, + "end_char": 438 + } + ] + } + ] + } + }, + "type": { + "extraction_status": "extracted", + "value": "categorical", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain > Neutral", + "start_char": 408, + "end_char": 422 + }, + { + "text": "Loss > Neutral", + "start_char": 424, + "end_char": 438 + } + ] + } + ] + } + }, + "variation_level": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "levels": [ + { + "level": { + "extraction_status": "extracted", + "value": "Gain", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain", + "start_char": 408, + "end_char": 412 + } + ] + } + ] + } + }, + "conditions": [ + "cnd_gain" + ] + }, + { + "level": { + "extraction_status": "extracted", + "value": "Loss", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss", + "start_char": 424, + "end_char": 428 + } + ] + } + ] + } + }, + "conditions": [ + "cnd_loss" + ] + }, + { + "level": { + "extraction_status": "extracted", + "value": "Neutral", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Neutral", + "start_char": 415, + "end_char": 422 + } + ] + } + ] + } + }, + "conditions": [ + "cnd_neutral" + ] + } + ] + } + ] + } + ], + "inference_settings": [ + { + "local_id": "inf_cluster_fwe", + "inference_level": { + "extraction_status": "extracted", + "value": "cluster", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "clusters were family-wise error corrected at p < 0.05", + "start_char": 277, + "end_char": 330 + } + ] + } + ] + } + }, + "height_threshold_value": { + "extraction_status": "extracted", + "value": 0.001, + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "p < 0.001 uncorrected", + "start_char": 214, + "end_char": 235 + } + ] + } + ] + } + }, + "height_threshold_type": { + "extraction_status": "extracted", + "value": "p", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "p < 0.001 uncorrected", + "start_char": 214, + "end_char": 235 + } + ] + } + ] + } + }, + "cluster_extent_threshold": { + "extraction_status": "extracted", + "value": 20, + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "cluster extent of 20 voxels", + "start_char": 244, + "end_char": 271 + } + ] + } + ] + } + }, + "clusterwise_threshold_value": { + "extraction_status": "extracted", + "value": 0.05, + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "family-wise error corrected at p < 0.05", + "start_char": 291, + "end_char": 330 + } + ] + } + ] + } + }, + "multiple_comparison_method": { + "extraction_status": "extracted", + "value": "family-wise error", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "family-wise error corrected", + "start_char": 291, + "end_char": 318 + } + ] + } + ] + } + }, + "correction_scope": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + } + } + ], + "measures": [ + { + "local_id": "mea_activation", + "family": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "type": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + } + } + ], + "analyses": [ + { + "local_id": "ana_gain_neutral", + "name": { + "extraction_status": "extracted", + "value": "Gain > Neutral", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain > Neutral", + "start_char": 408, + "end_char": 422 + } + ] + } + ] + } + }, + "definition": { + "extraction_status": "extracted", + "value": "Gain > Neutral", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain > Neutral", + "start_char": 408, + "end_char": 422 + } + ] + } + ] + } + }, + "prespecification": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "outcome": { + "extraction_status": "extracted", + "value": "significant_effect", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Table 1 lists the peaks for Gain > Neutral.", + "start_char": 380, + "end_char": 423 + } + ] + } + ] + } + }, + "interpretations": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "spatial_scope": { + "extraction_status": "extracted", + "value": "whole_brain", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Images were thresholded at p < 0.001 uncorrected, with a cluster extent of 20 voxels, and clusters were family-wise error corrected at p < 0.05.", + "start_char": 187, + "end_char": 331 + } + ] + } + ] + } + }, + "measure": "mea_activation", + "groups": [ + { + "group": "grp_participants", + "n": { + "extraction_status": "extracted", + "value": 20, + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Twenty participants", + "start_char": 115, + "end_char": 134 + } + ] + } + ] + } + } + } + ], + "effect": { + "cells": [ + { + "term": "trm_condition", + "level": { + "extraction_status": "extracted", + "value": "Gain", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain", + "start_char": 408, + "end_char": 412 + } + ] + } + ] + } + }, + "direction": { + "extraction_status": "extracted", + "value": "positive", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain > Neutral", + "start_char": 408, + "end_char": 422 + } + ] + } + ] + } + } + }, + { + "term": "trm_condition", + "level": { + "extraction_status": "extracted", + "value": "Neutral", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Neutral", + "start_char": 415, + "end_char": 422 + } + ] + } + ] + } + }, + "direction": { + "extraction_status": "extracted", + "value": "negative", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain > Neutral", + "start_char": 408, + "end_char": 422 + } + ] + } + ] + } + } + } + ], + "statistic": { + "family": { + "extraction_status": "extracted", + "value": "t", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Region | x | y | z | t", + "start_char": 531, + "end_char": 553 + } + ] + } + ] + } + } + } + }, + "details": { + "details_type": "MassUnivariateDetails" + }, + "tasks": [ + "tsk_monetary_incentive_delay" + ], + "model_estimation": "mod_condition", + "inference_settings": "inf_cluster_fwe", + "coordinate_space": { + "extraction_status": "extracted", + "value": "MNI", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Coordinates are reported in MNI space.", + "start_char": 332, + "end_char": 370 + } + ] + } + ] + } + }, + "tables": [ + "tbl1" + ] + }, + { + "local_id": "ana_loss_neutral", + "name": { + "extraction_status": "extracted", + "value": "Loss > Neutral", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss > Neutral", + "start_char": 424, + "end_char": 438 + } + ] + } + ] + } + }, + "definition": { + "extraction_status": "extracted", + "value": "Loss > Neutral", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss > Neutral", + "start_char": 424, + "end_char": 438 + } + ] + } + ] + } + }, + "prespecification": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + }, + "outcome": { + "extraction_status": "extracted", + "value": "no_significant_effect", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss > Neutral produced no significant activation.", + "start_char": 424, + "end_char": 474 + } + ] + } + ] + } + }, + "interpretations": { + "extraction_status": "extracted", + "value": [ + "Loss > Neutral produced no significant activation." + ], + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss > Neutral produced no significant activation.", + "start_char": 424, + "end_char": 474 + } + ] + } + ] + } + }, + "spatial_scope": { + "extraction_status": "extracted", + "value": "whole_brain", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Images were thresholded at p < 0.001 uncorrected, with a cluster extent of 20 voxels, and clusters were family-wise error corrected at p < 0.05.", + "start_char": 187, + "end_char": 331 + } + ] + } + ] + } + }, + "measure": "mea_activation", + "groups": [ + { + "group": "grp_participants", + "n": { + "extraction_status": "extracted", + "value": 20, + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Twenty participants", + "start_char": 115, + "end_char": 134 + } + ] + } + ] + } + } + } + ], + "effect": { + "cells": [ + { + "term": "trm_condition", + "level": { + "extraction_status": "extracted", + "value": "Loss", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss", + "start_char": 424, + "end_char": 428 + } + ] + } + ] + } + }, + "direction": { + "extraction_status": "extracted", + "value": "positive", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss > Neutral", + "start_char": 424, + "end_char": 438 + } + ] + } + ] + } + } + }, + { + "term": "trm_condition", + "level": { + "extraction_status": "extracted", + "value": "Neutral", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Neutral", + "start_char": 415, + "end_char": 422 + } + ] + } + ] + } + }, + "direction": { + "extraction_status": "extracted", + "value": "negative", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Loss > Neutral", + "start_char": 424, + "end_char": 438 + } + ] + } + ] + } + } + } + ], + "statistic": { + "family": { + "extraction_status": "not_reported", + "evidence": { + "status": "not_applicable" + } + } + } + }, + "details": { + "details_type": "MassUnivariateDetails" + }, + "tasks": [ + "tsk_monetary_incentive_delay" + ], + "model_estimation": "mod_condition", + "inference_settings": "inf_cluster_fwe", + "coordinate_space": { + "extraction_status": "extracted", + "value": "MNI", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Coordinates are reported in MNI space.", + "start_char": 332, + "end_char": 370 + } + ] + } + ] + } + } + } + ], + "result_reporting": { + "extraction_status": "extracted", + "value": "analyses_reported", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Table 1 lists the peaks for Gain > Neutral.", + "start_char": 380, + "end_char": 423 + } + ] + } + ] + } + }, + "tables": [ + { + "local_id": "tbl1", + "table_number": { + "extraction_status": "extracted", + "value": "1", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Table 1. Gain > Neutral", + "start_char": 476, + "end_char": 499 + } + ] + } + ] + } + }, + "caption": { + "extraction_status": "extracted", + "value": "Gain > Neutral anticipation, MNI coordinates.", + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Gain > Neutral anticipation, MNI coordinates.", + "start_char": 485, + "end_char": 530 + } + ] + } + ] + } + }, + "column_headings": { + "extraction_status": "extracted", + "value": [ + "Region", + "x", + "y", + "z", + "t" + ], + "value_source": "reported", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Region | x | y | z | t", + "start_char": 531, + "end_char": 553 + } + ] + } + ] + } + }, + "purpose": { + "extraction_status": "extracted", + "value": "reported_effect", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Table 1 lists the peaks for Gain > Neutral.", + "start_char": 380, + "end_char": 423 + } + ] + } + ] + } + } + } + ], + "coordinate_sets": [ + { + "local_id": "cs_tbl1", + "role": { + "extraction_status": "extracted", + "value": "result", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Table 1 lists the peaks for Gain > Neutral.", + "start_char": 380, + "end_char": 423 + } + ] + } + ] + } + }, + "result_kind": { + "extraction_status": "extracted", + "value": "contrast", + "value_source": "generated", + "evidence": { + "status": "present", + "sets": [ + { + "source": "model_quote", + "spans": [ + { + "text": "Table 1 lists the peaks for Gain > Neutral.", + "start_char": 380, + "end_char": 423 + } + ] + } + ] + } + }, + "analysis": "ana_gain_neutral" + } + ], + "external_datasets": [] } diff --git a/schema-tutorial.md b/schema-tutorial.md index 3c17edb..0a50195 100644 --- a/schema-tutorial.md +++ b/schema-tutorial.md @@ -1626,7 +1626,7 @@ same order is what stops you inventing entities to satisfy a cell. on each level; `inputs_from` where the source describes a lower stage (chapter 8). 5. **Analyses.** One per distinct tested effect. For each: cells, statistic, payload, measure, cohorts and n, tables, scope and regions (chapters 9–10). -6. **Tables.** Every table either named by an analysis or marked with `non_analysis_content` (§10.6). +6. **Tables.** Every table either named by an analysis or given a `purpose` that says what its rows are (§10.6). Two habits worth forming. When a step needs an entity that does not exist, **go back and declare it** rather than writing the fact in prose — an entity named only in an `Analysis.definition` is invisible @@ -1635,47 +1635,66 @@ change if a different contrast came off the same model? ### 12.3 What the validator checks -`pondie.extraction.record.validate` (with the rules in `pondie/extraction/record/rules.py`) runs three tiers over a finished extraction record. Errors mean the record -is wrong; warnings mean a reviewer has to look. - -**Tier 1 — conformance (errors).** Types, required slots, closed vocabularies, the evidence -invariants of §2.3, span offsets against the normalized text, and the source-text hash. Plus the -class `rules` the projection dropped — the two on `Analysis` and the non-empty `cells` rule on -`Effect`. - -**Tier 2 — structure (errors).** The invariants LinkML cannot express because they span a multivalued -nested slot or a sibling class: +`python -m pondie.extraction.record.validate --record .extraction.json --text ` runs two +modules over a finished record: `validate.py` (conformance to the schema) and `rules.py` (nineteen +rules about what a record means). Each finding is an **error**, meaning the record is wrong, or a +**warning**, meaning a reviewer has to look. The command exits non-zero on any error and prints +warnings without failing. Without `--text` the span offsets are not checked. + +**Conformance (`validate.py`).** Every key is a slot the class declares, every required slot is +present, and every value has the declared type and the single-or-list shape of its slot. A +null is not an absent slot: an unknown is a `not_reported` wrapper or an omitted key. Also: + +- **Closed vocabularies are errors; open ones are warnings.** A value outside a closed enum fails; + a value outside an open one (`any_of: [, string]`) is kept as free text with a warning. + `unstated` is refused on either, because missingness has one encoding (§11.1). +- **The evidence invariants of §2.3.** `not_reported` carries no `value` and `evidence.status: + not_applicable`; `extracted` carries no `unreported_reason`; `present` has at least one set, each + with at least one span. +- **Span offsets** (with `--text`): each span's `text` equals the normalized source at + `[start_char, end_char)`. The `source_text_hash`, when the record states one, must be the hash of + that text. +- **Identifiers and references.** A `local_id` is a non-empty string. A reference to an id nothing + declares is a warning; one that resolves to the wrong class is an error. +- **Class `rules`.** The storage schema's rules, which the projection drops, are evaluated here: + whole-brain and searchlight scopes name no region and an ROI scope names its regions (on + `Analysis` and again on `InferenceSettings.correction_scope`), the non-empty `cells` rule on + `Effect`, and the rules on `Region`, `Bibliography` and `License`. + +**Rules that are errors (`rules.py`).** Invariants LinkML cannot express because they span a +multivalued nested slot or a sibling class: | Check | Catches | |---|---| -| `Cell.term` in scope | A cell naming no term, or a term of a model this analysis does not reach through `inputs_from` | -| `Cell.level` agrees with its term | A level the term does not declare — the string join the mapper depends on | -| `Effect.mediation.mediator` in scope | The same, for the one other term pointer | -| `inputs_from` acyclic | A model fitted on its own output; a hang as well as a falsehood | -| term names unique across a chain | A first-level `motion` indistinguishable from a group-level one | +| `effect_kind` | An `Effect.kind` that disagrees with the kind the cells derive (§9.6), or cells that derive no kind | +| `cell_terms` | A cell naming no term, or a term of a model this analysis does not reach through `inputs_from`; a `level` the term does not declare; the same for `Effect.mediation.mediator` | +| `cell_level_polarity` | A cell whose `level` and `direction` state opposite signs of one effect | +| `model_stages` | An `inputs_from` cycle; a term name repeated across a chain | | references resolve | A dangling `Analysis.measure`, `Analysis.inference_settings`, `Acquisition.device` or `Group.diagnostic_instrument` | -| table purpose | A table marked non-analysis that an analysis nonetheless names | +| `table_purpose` | A table whose `purpose` says it is not a reported effect that an analysis nonetheless names | -One invariant is stated on the field it constrains and is **not** checked here: the `is_healthy` -contradiction of §4.2, which a record audit in the review layer catches instead. - -**Tier 3 — prose-triggered review (warnings).** These read a name or a definition, so they route a -record to review rather than rejecting it. Each catches a defect that is invisible to tiers 1 and 2 — -every cell resolves, every level agrees, and the record is structurally perfect while the comparison -the paper reported has gone missing. +**Rules that are warnings (`rules.py`).** These read a name or a definition, so they route a +record to review rather than rejecting it. Each catches a defect invisible to conformance and to +the errors above: every cell resolves, every level agrees, and the record is structurally perfect +while the comparison the paper reported has gone missing. | Check | The defect | |---|---| -| crossings | Prose names an interaction that the cells do not record: no product column and fewer than two crossed terms. Or two analyses with identical cells on one model whose prose disagrees about what was tested | -| product columns | A product column whose components are outside its chain, or that no cell anywhere names | -| unsigned cells | `held` on a cell with no level, or on *every* declared level of a factor | -| occasion factors | A continuous term named after a comparison; or several timepoints declared, analyses reporting change over time, and no `FactorLevel.timepoints` naming any | -| arm reachability | Prose names an arm that neither `FactorLevel.arms` nor `Group.arm` reaches (§3.5) | -| derived columns | A derived column with no `source_definition`, or none naming its `assessment` | -| table purpose | A table no analysis names and that carries no `non_analysis_content` | +| `crossings` | Prose names an interaction that the cells do not record: no product column and fewer than two crossed terms. Or two analyses with identical cells on one model whose prose disagrees about what was tested | +| `product_columns` | A product column whose components are outside its chain, or that no cell anywhere names | +| `unsigned_cells` | `held` on a cell with no level, or on *every* declared level of a factor | +| `occasion_factors` | A continuous term named after a comparison; or several timepoints declared, analyses reporting change over time, and no `FactorLevel.timepoints` naming any | +| `arm_reachability` | Prose names an arm that neither `FactorLevel.arms` nor `Group.arm` reaches (§3.5) | +| `derived_columns` | A derived column with no `source_definition`, or none naming its `assessment` | +| `table_purpose` | A table no analysis names and whose `purpose` is empty, so nothing says whether it was left out on purpose | +| `one_protocol_per_acquisition` | Several echo times and several repetition times on one acquisition: two protocols in one entity | +| `acquisition_subclass` | A modality whose parameters have no class to live in | +| `modality_measures` | A measure the acquisition's modality could not have produced | +| `value_source_honesty` | `value_source: reported` on a value with no supporting sentence; say `generated` instead | +| `counts_add_up` | A `CategoryDistribution` that does not sum to its denominator, or an enrolment funnel that increases | -A record is also routed to human review by count: `not_reported` fields per record, required fields -left empty, a failing structural check, and any value outside a vocabulary. +One invariant is stated on the field it constrains and is **not** checked here: the `is_healthy` +contradiction of §4.2, which a record audit in the review layer catches instead. ### 12.4 Reading a record back out @@ -1739,7 +1758,7 @@ Worth knowing before you go hunting for a slot. The full list with evidence is i 2. Every covariate the paper mentions appears in some stage's `terms`. 3. No two analyses with different design matrices share a `ModelEstimation`. 4. Every declared `Timepoint` and `Arm` is reached by a `FactorLevel` or a `Group.arm`. -5. Every table is either named by an analysis or marked with `non_analysis_content`. +5. Every table is either named by an analysis or has a `purpose` that says what its rows are. --- diff --git a/tests/test_example_record.py b/tests/test_example_record.py new file mode 100644 index 0000000..3d3d693 --- /dev/null +++ b/tests/test_example_record.py @@ -0,0 +1,126 @@ +"""The worked example record is a finished extraction of the example paper's text. + +The models and the JSON Schema accept a record whose every slot is null, so +`test_examples_validate` alone cannot tell a finished record from the skeleton. These +tests read the record against the text it was extracted from, which only pondie's +validator does otherwise: evidence invariants, span offsets, source hash, references. +Rules that read across classes (a cell's level against its term's levels, the effect +kind a set of cells derives) stay with pondie's record rules. +""" + +import hashlib +import json + +import pytest + +from conftest import load +from study_schema.models import extraction +from study_schema.models.paper_parse import CoordinateParse + +# Slots whose value is the local_id of an entity declared elsewhere in the record. +REFERENCE_SLOTS = { + "analysis", "conditions", "group", "inference_settings", "measure", + "model_estimation", "tables", "tasks", "term", +} + + +@pytest.fixture +def record(example_paths): + return load(example_paths["record"]) + + +@pytest.fixture +def text(example_paths): + return (example_paths["bundle"] / "processed" / "pubget" / "text.txt").read_text(encoding="utf-8") + + +def walk(node, path="record"): + yield path, node + if isinstance(node, dict): + for key, value in node.items(): + yield from walk(value, f"{path}.{key}") + elif isinstance(node, list): + for index, value in enumerate(node): + yield from walk(value, f"{path}[{index}]") + + +def wrappers(record): + return [(p, n) for p, n in walk(record) if isinstance(n, dict) and "extraction_status" in n] + + +def test_no_slot_is_null(record): + # An unknown is `extraction_status: not_reported` or an absent key, never null. + assert [p for p, n in walk(record) if n is None] == [] + + +def test_every_entity_list_is_filled_or_declared_empty(record): + assert set(record) >= set(extraction.Study.model_fields) - {"stable_id"} + assert record["analyses"] and record["tables"] and record["groups"] + + +def test_not_reported_and_extracted_carry_what_the_status_requires(record): + found = wrappers(record) + assert found + for path, node in found: + evidence = node["evidence"] + if node["extraction_status"] == "not_reported": + assert "value" not in node and evidence == {"status": "not_applicable"}, path + else: + assert node["value_source"] in {"reported", "generated"}, path + assert evidence["status"] == "present" and evidence["sets"], path + assert "unreported_reason" not in node, path + + +def test_every_span_is_the_text_at_its_offsets(record, text): + spans = [n for p, n in walk(record) if p.endswith("]") and isinstance(n, dict) and set(n) == {"text", "start_char", "end_char"}] + assert len(spans) > 20 + for span in spans: + assert text[span["start_char"]:span["end_char"]] == span["text"] + + +def test_a_reported_value_is_quoted_in_its_own_evidence(record): + for path, node in wrappers(record): + if node.get("value_source") != "reported" or not isinstance(node["value"], str): + continue + quoted = " ".join(s["text"] for e in node["evidence"]["sets"] for s in e["spans"]) + assert node["value"].lower() in quoted.lower(), path + + +def test_source_text_hash_is_the_text_the_parse_hashes(record, text, example_paths): + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + parse = CoordinateParse.model_validate(load(example_paths["coordinate_parse"])) + assert record["extraction_metadata"]["source_text_hash"] == digest == parse.text_sha256 + + +def test_every_reference_names_a_declared_entity(record): + declared = {n["local_id"] for _, n in walk(record) if isinstance(n, dict) and "local_id" in n} + for path, node in walk(record): + if not isinstance(node, dict): + continue + for slot in REFERENCE_SLOTS & set(node): + value = node[slot] + for ref in value if isinstance(value, list) else [value]: + if isinstance(ref, str): + assert ref in declared, f"{path}.{slot} -> {ref}" + + +def test_each_parsed_analysis_is_an_analysis_of_the_record(record, example_paths): + parse = CoordinateParse.model_validate(load(example_paths["coordinate_parse"])) + names = {a["name"]["value"] for a in record["analyses"]} + assert {a.name for a in parse.analyses} <= names + assert {a.table_id for a in parse.analyses} <= {t["local_id"] for t in record["tables"]} + + +def test_a_null_result_is_an_analysis_with_no_table(record): + outcomes = {a["name"]["value"]: a["outcome"]["value"] for a in record["analyses"]} + assert outcomes == {"Gain > Neutral": "significant_effect", "Loss > Neutral": "no_significant_effect"} + loss = next(a for a in record["analyses"] if a["name"]["value"] == "Loss > Neutral") + assert "tables" not in loss + + +def test_each_cell_names_a_level_its_term_declares(record): + terms = {t["local_id"]: {lv["level"]["value"] for lv in t["levels"]} + for m in record["model_estimations"] for t in m["terms"]} + for analysis in record["analyses"]: + for cell in analysis["effect"]["cells"]: + assert cell["level"]["value"] in terms[cell["term"]] diff --git a/tests/test_stable_id.py b/tests/test_stable_id.py index 519cc56..fb7afa3 100644 --- a/tests/test_stable_id.py +++ b/tests/test_stable_id.py @@ -32,7 +32,7 @@ def test_record_with_stable_id_on_its_entity_validates(example_paths): def test_record_without_stable_id_still_validates(example_paths): validator = pytest.importorskip("jsonschema", reason="optional: JSON Schema validation") record = load(example_paths["record"]) - record.pop("stable_id") + record.pop("stable_id", None) extraction.Study.model_validate(record) validator.validate(record, jsonschema.load("extraction-record"))