Report MFE representatives for ensemble predictions

ChangeLog

@@ -14,6 +14,10 @@
 
 ## Interface and handling
 
+- report a representative minimum-free-energy structure and its energy terms
+  for every ensemble-selected site; expose and sort by the retained `Eens`
+  value in native and browser output (#319)
+
 - select and visualize the first rank-sorted CopomuS mutation automatically
   when browser results arrive (#317)
 
@@ -348,6 +352,23 @@ energies, restricted partition sums, and trackers.
 ################################################################################  
 ################################################################################
 
+261104 Alexander Mitrofanov
+ * src/IntaRNA/{Interaction,OutputConstraint,OutputHandlerMfe,OutputHandlerCsv,
+   OutputHandlerText,OutputHandlerSeedCandidates}.{h,cpp}, src/bin/
+   * Retain the ensemble energy used for model-P ranking and trace the
+     representative minimum-free-energy interaction only after final output
+     selection, including seeded and explicit-seed predictions.
+   + Add the Eens CSV column, make model-P CSV output include and sort by it by
+     default, and report both MFE and ensemble energies in text output.
+ * gui/{app.js,columns.js,options.js,parameter-file.js,protocol.js,results.js},
+   gui/tests/, scripts/wasm/test-browser.py
+   * Offer Eens only for ensemble predictions, use it for default browser
+     sorting and pair-relative energy differences, and preserve the MFE energy
+     and structure columns for visualization.
+ * README.md, tests/
+   * Document and test ensemble-site ranking with final MFE traceback (issue
+     #319).
+
 261027 Alexander Mitrofanov
  * gui/copomus.js, scripts/wasm/test-browser.py
    * Select the first rank-sorted CopomuS mutation after populating the browser

README.md

@@ -666,12 +666,16 @@ for the given site `S` multiplied with the Boltzmann weights of the respective
 accessibility penalties `ED1` and `ED2`.
 Here, `R` denotes the gas constant and `T` the temperature of the system.
 At `T=37` degree Celsius, the product `RT` is about `RT=0.6163173043012`. 
-The site's partition function can be 
+The site's partition function can be
 used to compute the ensemble energy of all interactions of a given site via
-`E(S) = -RT log(Z(S))` (`log` = natural logarithm), which is reported for the optimal site 
-(see [CSV output](#outModeCsv)).  
-This abstracts from individual inter-molecular base pairing and incorporates the 
-dynamics and flexibility of the interactions formed by two regions.
+`Eens(S) = -RT log(Z(S))` (`log` = natural logarithm). IntaRNA selects and ranks
+sites by `Eens`, then traces the minimum-free-energy interaction within each
+finally reported site. Thus, the reported base pairs and `E` energy terms
+describe a concrete representative structure, while the `Eens` CSV column and
+the additional line in text output retain the site's ensemble energy. The MFE
+traceback is performed only for interactions that reach final output (see
+[CSV output](#outModeCsv)). This combines the dynamics and flexibility captured
+by the site ensemble with a representative inter-molecular base-pair pattern.
 
 The overall partition function is given by
 `Zall = sum Z(S)`, i.e. summation over all possible sites,
@@ -1582,7 +1586,8 @@ are
 - `hybridDB` : hybrid in dot-bar notation (interactin sites only)
 - `hybridDBfull` : hybrid in dot-bar notation (full sequence length)
 - `bpList` : list of hybrid base pairs, e.g. '(4,3):(5,2):(7,1)'
-- `E` : overall interaction energy = E_hybrid + ED1 + ED2
+- `E` : minimum free energy of the reported interaction structure = E_hybrid + ED1 + ED2
+- `Eens` : ensemble energy used to select and rank a `--model=P` interaction site
 - `ED1` : ED value of seq1
 - `ED2` : ED value of seq2
 - `Pu1` : probability to be accessible for seq1
@@ -1628,6 +1633,10 @@ to the RNAplfold values from which the ED values are derived from!
 
 Using `--outCsvCols '*'`, all available columns are added to the output.
 
+For `--model=P`, the default CSV columns additionally contain `Eens`, and rows
+are sorted by `Eens` unless `--outCsvSort` is set explicitly. For other models,
+`Eens` is not available and is printed as `NAN` when requested.
+
 Energies are provided in unit *kcal/mol*, probabilities in the interval [0,1].
 Position annotations start indexing with 1 (or the values set via `--qIdxPos0` and `--tIdxPos0`).
 

gui/app.js

@@ -157,9 +157,10 @@ csvBody.append(element('h3', 'CSV output columns', 'subheading'));
 csvBody.append(element('p', 'seq1 is the target; seq2 is the query. Hover over a column name for details. Select columns for the table and CSV download; table visibility can be changed after computation.', 'input-help'));
 const selectionTools = element('div', '', 'selection-tools'); csvBody.append(selectionTools);
 const selectColumns = (selection) => {
-  csvOrder = [...selection]; for (const input of csvInputs) input.checked = selection.includes(input.value); inputsChanged();
+  const available = selection.filter((key) => !csvInputs.find((input) => input.value === key)?.disabled);
+  csvOrder = [...available]; for (const input of csvInputs) input.checked = available.includes(input.value); inputsChanged();
 };
-for (const [name, selection] of [['Select all', () => CSV_CATALOG.map((c) => c.key)], ['Task defaults', () => defaultColumns(getTask())], ['Clear selection', () => []]]) {
+for (const [name, selection] of [['Select all', () => CSV_CATALOG.map((c) => c.key)], ['Task defaults', () => defaultColumns(getTask(), byId('model').value)], ['Clear selection', () => []]]) {
   const button = element('button', name); button.type = 'button';
   button.addEventListener('click', () => selectColumns(selection())); selectionTools.append(button);
 }
@@ -170,7 +171,7 @@ for (const group of CSV_GROUPS) {
     const button = element('button', name, 'text-button'); button.type = 'button';
     button.setAttribute('aria-label', `${name} ${group.label} columns`);
     button.addEventListener('click', () => {
-      for (const input of csvInputs.filter((input) => group.keys.includes(input.value))) input.checked = checked;
+      for (const input of csvInputs.filter((input) => group.keys.includes(input.value) && !input.disabled)) input.checked = checked;
       inputsChanged();
     }); legend.append(button);
   }
@@ -203,7 +204,7 @@ function updateLength(name) {
     byId(`${name}-length`).textContent = `${parsed.records.length} record${parsed.records.length === 1 ? '' : 's'}, ${parsed.records.reduce((sum, r) => sum + r.sequence.length, 0).toLocaleString()} nucleotides`;
   } catch { byId(`${name}-length`).textContent = 'Check sequence format and length'; }
 }
-function updateAvailability() {
+function updateAvailability({ preserveCsv = false } = {}) {
   const evaluation = getTask() === 'evaluation', mutation = getTask() === 'mutation';
   for (const button of byId('example-options').querySelectorAll('[data-example-task]')) {
     button.hidden = button.dataset.exampleTask !== getTask();
@@ -239,6 +240,25 @@ function updateAvailability() {
   // Upload controls follow the text area they populate, including async reads.
   for (const upload of form.querySelectorAll('input[data-for]')) upload.disabled = byId(upload.dataset.for).disabled;
   for (const input of outputInputs) { input.disabled = mutation || evaluation && EXTRA_OUTPUTS.find((entry) => entry.key === input.value).prediction; if (input.disabled) input.checked = false; }
+  // A site ensemble energy exists only for model-P predictions. On first
+  // entering that model, select and sort by Eens by default; later user edits
+  // remain untouched until the model becomes unavailable again.
+  const eens = csvInputs.find((input) => input.value === 'Eens');
+  const ensemble = !evaluation && !mutation && byId('model').value === 'P';
+  const becomingAvailable = eens.disabled && ensemble;
+  eens.disabled = !ensemble;
+  eens.parentElement.hidden = !ensemble;
+  const eensSort = [...byId('outCsvSort').options].find((option) => option.value === 'Eens');
+  eensSort.disabled = !ensemble;
+  if (!ensemble) {
+    eens.checked = false;
+    csvOrder = csvOrder.filter((key) => key !== 'Eens');
+    if (byId('outCsvSort').value === 'Eens') byId('outCsvSort').value = '';
+  } else if (becomingAvailable && !preserveCsv) {
+    eens.checked = true;
+    if (!csvOrder.includes('Eens')) csvOrder.splice(csvOrder.indexOf('E')+1,0,'Eens');
+    byId('outCsvSort').value = 'Eens';
+  }
   const explicitMutations = !!byId('copomus-mutation-encoding').value.trim();
   for (const input of [byId('copomus-candidate-selection'), byId('copomus-generator'), ...copomusFilterInputs]) input.disabled = !mutation || explicitMutations;
   for (const input of [...copomusMeasureInputs, byId('copomus-alpha'), byId('copomus-beta'), byId('copomus-mutation-encoding')]) input.disabled = !mutation;
@@ -251,7 +271,7 @@ function resetParameters({ clearInputs = false } = {}) {
     for (const input of form.querySelectorAll('input[type=file]')) input.value = '';
     byId('examples-menu').open = false; activateGroup('sequence-input');
   }
-  csvOrder = getTask() === 'mutation' ? CSV_COLUMNS.split(',') : defaultColumns(getTask());
+  csvOrder = getTask() === 'mutation' ? CSV_COLUMNS.split(',') : defaultColumns(getTask(), UI_DEFAULTS.model);
   byId('seed-mode').value = 'identify';
   for (const field of FIELDS) { const input = byId(field.key); if (field.type === 'checkbox') input.checked = UI_DEFAULTS[field.key]; else input.value = UI_DEFAULTS[field.key]; }
   syncAccessibilityFlags();
@@ -306,7 +326,7 @@ function applyParameters(request, message = 'Parameters loaded. Compute to analy
   byId('copomus-generator').value = copomus.generator;
   byId('copomus-mutation-encoding').value = copomus.mutationEncoding;
   byId('copomus-alpha').value = copomus.alpha; byId('copomus-beta').value = copomus.beta;
-  updateAvailability(); activateGroup('sequence-input'); inputsChanged(message);
+  updateAvailability({ preserveCsv: true }); activateGroup('sequence-input'); inputsChanged(message);
 }
 for (const strand of ['query', 'target']) {
   const input = byId(strand), picker = byId(`${strand}-upload`);

gui/columns.js

@@ -105,7 +105,13 @@ export const CSV_CATALOG = [
   {
     "key": "E",
     "label": "Energy (kcal/mol)",
-    "description": "overall interaction energy",
+    "description": "minimum free energy of the reported interaction structure",
+    "numeric": true
+  },
+  {
+    "key": "Eens",
+    "label": "Ensemble energy (kcal/mol)",
+    "description": "ensemble energy used to select and rank a model-P interaction site",
     "numeric": true
   },
   {
@@ -317,19 +323,19 @@ export const CSV_CATALOG = [
 // Every native column appears once. Ensemble energies and their components are
 // grouped with probabilities so users can see the cost/meaning of those outputs.
 export const CSV_GROUPS = [
-  { label: 'Energy', keys: ['E', 'ED1', 'ED2', 'E_init', 'E_loops', 'E_dangleL', 'E_dangleR', 'E_endL', 'E_endR', 'E_hybrid', 'E_norm', 'E_hybridNorm', 'E_add', 'seedE', 'seedED1', 'seedED2'] },
+  { label: 'Energy', keys: ['E', 'Eens', 'ED1', 'ED2', 'E_init', 'E_loops', 'E_dangleL', 'E_dangleR', 'E_endL', 'E_endR', 'E_hybrid', 'E_norm', 'E_hybridNorm', 'E_add', 'seedE', 'seedED1', 'seedED2'] },
   { label: 'Output', keys: ['subseqDP', 'subseqDB', 'hybridDP', 'hybridDB', 'hybridDPfull', 'hybridDBfull', 'bpList'] },
   { label: 'Sequence', keys: ['id1', 'id2', 'seq1', 'seq2', 'subseq1', 'subseq2'] },
   { label: 'Interaction', keys: ['start1', 'end1', 'start2', 'end2', 'seedStart1', 'seedEnd1', 'seedStart2', 'seedEnd2'] },
   { label: 'Partition Function', keys: ['Pu1', 'Pu2', 'seedPu1', 'seedPu2', 'w', 'Etotal', 'Eall', 'Eall1', 'Eall2', 'Zall', 'Zall1', 'Zall2', 'EallTotal', 'P_E', 'RT'] },
 ];
 
-export function defaultColumns(task) {
+export function defaultColumns(task, model = 'X') {
   const context = ['id1', 'start1', 'end1', 'id2', 'start2', 'end2', 'subseqDP', 'hybridDP', 'E'];
   // Seed energies remain available in evaluation, where native output is NAN.
   // List them explicitly: seedEnd1/2 are coordinates, not energy columns.
   const extra = task === 'evaluation'
-    ? [...CSV_CATALOG.filter(({ key }) => key.startsWith('E')).map(({ key }) => key), 'seedE', 'seedED1', 'seedED2']
+    ? [...CSV_CATALOG.filter(({ key }) => key.startsWith('E') && key !== 'Eens').map(({ key }) => key), 'seedE', 'seedED1', 'seedED2']
     : ['ED1', 'ED2', 'seedE', 'seedStart1', 'seedEnd1', 'seedStart2', 'seedEnd2'];
-  return [...new Set([...context, ...extra])];
+  return [...new Set([...context, ...(task === 'prediction' && model === 'P' ? ['Eens'] : []), ...extra])];
 }

gui/options.js

@@ -185,4 +185,4 @@ export const EXTRA_OUTPUTS = [
   ['detailed', 'Detailed interaction report', false, 'Expanded text report with interaction structures, coordinates and energy contributions.'],
   ['ensemble', 'Ensemble report', false, 'Text report of interaction and intramolecular ensemble energies and partition functions.'],
 ].map(([key, label, predictionOnly, description]) => ({ key, label, prediction: predictionOnly, description }));
-export const ENSEMBLE_COLUMNS = ['Eall', 'Zall', 'EallTotal', 'P_E'];
+export const ENSEMBLE_COLUMNS = ['Eens', 'Eall', 'Zall', 'EallTotal', 'P_E'];

gui/parameter-file.js

@@ -127,6 +127,8 @@ export function importParameters(source) {
   const normalized = normalizeParameters(request);
   request.qIdxPos0 = normalized.qIdxPos0; request.tIdxPos0 = normalized.tIdxPos0;
   request.parameters = normalized;
+  if (request.task === 'prediction' && normalized.model === 'P'
+      && !Object.hasOwn(document.parameters, 'outCsvCols')) request.csvColumns.push('Eens');
   if (request.task === 'mutation') request.copomus = normalizeCopomus(request.copomus);
   return request;
 }

gui/protocol.js

@@ -181,8 +181,10 @@ export function prepareRequest(request) {
     if (!/^(D(m-?\d+(\.\d+)?)?(b-?\d+(\.\d+)?)?|Z(b\d+(\.\d+)?)?|W)$/.test(parameters[`${p}ShapeMethod`])) throw new Error(`${p}ShapeMethod: invalid SHAPE method encoding.`);
     if (!/^(M|C(\d+(\.\d+)?)?|S|[LO](s-?\d+(\.\d+)?i-?\d+(\.\d+)?)?)$/.test(parameters[`${p}ShapeConversion`])) throw new Error(`${p}ShapeConversion: invalid SHAPE conversion encoding.`);
   }
-  const csvColumns = request.csvColumns ?? CSV_COLUMNS.split(',');
+  const csvColumns = request.csvColumns
+    ?? `${CSV_COLUMNS}${!evaluating && parameters.model === 'P' ? ',Eens' : ''}`.split(',');
   if (!Array.isArray(csvColumns) || !csvColumns.length || new Set(csvColumns).size !== csvColumns.length || csvColumns.some((key) => !CSV_CATALOG.some((c) => c.key === key))) throw new Error('Select at least one available CSV column, without duplicates.');
+  if (csvColumns.includes('Eens') && (evaluating || parameters.model !== 'P')) throw new Error('Eens output is available only for ensemble model P predictions.');
   if (parameters.outCsvSort && !csvColumns.includes(parameters.outCsvSort)) throw new Error('The CSV sort column must be selected for output.');
   const outputs = request.outputs ?? [];
   if (!Array.isArray(outputs) || new Set(outputs).size !== outputs.length || outputs.some((key) => !EXTRA_OUTPUTS.some((entry) => entry.key === key && (!evaluating || !entry.prediction)))) throw new Error('Choose available additional output formats for this task.');

gui/results.js

@@ -2,7 +2,10 @@ import { CSV_CATALOG } from './columns.js';
 import { CONTEXT_COLUMNS } from './protocol.js';
 import { InteractionVisualization } from './visualization.js';
 
-export const COLUMNS = Object.freeze([...CSV_CATALOG, { key: 'deltaE', label: 'mfe Δ (kcal/mol)', numeric: true }]);
+export const COLUMNS = Object.freeze([...CSV_CATALOG, {
+  key: 'deltaE', label: 'Energy Δ (kcal/mol)', numeric: true,
+  description: 'Difference from the best ranking energy for this sequence pair (Eens for model P, otherwise E).',
+}]);
 const catalog = new Map(COLUMNS.map((column) => [column.key, column]));
 
 // Native CSV is unquoted, semicolon-separated; unavailable optional fields use
@@ -28,11 +31,12 @@ export function parseResults(csv) {
   });
   // Each sequence pair has its own optimum in a multi-FASTA screen.
   const best = new Map();
+  const rankingEnergy = (row) => Number.isFinite(Number(row.Eens)) ? Number(row.Eens) : Number(row.E);
   for (const row of rows) {
     const pair = JSON.stringify([row.id1, row.id2]);
-    best.set(pair, Math.min(best.get(pair) ?? Infinity, row.E));
+    best.set(pair, Math.min(best.get(pair) ?? Infinity, rankingEnergy(row)));
   }
-  return rows.map((row) => ({ ...row, deltaE: Number((row.E - best.get(JSON.stringify([row.id1, row.id2]))).toFixed(2)) }));
+  return rows.map((row) => ({ ...row, deltaE: Number((rankingEnergy(row) - best.get(JSON.stringify([row.id1, row.id2]))).toFixed(2)) }));
 }
 function numeric(value) { return String(value).trim() && Number.isFinite(Number(value)) ? Number(value) : null; }
 export function selectRows(rows, search, key, direction) {
@@ -96,7 +100,7 @@ export class ResultsView {
       });
       label.append(input, document.createTextNode(column.label)); this.picker.append(label);
     }
-    this.search.value = ''; this.key = columns.includes('E') ? 'E' : columns[0]; this.direction = 1;
+    this.search.value = ''; this.key = columns.includes('Eens') ? 'Eens' : columns.includes('E') ? 'E' : columns[0]; this.direction = 1;
     this.root.hidden = false; this.render(true); return this.rows.length;
   }
   render(selectFirst = false) {
@@ -142,7 +146,7 @@ export class ResultsView {
       action.append(view); tr.append(action);
       for (const column of columns) {
         const cell = document.createElement('td');
-        cell.textContent = ['E', 'deltaE'].includes(column.key) ? Number(row[column.key]).toFixed(2) : String(row[column.key]);
+        cell.textContent = ['E', 'Eens', 'deltaE'].includes(column.key) ? Number(row[column.key]).toFixed(2) : String(row[column.key]);
         if (column.numeric) cell.className = 'number'; tr.append(cell);
       }
       this.body.append(tr);

gui/tests/parameter-file.test.mjs

@@ -43,6 +43,8 @@ test('invalid/native-only settings are rejected before replacing the form', () =
   assert.throws(() => importParameters('{invalid'));
   assert.throws(() => importParameters(' '.repeat(5000001)), /5 MB/);
   assert.deepEqual(importParameters('\ufeff'+JSON.stringify(base)).csvColumns, CSV_COLUMNS.split(','));
+  assert.deepEqual(importParameters({ ...base, parameters: { ...base.parameters, model: 'P' } }).csvColumns,
+    [...CSV_COLUMNS.split(','), 'Eens']);
 });
 
 test('maintainer examples use the same JSON import and prediction validation', async () => {