Include failed QoR metrics in flow reports

flow/test/test_genReport.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+import importlib.util
+import os
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+GEN_REPORT_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "util", "genReport.py"
+)
+with patch("sys.argv", [GEN_REPORT_PATH]), patch("os.chdir"):
+    SPEC = importlib.util.spec_from_file_location("genReport", GEN_REPORT_PATH)
+    genReport = importlib.util.module_from_spec(SPEC)
+    SPEC.loader.exec_module(genReport)
+
+
+class TestGenReport(unittest.TestCase):
+    def test_parse_qor_regressions_omits_the_optional_heading(self):
+        with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log:
+            log.write("Ignored UTF-8 text: µ\n")
+            log.write(
+                "QoR regression detected for nangate45/gcd:\n"
+                "  cts__timing__setup__tns -12.2456 "
+                "(base -6.95522, limit -8.346264)\n"
+            )
+            log.flush()
+
+            self.assertEqual(
+                genReport.parse_qor_regressions(log.name),
+                [
+                    "cts__timing__setup__tns -12.2456 "
+                    "(base -6.95522, limit -8.346264)"
+                ],
+            )
+
+    def test_qor_metrics_are_nested_under_the_failing_run(self):
+        name = "nangate45 gcd (base)"
+        data = {
+            "status": genReport.STATUS_RED,
+            "finished": True,
+            "last_log": "",
+            "log_errors": [],
+            "metrics_logs_errors": [],
+            "metrics_errors": [
+                "[ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, "
+                "0 missing (478 metrics sent)",
+                "[ERROR] Failing runs: nangate45/gcd/base",
+            ],
+            "metrics_regressions": [
+                "cts__timing__setup__tns -12.2456 " "(base -6.95522, limit -8.346264)",
+                "cts__timing__setup__ws -0.261172 " "(base -0.156758, limit -0.169258)",
+                "finish__timing__setup__tns -12.4146 "
+                "(base -7.19445, limit -8.63334)",
+                "finish__timing__setup__ws -0.271991 "
+                "(base -0.163541, limit -0.176041)",
+                "globalroute__timing__setup__tns -12.7365 "
+                "(base -7.5407, limit -9.04884)",
+                "globalroute__timing__setup__ws -0.275208 "
+                "(base -0.171167, limit -0.183667)",
+            ],
+            "calibre_errors": [],
+            "drcs": {},
+        }
+
+        report = genReport.gen_report(name, data)
+
+        self.assertEqual(
+            report,
+            "\n".join(
+                [
+                    name,
+                    "  Flow reached last stage.",
+                    "  Found 2 metrics failures.",
+                    "      [ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, "
+                    "0 missing (478 metrics sent)",
+                    "          cts__timing__setup__tns -12.2456 "
+                    "(base -6.95522, limit -8.346264)",
+                    "          cts__timing__setup__ws -0.261172 "
+                    "(base -0.156758, limit -0.169258)",
+                    "          finish__timing__setup__tns -12.4146 "
+                    "(base -7.19445, limit -8.63334)",
+                    "          finish__timing__setup__ws -0.271991 "
+                    "(base -0.163541, limit -0.176041)",
+                    "          globalroute__timing__setup__tns -12.7365 "
+                    "(base -7.5407, limit -9.04884)",
+                    "          globalroute__timing__setup__ws -0.275208 "
+                    "(base -0.171167, limit -0.183667)",
+                    "      [ERROR] Failing runs: nangate45/gcd/base",
+                ]
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

flow/util/checkQorMetrics.py

@@ -99,6 +99,11 @@ def fmt_value(value):
         return str(value)
 
 
+def fmt_exact_value(value):
+    """Render a metric value without rounding away dashboard precision."""
+    return "n/a" if value is None else str(value)
+
+
 def fmt_delta(value):
     """Same tolerance as fmt_value, with the sign the delta column wants."""
     if value is None:
@@ -366,6 +371,19 @@ def render_run(result, verbose):
                 )
             )
 
+    failed_metrics = [m for m in metrics if m.get("status") == "fail"]
+    if failed_metrics:
+        lines.append(
+            f"QoR regression detected for {result['platform']}/{result['design']}:"
+        )
+        for metric in sorted(failed_metrics, key=lambda m: m.get("metricName", "")):
+            lines.append(
+                f"  {metric.get('metricName', '?')} "
+                f"{fmt_exact_value(metric.get('targetValue'))} "
+                f"(base {fmt_exact_value(metric.get('baseValue'))}, "
+                f"limit {fmt_exact_value(metric.get('thresholdValue'))})"
+            )
+
     # A design can have hundreds of absent metrics, so name them only on request.
     absent = sorted(
         metric.get("metricName", "?")

flow/util/genReport.py

@@ -80,6 +80,37 @@ def parse_messages(filename, print_missing=True):
     return errors, warnings
 
 
+def parse_qor_regressions(filename):
+    """Return failed metric details from a QoR check log, if present."""
+    regressions = []
+    in_regressions = False
+    try:
+        with open(filename, "r", encoding="utf-8") as file:
+            for line in file:
+                if line.startswith("QoR regression detected for "):
+                    in_regressions = True
+                elif in_regressions and line.startswith("  ") and line.strip():
+                    regressions.append(line.strip())
+                elif in_regressions:
+                    break
+    except OSError:
+        pass
+    return regressions
+
+
+def append_metrics_errors(errors, regressions, text):
+    if not errors:
+        return text
+
+    text += f"  Found {len(errors)} metrics failures.\n"
+    for error in errors:
+        text += f"      {error}\n"
+        if error.startswith("[ERROR] ") and ": FAIL " in error:
+            for regression in regressions:
+                text += f"          {regression}\n"
+    return text
+
+
 def append_text(list_, text, sup, regex, verbose):
     """
     TODO
@@ -133,8 +164,8 @@ def gen_report(name, data):
         REGEX_ERROR,
         args.verbose,
     )
-    output = append_text(
-        data["metrics_errors"], output, "metrics failures", REGEX_ERROR, args.verbose
+    output = append_metrics_errors(
+        data["metrics_errors"], data["metrics_regressions"], output
     )
     output = append_text(
         data["calibre_errors"], output, "calibre failures", REGEX_ERROR, args.verbose
@@ -283,6 +314,9 @@ def write_summary():
     d["metrics_errors"], d["metrics_warnings"] = parse_messages(
         METRICS_CHECK_FMT.format(report_dir), print_missing=d["finished"]
     )
+    d["metrics_regressions"] = parse_qor_regressions(
+        METRICS_CHECK_FMT.format(report_dir)
+    )
 
     # check if calibre was run and if drc check passed
     d["calibre_errors"], d["calibre_warnings"] = parse_messages(