From 0505175e4e2c51b5959b2b3c96dedea345fd0602 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 14 Oct 2026 02:28:33 -0300 Subject: [PATCH 1/4] fix(genReport): show failed QoR metrics Signed-off-by: Vitor Bandeira --- flow/util/checkQorMetrics.py | 18 ++++++++++++++++++ flow/util/genReport.py | 26 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/flow/util/checkQorMetrics.py b/flow/util/checkQorMetrics.py index 7146c53e507..de16525b58a 100755 --- a/flow/util/checkQorMetrics.py +++ b/flow/util/checkQorMetrics.py @@ -99,6 +99,11 @@ def fmt_value(value): return str(value) +def fmt_exact_value(value): + """Render a metric value without rounding away dashboard precision.""" + return "n/a" if value is None else str(value) + + def fmt_delta(value): """Same tolerance as fmt_value, with the sign the delta column wants.""" if value is None: @@ -366,6 +371,19 @@ def render_run(result, verbose): ) ) + failed_metrics = [m for m in metrics if m.get("status") == "fail"] + if failed_metrics: + lines.append( + f"QoR regression detected for {result['platform']}/{result['design']}:" + ) + for metric in sorted(failed_metrics, key=lambda m: m.get("metricName", "")): + lines.append( + f" {metric.get('metricName', '?')} " + f"{fmt_exact_value(metric.get('targetValue'))} " + f"(base {fmt_exact_value(metric.get('baseValue'))}, " + f"limit {fmt_exact_value(metric.get('thresholdValue'))})" + ) + # A design can have hundreds of absent metrics, so name them only on request. absent = sorted( metric.get("metricName", "?") diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 02b898775b6..7b2c532e887 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -80,6 +80,25 @@ def parse_messages(filename, print_missing=True): return errors, warnings +def parse_qor_regressions(filename): + """Return the failed metric details from a QoR check log, if present.""" + regressions = [] + in_regressions = False + try: + with open(filename, "r") as file: + for line in file: + if line.startswith("QoR regression detected for "): + regressions.append(line.strip()) + in_regressions = True + elif in_regressions and line.startswith(" ") and line.strip(): + regressions.append(line.strip()) + elif in_regressions: + break + except OSError: + pass + return regressions + + def append_text(list_, text, sup, regex, verbose): """ TODO @@ -136,6 +155,10 @@ def gen_report(name, data): output = append_text( data["metrics_errors"], output, "metrics failures", REGEX_ERROR, args.verbose ) + if data["metrics_regressions"]: + output += f" {data['metrics_regressions'][0]}\n" + for regression in data["metrics_regressions"][1:]: + output += f" {regression}\n" output = append_text( data["calibre_errors"], output, "calibre failures", REGEX_ERROR, args.verbose ) @@ -283,6 +306,9 @@ def write_summary(): d["metrics_errors"], d["metrics_warnings"] = parse_messages( METRICS_CHECK_FMT.format(report_dir), print_missing=d["finished"] ) + d["metrics_regressions"] = parse_qor_regressions( + METRICS_CHECK_FMT.format(report_dir) + ) # check if calibre was run and if drc check passed d["calibre_errors"], d["calibre_warnings"] = parse_messages( From 446f0d52a27f472fe6d4276dbdcfb39735647595 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 14 Oct 2026 03:23:38 -0300 Subject: [PATCH 2/4] fix(genReport): nest failed QoR metrics Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 97 +++++++++++++++++++++++++++++++++++++ flow/util/genReport.py | 26 ++++++---- 2 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 flow/test/test_genReport.py diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py new file mode 100644 index 00000000000..e904dad255d --- /dev/null +++ b/flow/test/test_genReport.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +import importlib.util +import os +import sys +import tempfile +import unittest +from unittest.mock import patch + + +GEN_REPORT_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "util", "genReport.py" +) +with patch("sys.argv", [GEN_REPORT_PATH]), patch("os.chdir"): + SPEC = importlib.util.spec_from_file_location("genReport", GEN_REPORT_PATH) + genReport = importlib.util.module_from_spec(SPEC) + SPEC.loader.exec_module(genReport) + + +class TestGenReport(unittest.TestCase): + def test_parse_qor_regressions_omits_the_optional_heading(self): + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log: + log.write( + "QoR regression detected for nangate45/gcd:\n" + " cts__timing__setup__tns -12.2456 " + "(base -6.95522, limit -8.346264)\n" + ) + log.flush() + + self.assertEqual( + genReport.parse_qor_regressions(log.name), + ["cts__timing__setup__tns -12.2456 " + "(base -6.95522, limit -8.346264)"], + ) + + def test_qor_metrics_are_nested_under_the_failing_run(self): + name = "nangate45/gcd/base" + data = { + "status": genReport.STATUS_RED, + "finished": True, + "last_log": "", + "log_errors": [], + "metrics_logs_errors": [], + "metrics_errors": [ + "[ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, " + "0 missing (478 metrics sent)", + "[ERROR] Failing runs: nangate45/gcd/base", + ], + "metrics_regressions": [ + "cts__timing__setup__tns -12.2456 " + "(base -6.95522, limit -8.346264)", + "cts__timing__setup__ws -0.261172 " + "(base -0.156758, limit -0.169258)", + "finish__timing__setup__tns -12.4146 " + "(base -7.19445, limit -8.63334)", + "finish__timing__setup__ws -0.271991 " + "(base -0.163541, limit -0.176041)", + "globalroute__timing__setup__tns -12.7365 " + "(base -7.5407, limit -9.04884)", + "globalroute__timing__setup__ws -0.275208 " + "(base -0.171167, limit -0.183667)", + ], + "calibre_errors": [], + "drcs": {}, + } + + report = genReport.gen_report(name, data) + + self.assertEqual( + report, + "\n".join( + [ + name, + " Flow reached last stage.", + " Found 2 metrics failures.", + " [ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, " + "0 missing (478 metrics sent)", + " cts__timing__setup__tns -12.2456 " + "(base -6.95522, limit -8.346264)", + " cts__timing__setup__ws -0.261172 " + "(base -0.156758, limit -0.169258)", + " finish__timing__setup__tns -12.4146 " + "(base -7.19445, limit -8.63334)", + " finish__timing__setup__ws -0.271991 " + "(base -0.163541, limit -0.176041)", + " globalroute__timing__setup__tns -12.7365 " + "(base -7.5407, limit -9.04884)", + " globalroute__timing__setup__ws -0.275208 " + "(base -0.171167, limit -0.183667)", + " [ERROR] Failing runs: nangate45/gcd/base", + ] + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 7b2c532e887..4e435e8e101 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -81,14 +81,13 @@ def parse_messages(filename, print_missing=True): def parse_qor_regressions(filename): - """Return the failed metric details from a QoR check log, if present.""" + """Return failed metric details from a QoR check log, if present.""" regressions = [] in_regressions = False try: - with open(filename, "r") as file: + with open(filename, "r", encoding="utf-8") as file: for line in file: if line.startswith("QoR regression detected for "): - regressions.append(line.strip()) in_regressions = True elif in_regressions and line.startswith(" ") and line.strip(): regressions.append(line.strip()) @@ -99,6 +98,19 @@ def parse_qor_regressions(filename): return regressions +def append_metrics_errors(errors, regressions, name, text): + if not errors: + return text + + text += f" Found {len(errors)} metrics failures.\n" + for error in errors: + text += f" {error}\n" + if error.startswith(f"[ERROR] {name}: FAIL"): + for regression in regressions: + text += f" {regression}\n" + return text + + def append_text(list_, text, sup, regex, verbose): """ TODO @@ -152,13 +164,9 @@ def gen_report(name, data): REGEX_ERROR, args.verbose, ) - output = append_text( - data["metrics_errors"], output, "metrics failures", REGEX_ERROR, args.verbose + output = append_metrics_errors( + data["metrics_errors"], data["metrics_regressions"], name, output ) - if data["metrics_regressions"]: - output += f" {data['metrics_regressions'][0]}\n" - for regression in data["metrics_regressions"][1:]: - output += f" {regression}\n" output = append_text( data["calibre_errors"], output, "calibre failures", REGEX_ERROR, args.verbose ) From 4df3c6262b5906f71532c1cf0e75fd619d4b6ab6 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 14 Oct 2026 04:00:03 -0300 Subject: [PATCH 3/4] fix(genReport): match metric errors by run line Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 2 +- flow/util/genReport.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index e904dad255d..534efcd15b6 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -34,7 +34,7 @@ def test_parse_qor_regressions_omits_the_optional_heading(self): ) def test_qor_metrics_are_nested_under_the_failing_run(self): - name = "nangate45/gcd/base" + name = "nangate45 gcd (base)" data = { "status": genReport.STATUS_RED, "finished": True, diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 4e435e8e101..923469d9490 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -98,14 +98,14 @@ def parse_qor_regressions(filename): return regressions -def append_metrics_errors(errors, regressions, name, text): +def append_metrics_errors(errors, regressions, text): if not errors: return text text += f" Found {len(errors)} metrics failures.\n" for error in errors: text += f" {error}\n" - if error.startswith(f"[ERROR] {name}: FAIL"): + if error.startswith("[ERROR] ") and ": FAIL " in error: for regression in regressions: text += f" {regression}\n" return text @@ -165,7 +165,7 @@ def gen_report(name, data): args.verbose, ) output = append_metrics_errors( - data["metrics_errors"], data["metrics_regressions"], name, output + data["metrics_errors"], data["metrics_regressions"], output ) output = append_text( data["calibre_errors"], output, "calibre failures", REGEX_ERROR, args.verbose From 884f3ae140780cd1a94d96f3e6f7591b5e3c94f8 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 14 Oct 2026 04:02:41 -0300 Subject: [PATCH 4/4] test(genReport): cover UTF-8 QoR log parsing Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index 534efcd15b6..38dafe4643a 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -7,7 +7,6 @@ import unittest from unittest.mock import patch - GEN_REPORT_PATH = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "util", "genReport.py" ) @@ -20,6 +19,7 @@ class TestGenReport(unittest.TestCase): def test_parse_qor_regressions_omits_the_optional_heading(self): with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log: + log.write("Ignored UTF-8 text: µ\n") log.write( "QoR regression detected for nangate45/gcd:\n" " cts__timing__setup__tns -12.2456 " @@ -29,8 +29,10 @@ def test_parse_qor_regressions_omits_the_optional_heading(self): self.assertEqual( genReport.parse_qor_regressions(log.name), - ["cts__timing__setup__tns -12.2456 " - "(base -6.95522, limit -8.346264)"], + [ + "cts__timing__setup__tns -12.2456 " + "(base -6.95522, limit -8.346264)" + ], ) def test_qor_metrics_are_nested_under_the_failing_run(self): @@ -47,10 +49,8 @@ def test_qor_metrics_are_nested_under_the_failing_run(self): "[ERROR] Failing runs: nangate45/gcd/base", ], "metrics_regressions": [ - "cts__timing__setup__tns -12.2456 " - "(base -6.95522, limit -8.346264)", - "cts__timing__setup__ws -0.261172 " - "(base -0.156758, limit -0.169258)", + "cts__timing__setup__tns -12.2456 " "(base -6.95522, limit -8.346264)", + "cts__timing__setup__ws -0.261172 " "(base -0.156758, limit -0.169258)", "finish__timing__setup__tns -12.4146 " "(base -7.19445, limit -8.63334)", "finish__timing__setup__ws -0.271991 "