From da4f699f744071d2dcaafc00c7a7eaf8c6d11325 Mon Sep 17 00:00:00 2001 From: Nigel Tan <79140620+jinchuntan@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:38:58 -0700 Subject: [PATCH 1/2] fix(report): classify metric generation errors as failures Include metric generator errors when classifying completed flows. A successful QoR check must not hide errors in metadata-generate.log. Cover generator failures, warning-only logs and a successful metrics rerun through the report CLI. Prepared with Codex assistance. Signed-off-by: Nigel Tan <79140620+jinchuntan@users.noreply.github.com> --- flow/test/test_genReport.py | 50 +++++++++++++++++++++++++++++++++++++ flow/util/genReport.py | 7 +++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index b7707c6c66..87a0349b13 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -156,6 +156,56 @@ def test_dashboard_verdict_reaches_the_summary(self): self.assertIn("limit -8.0", summary) self.assertIn("https://dashboard.example/builds/7", summary) + def test_metrics_generation_errors_fail_a_finished_flow(self): + self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n") + self.write_file( + "reports/nangate45/gcd/base/metadata-generate.log", + "[ERROR] Failed to open file: missing.rpt\n", + ) + self.write_file( + "reports/nangate45/gcd/base/metadata-check.log", + "[INFO] QoR check: PASS\n", + ) + + summary = self.summary() + + self.assertIn("Failing designs", summary) + self.assertIn("nangate45 gcd (base)\n Flow reached last stage.", summary) + self.assertIn("Found 1 errors in the metrics logs.", summary) + self.assertIn("[ERROR] Failed to open file: missing.rpt", summary) + self.assertNotIn("Passing designs", summary) + + def test_metrics_generation_warnings_do_not_fail_a_finished_flow(self): + self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n") + self.write_file( + "reports/nangate45/gcd/base/metadata-generate.log", + "[WARNING] Optional metric not found. Will use N/A.\n", + ) + self.write_file( + "reports/nangate45/gcd/base/metadata-check.log", + "[INFO] QoR check: PASS\n", + ) + + summary = self.summary() + + self.assertIn("Number of designs: 1", summary) + self.assertNotIn("Failing designs", summary) + + def test_successful_metrics_regeneration_clears_the_failure(self): + self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n") + self.write_file( + "reports/nangate45/gcd/base/metadata-generate.log", + "[ERROR] Failed to open file: missing.rpt\n", + ) + self.assertIn("Failing designs", self.summary()) + + self.write_file("reports/nangate45/gcd/base/metadata-generate.log", "") + summary = self.summary() + + self.assertIn("Number of designs: 1", summary) + self.assertNotIn("Failing designs", summary) + self.assertNotIn("errors in the metrics logs", summary) + class TestGenReport(unittest.TestCase): def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self): diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 52ec6115dd..f85d5b18f6 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -343,7 +343,12 @@ def write_summary(): if d["finished"]: print(f"Failed to open {DRC_FILENAME}.") - if d["log_errors"] or d["metrics_errors"] or d["calibre_errors"]: + if ( + d["log_errors"] + or d["metrics_logs_errors"] + or d["metrics_errors"] + or d["calibre_errors"] + ): d["status"] = STATUS_RED else: if not d["finished"]: From 5caf6bdf2ed64d26d5f8a65a772b542f835259c7 Mon Sep 17 00:00:00 2001 From: Nigel Tan Date: Sun, 18 Oct 2026 01:07:25 +0800 Subject: [PATCH 2/2] test(report): provide a passing QoR log for the rerun fixture Keep the metrics regeneration regression focused on the generator log. Prepared with Codex assistance. Signed-off-by: Nigel Tan --- flow/test/test_genReport.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index 87a0349b13..d154b55ff9 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -197,6 +197,10 @@ def test_successful_metrics_regeneration_clears_the_failure(self): "reports/nangate45/gcd/base/metadata-generate.log", "[ERROR] Failed to open file: missing.rpt\n", ) + self.write_file( + "reports/nangate45/gcd/base/metadata-check.log", + "[INFO] QoR check: PASS\n", + ) self.assertIn("Failing designs", self.summary()) self.write_file("reports/nangate45/gcd/base/metadata-generate.log", "")