Skip to content

fix(report): classify metric generation errors as failures #4695

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jinchuntan wants to merge 2 commits into
base: master
Choose a base branch
from
Open

fix(report): classify metric generation errors as failures #4695

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
da4f699
fix(report): classify metric generation errors as failures
jinchuntan Oct 17, 2026
5caf6bd
test(report): provide a passing QoR log for the rerun fixture
jinchuntan Oct 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions flow/test/test_genReport.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,60 @@ def test_dashboard_verdict_reaches_the_summary(self):
self.assertIn("limit -8.0", summary)
self.assertIn("https://dashboard.example/builds/7", summary)

def test_metrics_generation_errors_fail_a_finished_flow(self):
self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n")
self.write_file(
"reports/nangate45/gcd/base/metadata-generate.log",
"[ERROR] Failed to open file: missing.rpt\n",
)
self.write_file(
"reports/nangate45/gcd/base/metadata-check.log",
"[INFO] QoR check: PASS\n",
)

summary = self.summary()

self.assertIn("Failing designs", summary)
self.assertIn("nangate45 gcd (base)\n Flow reached last stage.", summary)
self.assertIn("Found 1 errors in the metrics logs.", summary)
self.assertIn("[ERROR] Failed to open file: missing.rpt", summary)
self.assertNotIn("Passing designs", summary)

def test_metrics_generation_warnings_do_not_fail_a_finished_flow(self):
self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n")
self.write_file(
"reports/nangate45/gcd/base/metadata-generate.log",
"[WARNING] Optional metric not found. Will use N/A.\n",
)
self.write_file(
"reports/nangate45/gcd/base/metadata-check.log",
"[INFO] QoR check: PASS\n",
)

summary = self.summary()

self.assertIn("Number of designs: 1", summary)
self.assertNotIn("Failing designs", summary)

def test_successful_metrics_regeneration_clears_the_failure(self):
self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n")
self.write_file(
"reports/nangate45/gcd/base/metadata-generate.log",
"[ERROR] Failed to open file: missing.rpt\n",
)
self.write_file(
"reports/nangate45/gcd/base/metadata-check.log",
"[INFO] QoR check: PASS\n",
)
self.assertIn("Failing designs", self.summary())
Comment on lines +194 to +204

@gemini-code-assist gemini-code-assist Bot Oct 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The test does not create a metadata-check.log file. Since the flow is marked as finished, genReport.py will attempt to parse metadata-check.log and print a Failed to open... warning to stdout when self.summary() is called. Creating a dummy metadata-check.log file will prevent this warning from polluting the test output.

Suggested change
def test_successful_metrics_regeneration_clears_the_failure(self):
self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n")
self.write_file(
"reports/nangate45/gcd/base/metadata-generate.log",
"[ERROR] Failed to open file: missing.rpt\n",
)
self.assertIn("Failing designs", self.summary())
def test_successful_metrics_regeneration_clears_the_failure(self):
self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\\n")
self.write_file(
"reports/nangate45/gcd/base/metadata-generate.log",
"[ERROR] Failed to open file: missing.rpt\\n",
)
self.write_file(
"reports/nangate45/gcd/base/metadata-check.log",
"[INFO] QoR check: PASS\\n",
)
self.assertIn("Failing designs", self.summary())


self.write_file("reports/nangate45/gcd/base/metadata-generate.log", "")
summary = self.summary()

self.assertIn("Number of designs: 1", summary)
self.assertNotIn("Failing designs", summary)
self.assertNotIn("errors in the metrics logs", summary)


class TestGenReport(unittest.TestCase):
def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self):
Expand Down
7 changes: 6 additions & 1 deletion flow/util/genReport.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,12 @@ def write_summary():
if d["finished"]:
print(f"Failed to open {DRC_FILENAME}.")

if d["log_errors"] or d["metrics_errors"] or d["calibre_errors"]:
if (
d["log_errors"]
or d["metrics_logs_errors"]
or d["metrics_errors"]
or d["calibre_errors"]
):
d["status"] = STATUS_RED
else:
if not d["finished"]:
Expand Down