From a1a46221f99d70ba17d7121f709a8969412476f7 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Thu, 15 Oct 2026 13:23:38 -0300 Subject: [PATCH 1/2] fix(test): keep full-flow steps with metadata generation Allow an external QoR checker to own the verdict without skipping simulation, power, or issue packaging after metadata generation. Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 16 +++++++ flow/test/test_helper.sh | 4 +- flow/test/test_test_helper.py | 82 +++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 flow/test/test_test_helper.py diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index ffc171b556..c2bc54ae40 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -140,6 +140,22 @@ def test_empty_log_directory_is_reported_as_incomplete(self): self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) self.assertIn("No log file found.", summary) + def test_dashboard_verdict_reaches_the_summary(self): + self.write_file("logs/gf12/aes/base/6_report.log", "Finished\n") + self.write_file( + "reports/gf12/aes/base/metadata-check.log", + "[ERROR] Dashboard QoR check: FAIL (gf12/aes/base)\n" + "QoR regression detected for gf12/aes/base:\n" + " finish__timing__setup__tns -253.714 (base -51.7363, limit -123.7363)\n" + " Baseline: https://dashboard.example/builds/33923\n", + ) + summary = self.summary() + self.assertIn("Failing designs", summary) + self.assertIn("gf12 aes (base)", summary) + self.assertIn("finish__timing__setup__tns -253.714", summary) + self.assertIn("limit -123.7363", summary) + self.assertIn("https://dashboard.example/builds/33923", summary) + class TestGenReport(unittest.TestCase): def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self): diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh index 3c365f92ff..5b8c3a2e8b 100755 --- a/flow/test/test_helper.sh +++ b/flow/test/test_helper.sh @@ -17,6 +17,8 @@ Options: --design-path DIR Root path containing 'designs/' (default: ./) --variant NAME Flow variant (default: unset) --target STR Make target(s), space-separated (default: 'finish metadata') + 'finish metadata-generate' keeps all full-flow follow-up steps + and leaves QoR checking to an external caller --work-home DIR Work home (default: .) --private-dir DIR Private tool scripts dir (default: ../../private_tool_scripts) --save-to-db Save metrics to DB (requires private.mk) @@ -106,7 +108,7 @@ if [ "$ret" -ne 0 ] && ! compgen -G "$REPORT_LOG_DIR/*.log" > /dev/null; then command cp "$LOG_FILE" "$REPORT_LOG_DIR/0_make.log" fi -if [ "${TARGET}" != "finish metadata" ]; then +if [ "${TARGET}" != "finish metadata" ] && [ "${TARGET}" != "finish metadata-generate" ]; then exit $ret fi diff --git a/flow/test/test_test_helper.py b/flow/test/test_test_helper.py new file mode 100644 index 0000000000..d3ae793296 --- /dev/null +++ b/flow/test/test_test_helper.py @@ -0,0 +1,82 @@ +"""Run the real flow helper against a disposable Makefile.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +HELPER = Path(__file__).with_name("test_helper.sh") + + +class TestHelper(unittest.TestCase): + def run_helper(self, target, fail_target=""): + with tempfile.TemporaryDirectory() as directory: + flow = Path(directory) + (flow / "test").mkdir() + helper = flow / "test/test_helper.sh" + shutil.copyfile(HELPER, helper) + (flow / "Makefile").write_text( + ".PHONY: clean_all clean_metadata finish metadata metadata-generate " + "metadata-check final_report_issue simulate power\n" + "metadata: finish metadata-generate metadata-check\n" + "print-LOG_DIR:\n\t@echo logs/nangate45/gcd/base\n" + "clean_all clean_metadata finish metadata-generate metadata-check " + "final_report_issue simulate power:\n" + "\t@echo $@ >> targets\n" + f"\t@test '$@' != '{fail_target}'\n" + ) + env = dict( + os.environ, + TARGET=target, + MAKE_ISSUE="1", + PRIVATE_DIR=str(flow / "private"), + ) + result = subprocess.run( + ["bash", str(helper), "gcd", "nangate45"], + cwd=flow, + env=env, + capture_output=True, + text=True, + ) + return result, (flow / "targets").read_text().splitlines() + + def test_metadata_generation_keeps_follow_up_steps(self): + result, targets = self.run_helper("finish metadata-generate") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + targets, + [ + "clean_all", + "clean_metadata", + "finish", + "metadata-generate", + "final_report_issue", + "simulate", + "power", + ], + ) + + def test_default_target_still_checks_qor(self): + result, targets = self.run_helper("finish metadata") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("metadata-check", targets) + self.assertEqual(targets[-3:], ["final_report_issue", "simulate", "power"]) + + def test_custom_target_still_stops_after_that_target(self): + result, targets = self.run_helper("finish") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(targets, ["clean_all", "clean_metadata", "finish"]) + + def test_failed_generation_is_not_a_pass(self): + result, targets = self.run_helper( + "finish metadata-generate", "metadata-generate" + ) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("simulate", targets) + self.assertNotIn("power", targets) + + +if __name__ == "__main__": + unittest.main() From 4d62e8d855cdba5fbc4dc11d33893ba791ba9152 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Thu, 15 Oct 2026 15:27:25 -0300 Subject: [PATCH 2/2] test(report): use public synthetic dashboard data Use a public platform and synthetic timing values in the report fixture. Keep the same assertions for the failed metric, limit, and baseline link. Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index c2bc54ae40..b7707c6c66 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -141,20 +141,20 @@ def test_empty_log_directory_is_reported_as_incomplete(self): self.assertIn("No log file found.", summary) def test_dashboard_verdict_reaches_the_summary(self): - self.write_file("logs/gf12/aes/base/6_report.log", "Finished\n") + self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n") self.write_file( - "reports/gf12/aes/base/metadata-check.log", - "[ERROR] Dashboard QoR check: FAIL (gf12/aes/base)\n" - "QoR regression detected for gf12/aes/base:\n" - " finish__timing__setup__tns -253.714 (base -51.7363, limit -123.7363)\n" - " Baseline: https://dashboard.example/builds/33923\n", + "reports/nangate45/gcd/base/metadata-check.log", + "[ERROR] Dashboard QoR check: FAIL (nangate45/gcd/base)\n" + "QoR regression detected for nangate45/gcd/base:\n" + " finish__timing__setup__tns -12.0 (base -6.0, limit -8.0)\n" + " Baseline: https://dashboard.example/builds/7\n", ) summary = self.summary() self.assertIn("Failing designs", summary) - self.assertIn("gf12 aes (base)", summary) - self.assertIn("finish__timing__setup__tns -253.714", summary) - self.assertIn("limit -123.7363", summary) - self.assertIn("https://dashboard.example/builds/33923", summary) + self.assertIn("nangate45 gcd (base)", summary) + self.assertIn("finish__timing__setup__tns -12.0", summary) + self.assertIn("limit -8.0", summary) + self.assertIn("https://dashboard.example/builds/7", summary) class TestGenReport(unittest.TestCase):