diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index ffc171b556..b7707c6c66 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -140,6 +140,22 @@ def test_empty_log_directory_is_reported_as_incomplete(self): self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) self.assertIn("No log file found.", summary) + def test_dashboard_verdict_reaches_the_summary(self): + self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n") + self.write_file( + "reports/nangate45/gcd/base/metadata-check.log", + "[ERROR] Dashboard QoR check: FAIL (nangate45/gcd/base)\n" + "QoR regression detected for nangate45/gcd/base:\n" + " finish__timing__setup__tns -12.0 (base -6.0, limit -8.0)\n" + " Baseline: https://dashboard.example/builds/7\n", + ) + summary = self.summary() + self.assertIn("Failing designs", summary) + self.assertIn("nangate45 gcd (base)", summary) + self.assertIn("finish__timing__setup__tns -12.0", summary) + self.assertIn("limit -8.0", summary) + self.assertIn("https://dashboard.example/builds/7", summary) + class TestGenReport(unittest.TestCase): def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self): diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh index 3c365f92ff..5b8c3a2e8b 100755 --- a/flow/test/test_helper.sh +++ b/flow/test/test_helper.sh @@ -17,6 +17,8 @@ Options: --design-path DIR Root path containing 'designs/' (default: ./) --variant NAME Flow variant (default: unset) --target STR Make target(s), space-separated (default: 'finish metadata') + 'finish metadata-generate' keeps all full-flow follow-up steps + and leaves QoR checking to an external caller --work-home DIR Work home (default: .) --private-dir DIR Private tool scripts dir (default: ../../private_tool_scripts) --save-to-db Save metrics to DB (requires private.mk) @@ -106,7 +108,7 @@ if [ "$ret" -ne 0 ] && ! compgen -G "$REPORT_LOG_DIR/*.log" > /dev/null; then command cp "$LOG_FILE" "$REPORT_LOG_DIR/0_make.log" fi -if [ "${TARGET}" != "finish metadata" ]; then +if [ "${TARGET}" != "finish metadata" ] && [ "${TARGET}" != "finish metadata-generate" ]; then exit $ret fi diff --git a/flow/test/test_test_helper.py b/flow/test/test_test_helper.py new file mode 100644 index 0000000000..d3ae793296 --- /dev/null +++ b/flow/test/test_test_helper.py @@ -0,0 +1,82 @@ +"""Run the real flow helper against a disposable Makefile.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +HELPER = Path(__file__).with_name("test_helper.sh") + + +class TestHelper(unittest.TestCase): + def run_helper(self, target, fail_target=""): + with tempfile.TemporaryDirectory() as directory: + flow = Path(directory) + (flow / "test").mkdir() + helper = flow / "test/test_helper.sh" + shutil.copyfile(HELPER, helper) + (flow / "Makefile").write_text( + ".PHONY: clean_all clean_metadata finish metadata metadata-generate " + "metadata-check final_report_issue simulate power\n" + "metadata: finish metadata-generate metadata-check\n" + "print-LOG_DIR:\n\t@echo logs/nangate45/gcd/base\n" + "clean_all clean_metadata finish metadata-generate metadata-check " + "final_report_issue simulate power:\n" + "\t@echo $@ >> targets\n" + f"\t@test '$@' != '{fail_target}'\n" + ) + env = dict( + os.environ, + TARGET=target, + MAKE_ISSUE="1", + PRIVATE_DIR=str(flow / "private"), + ) + result = subprocess.run( + ["bash", str(helper), "gcd", "nangate45"], + cwd=flow, + env=env, + capture_output=True, + text=True, + ) + return result, (flow / "targets").read_text().splitlines() + + def test_metadata_generation_keeps_follow_up_steps(self): + result, targets = self.run_helper("finish metadata-generate") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + targets, + [ + "clean_all", + "clean_metadata", + "finish", + "metadata-generate", + "final_report_issue", + "simulate", + "power", + ], + ) + + def test_default_target_still_checks_qor(self): + result, targets = self.run_helper("finish metadata") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("metadata-check", targets) + self.assertEqual(targets[-3:], ["final_report_issue", "simulate", "power"]) + + def test_custom_target_still_stops_after_that_target(self): + result, targets = self.run_helper("finish") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(targets, ["clean_all", "clean_metadata", "finish"]) + + def test_failed_generation_is_not_a_pass(self): + result, targets = self.run_helper( + "finish metadata-generate", "metadata-generate" + ) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("simulate", targets) + self.assertNotIn("power", targets) + + +if __name__ == "__main__": + unittest.main()