Keep full-flow steps when generating metadata without QoR checks

flow/test/test_genReport.py

@@ -140,6 +140,22 @@ def test_empty_log_directory_is_reported_as_incomplete(self):
         self.assertIn("sky130hd gcd (base)\n  Flow did not reach last stage.", summary)
         self.assertIn("No log file found.", summary)
 
+    def test_dashboard_verdict_reaches_the_summary(self):
+        self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n")
+        self.write_file(
+            "reports/nangate45/gcd/base/metadata-check.log",
+            "[ERROR] Dashboard QoR check: FAIL (nangate45/gcd/base)\n"
+            "QoR regression detected for nangate45/gcd/base:\n"
+            "  finish__timing__setup__tns -12.0 (base -6.0, limit -8.0)\n"
+            "  Baseline: https://dashboard.example/builds/7\n",
+        )
+        summary = self.summary()
+        self.assertIn("Failing designs", summary)
+        self.assertIn("nangate45 gcd (base)", summary)
+        self.assertIn("finish__timing__setup__tns -12.0", summary)
+        self.assertIn("limit -8.0", summary)
+        self.assertIn("https://dashboard.example/builds/7", summary)
+
 
 class TestGenReport(unittest.TestCase):
     def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self):

flow/test/test_helper.sh

@@ -17,6 +17,8 @@ Options:
   --design-path DIR     Root path containing 'designs/' (default: ./)
   --variant NAME        Flow variant (default: unset)
   --target STR          Make target(s), space-separated (default: 'finish metadata')
+                        'finish metadata-generate' keeps all full-flow follow-up steps
+                        and leaves QoR checking to an external caller
   --work-home DIR       Work home (default: .)
   --private-dir DIR     Private tool scripts dir (default: ../../private_tool_scripts)
   --save-to-db          Save metrics to DB (requires private.mk)
@@ -106,7 +108,7 @@ if [ "$ret" -ne 0 ] && ! compgen -G "$REPORT_LOG_DIR/*.log" > /dev/null; then
   command cp "$LOG_FILE" "$REPORT_LOG_DIR/0_make.log"
 fi
 
-if [ "${TARGET}" != "finish metadata" ]; then
+if [ "${TARGET}" != "finish metadata" ] && [ "${TARGET}" != "finish metadata-generate" ]; then
   exit $ret
 fi
 

flow/test/test_test_helper.py

@@ -0,0 +1,82 @@
+"""Run the real flow helper against a disposable Makefile."""
+
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+HELPER = Path(__file__).with_name("test_helper.sh")
+
+
+class TestHelper(unittest.TestCase):
+    def run_helper(self, target, fail_target=""):
+        with tempfile.TemporaryDirectory() as directory:
+            flow = Path(directory)
+            (flow / "test").mkdir()
+            helper = flow / "test/test_helper.sh"
+            shutil.copyfile(HELPER, helper)
+            (flow / "Makefile").write_text(
+                ".PHONY: clean_all clean_metadata finish metadata metadata-generate "
+                "metadata-check final_report_issue simulate power\n"
+                "metadata: finish metadata-generate metadata-check\n"
+                "print-LOG_DIR:\n\t@echo logs/nangate45/gcd/base\n"
+                "clean_all clean_metadata finish metadata-generate metadata-check "
+                "final_report_issue simulate power:\n"
+                "\t@echo $@ >> targets\n"
+                f"\t@test '$@' != '{fail_target}'\n"
+            )
+            env = dict(
+                os.environ,
+                TARGET=target,
+                MAKE_ISSUE="1",
+                PRIVATE_DIR=str(flow / "private"),
+            )
+            result = subprocess.run(
+                ["bash", str(helper), "gcd", "nangate45"],
+                cwd=flow,
+                env=env,
+                capture_output=True,
+                text=True,
+            )
+            return result, (flow / "targets").read_text().splitlines()
+
+    def test_metadata_generation_keeps_follow_up_steps(self):
+        result, targets = self.run_helper("finish metadata-generate")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(
+            targets,
+            [
+                "clean_all",
+                "clean_metadata",
+                "finish",
+                "metadata-generate",
+                "final_report_issue",
+                "simulate",
+                "power",
+            ],
+        )
+
+    def test_default_target_still_checks_qor(self):
+        result, targets = self.run_helper("finish metadata")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("metadata-check", targets)
+        self.assertEqual(targets[-3:], ["final_report_issue", "simulate", "power"])
+
+    def test_custom_target_still_stops_after_that_target(self):
+        result, targets = self.run_helper("finish")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(targets, ["clean_all", "clean_metadata", "finish"])
+
+    def test_failed_generation_is_not_a_pass(self):
+        result, targets = self.run_helper(
+            "finish metadata-generate", "metadata-generate"
+        )
+        self.assertNotEqual(result.returncode, 0)
+        self.assertNotIn("simulate", targets)
+        self.assertNotIn("power", targets)
+
+
+if __name__ == "__main__":
+    unittest.main()