fix(report): include flows that fail before stage logs

flow/test/test_genReport.py

@@ -2,8 +2,11 @@
 
 import importlib.util
 import os
+import shutil
+import subprocess
 import sys
 import tempfile
+import textwrap
 import unittest
 from unittest.mock import patch
 
@@ -16,7 +19,156 @@
     SPEC.loader.exec_module(genReport)
 
 
+class TestReportCollection(unittest.TestCase):
+    def setUp(self):
+        self.tmp_dir = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp_dir.cleanup)
+        self.tmp = self.tmp_dir.name
+        for folder in ("test", "util", "reports"):
+            os.makedirs(os.path.join(self.tmp, folder))
+        shutil.copy2(GEN_REPORT_PATH, os.path.join(self.tmp, "util/genReport.py"))
+        self.helper = os.path.join(self.tmp, "test/test_helper.sh")
+        shutil.copy2(
+            os.path.join(os.path.dirname(GEN_REPORT_PATH), "../test/test_helper.sh"),
+            self.helper,
+        )
+        self.write_file(
+            "designs/sky130hd/gcd/config.mk",
+            "PLATFORM = sky130hd\nDESIGN_NICKNAME = gcd\n",
+        )
+        # Real Make resolves the run directory and fails on the missing input.
+        self.write_file(
+            "Makefile",
+            textwrap.dedent("""\
+                include $(DESIGN_CONFIG)
+                FLOW_VARIANT ?= base
+                LOG_DIR = $(WORK_HOME)/logs/$(PLATFORM)/$(DESIGN_NICKNAME)/$(FLOW_VARIANT)
+                .PHONY: clean_all clean_metadata finish metadata
+                clean_all clean_metadata:
+                \t@rm -f $(LOG_DIR)/[1-6]_*.log
+                finish: missing.v
+                \t@mkdir -p $(LOG_DIR)
+                \t@echo Finished > $(LOG_DIR)/6_report.log
+                metadata:
+                \t@true
+                print-%:
+                \t$(info $*: $($*))
+                \t@true
+                """),
+        )
+        self.env = dict(os.environ, WORK_HOME=self.tmp)
+        for key in ("FLOW_VARIANT", "MAKE_ISSUE", "SAVE_TO_DB", "RUN_CALIBRE"):
+            self.env.pop(key, None)
+
+    def write_file(self, path, content):
+        path = os.path.join(self.tmp, path)
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "w") as f:
+            f.write(content)
+
+    def run_helper(self):
+        return subprocess.run(
+            ["bash", self.helper, "gcd", "sky130hd"],
+            env=self.env,
+            capture_output=True,
+            text=True,
+        )
+
+    def summary(self):
+        result = subprocess.run(
+            [sys.executable, os.path.join(self.tmp, "util/genReport.py"), "-sv"],
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        with open(os.path.join(self.tmp, "reports/report-summary.log")) as f:
+            return f.read()
+
+    def test_summary_includes_make_failure_before_the_first_stage(self):
+        result = self.run_helper()
+        self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
+        self.assertRegex(result.stdout, r"No rule to make target .missing\.v.")
+        for platform in ("nangate45", "ihp-sg13g2"):
+            self.write_file(f"logs/{platform}/gcd/base/6_report.log", "Finished\n")
+            self.write_file(f"reports/{platform}/gcd/base/metadata-check.log", "")
+        self.write_file(
+            "reports/nangate45/gcd/base/metadata-check.log",
+            "[ERROR] nangate45/gcd/base: FAIL\n",
+        )
+
+        summary = self.summary()
+
+        self.assertIn("Number of designs: 3", summary)
+        self.assertIn("nangate45 gcd (base)", summary)
+        self.assertIn("sky130hd gcd (base)\n  Flow did not reach last stage.", summary)
+        self.assertRegex(summary, r"No rule to make target .missing\.v.")
+        self.assertNotIn("ihp-sg13g2 gcd (base)", summary)
+
+    def test_early_failure_uses_the_configured_variant_and_design_nickname(self):
+        self.write_file(
+            "designs/sky130hd/gcd/config.mk",
+            "PLATFORM = sky130hd\nDESIGN_NICKNAME = gcd_alias\nFLOW_VARIANT = alternate\n",
+        )
+        result = self.run_helper()
+        self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
+        self.assertIn(
+            "sky130hd gcd_alias (alternate)\n  Flow did not reach last stage.",
+            self.summary(),
+        )
+
+    def test_successful_rerun_does_not_keep_the_early_failure(self):
+        result = self.run_helper()
+        self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
+        self.write_file("missing.v", "")
+        result = self.run_helper()
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        summary = self.summary()
+        self.assertIn("Number of designs: 1", summary)
+        self.assertNotIn("Failing designs", summary)
+
+    def test_finish_log_is_recognized_with_a_later_auxiliary_log(self):
+        self.write_file("logs/sky130hd/gcd/base/6_report.log", "Finished\n")
+        self.write_file("logs/sky130hd/gcd/base/6_write_rc.log", "RC values\n")
+        summary = self.summary()
+        self.assertIn("Number of designs: 1", summary)
+        self.assertNotIn("Failing designs", summary)
+
+    def test_empty_log_directory_is_reported_as_incomplete(self):
+        os.makedirs(os.path.join(self.tmp, "logs/sky130hd/gcd/base"))
+        summary = self.summary()
+        self.assertIn("Number of designs: 1", summary)
+        self.assertIn("sky130hd gcd (base)\n  Flow did not reach last stage.", summary)
+        self.assertIn("No log file found.", summary)
+
+
 class TestGenReport(unittest.TestCase):
+    def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self):
+        name = "nangate45 gcd (base)"
+        data = {
+            "status": genReport.STATUS_RED,
+            "finished": False,
+            "last_log": "5_route.log",
+            "log_errors": [],
+            "metrics_logs_errors": [],
+            "metrics_errors": [],
+            "metrics_regressions": [],
+            "calibre_errors": [],
+            "drcs": {},
+        }
+
+        report = genReport.gen_report(name, data)
+
+        self.assertEqual(
+            report,
+            "\n".join(
+                [
+                    name,
+                    "  Flow did not reach last stage.",
+                    "  Last log file 5_route.log",
+                ]
+            ),
+        )
+
     def test_parse_qor_regressions_omits_the_optional_heading(self):
         with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log:
             log.write("Ignored UTF-8 text: µ\n")

flow/test/test_helper.sh

@@ -86,6 +86,10 @@ fi
 mkdir -p "$(dirname "$LOG_FILE")"
 "${__make[@]}" clean_all clean_metadata 2>&1 | tee "$LOG_FILE"
 
+REPORT_LOG_DIR=$("${__make[@]}" --no-print-directory -s print-LOG_DIR)
+REPORT_LOG_DIR=${REPORT_LOG_DIR#LOG_DIR: }
+rm -f "$REPORT_LOG_DIR/0_make.log"
+
 # turn off abort on error so we can always capture the result
 set +e
 
@@ -96,6 +100,12 @@ read -r -a TARGET_ARR <<< "$TARGET"
 # the results
 ret=$?
 
+# Make can fail on a prerequisite before any stage creates a log.
+if [ "$ret" -ne 0 ] && ! compgen -G "$REPORT_LOG_DIR/*.log" > /dev/null; then
+  mkdir -p "$REPORT_LOG_DIR"
+  command cp "$LOG_FILE" "$REPORT_LOG_DIR/0_make.log"
+fi
+
 if [ "${TARGET}" != "finish metadata" ]; then
   exit $ret
 fi

flow/util/genReport.py

@@ -150,7 +150,11 @@ def gen_report(name, data):
         if data["finished"]:
             output += "  Flow reached last stage.\n"
         else:
-            output += f"  Last log file {data['last_log']}\n"
+            output += "  Flow did not reach last stage.\n"
+            if data["last_log"]:
+                output += f"  Last log file {data['last_log']}\n"
+            else:
+                output += "  No log file found.\n"
     else:
         output = ""
 
@@ -302,7 +306,7 @@ def write_summary():
         d["log_warnings"] += temp_w
         if name_.endswith(".log"):
             d["last_log"] = name_
-    d["finished"] = d["last_log"] in LAST_EXPECTED_LOG
+    d["finished"] = any(name in files for name in LAST_EXPECTED_LOG)
 
     # check if metrics generation had issues
     d["metrics_logs_errors"], d["metrics_logs_warnings"] = parse_messages(
@@ -345,9 +349,10 @@ def write_summary():
         if not d["finished"]:
             d["status"] = STATUS_RED
             last_lines = "Could not find last lines of the log file."
-            with open(os.path.join(log_dir, d["last_log"]), "r") as last_file:
-                last_lines = last_file.readlines()[-10:]
-                last_lines = "          ".join(last_lines)
+            if d["last_log"]:
+                with open(os.path.join(log_dir, d["last_log"]), "r") as last_file:
+                    last_lines = last_file.readlines()[-10:]
+                    last_lines = "          ".join(last_lines)
             message = "[ERROR CI-0000] No error message found. "
             message += "Here are the last 10 lines of the log.\n"
             d["log_errors"] = [message + last_lines]