From 5c6022c772b6b8d69ae715595e02eb0b1372cce3 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 14 Oct 2026 09:03:37 -0300 Subject: [PATCH 1/2] fix(report): flag flows that miss the final stage Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 27 +++++++++++++++++++++++++++ flow/util/genReport.py | 1 + 2 files changed, 28 insertions(+) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index 38dafe4643..c47211c82f 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -17,6 +17,33 @@ class TestGenReport(unittest.TestCase): + def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self): + name = "nangate45 gcd (base)" + data = { + "status": genReport.STATUS_RED, + "finished": False, + "last_log": "5_route.log", + "log_errors": [], + "metrics_logs_errors": [], + "metrics_errors": [], + "metrics_regressions": [], + "calibre_errors": [], + "drcs": {}, + } + + report = genReport.gen_report(name, data) + + self.assertEqual( + report, + "\n".join( + [ + name, + " Flow did not reach last stage.", + " Last log file 5_route.log", + ] + ), + ) + def test_parse_qor_regressions_omits_the_optional_heading(self): with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log: log.write("Ignored UTF-8 text: ยต\n") diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 923469d949..5fe55ef31b 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -150,6 +150,7 @@ def gen_report(name, data): if data["finished"]: output += " Flow reached last stage.\n" else: + output += " Flow did not reach last stage.\n" output += f" Last log file {data['last_log']}\n" else: output = "" From 48dec858887188f287b4e691b391d25a4cad6181 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 14 Oct 2026 10:13:30 -0300 Subject: [PATCH 2/2] fix(report): include flows that fail before stage logs Make can fail before it creates the variant log directory. The summary then omits the failed design. Save the Make log when no stage log exists. Detect completion from the expected log, and handle empty log folders. Add CLI regression coverage for the missing-input failure and reruns. Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 125 ++++++++++++++++++++++++++++++++++++ flow/test/test_helper.sh | 10 +++ flow/util/genReport.py | 14 ++-- 3 files changed, 144 insertions(+), 5 deletions(-) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index c47211c82f..ffc171b556 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -2,8 +2,11 @@ import importlib.util import os +import shutil +import subprocess import sys import tempfile +import textwrap import unittest from unittest.mock import patch @@ -16,6 +19,128 @@ SPEC.loader.exec_module(genReport) +class TestReportCollection(unittest.TestCase): + def setUp(self): + self.tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp_dir.cleanup) + self.tmp = self.tmp_dir.name + for folder in ("test", "util", "reports"): + os.makedirs(os.path.join(self.tmp, folder)) + shutil.copy2(GEN_REPORT_PATH, os.path.join(self.tmp, "util/genReport.py")) + self.helper = os.path.join(self.tmp, "test/test_helper.sh") + shutil.copy2( + os.path.join(os.path.dirname(GEN_REPORT_PATH), "../test/test_helper.sh"), + self.helper, + ) + self.write_file( + "designs/sky130hd/gcd/config.mk", + "PLATFORM = sky130hd\nDESIGN_NICKNAME = gcd\n", + ) + # Real Make resolves the run directory and fails on the missing input. + self.write_file( + "Makefile", + textwrap.dedent("""\ + include $(DESIGN_CONFIG) + FLOW_VARIANT ?= base + LOG_DIR = $(WORK_HOME)/logs/$(PLATFORM)/$(DESIGN_NICKNAME)/$(FLOW_VARIANT) + .PHONY: clean_all clean_metadata finish metadata + clean_all clean_metadata: + \t@rm -f $(LOG_DIR)/[1-6]_*.log + finish: missing.v + \t@mkdir -p $(LOG_DIR) + \t@echo Finished > $(LOG_DIR)/6_report.log + metadata: + \t@true + print-%: + \t$(info $*: $($*)) + \t@true + """), + ) + self.env = dict(os.environ, WORK_HOME=self.tmp) + for key in ("FLOW_VARIANT", "MAKE_ISSUE", "SAVE_TO_DB", "RUN_CALIBRE"): + self.env.pop(key, None) + + def write_file(self, path, content): + path = os.path.join(self.tmp, path) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(content) + + def run_helper(self): + return subprocess.run( + ["bash", self.helper, "gcd", "sky130hd"], + env=self.env, + capture_output=True, + text=True, + ) + + def summary(self): + result = subprocess.run( + [sys.executable, os.path.join(self.tmp, "util/genReport.py"), "-sv"], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + with open(os.path.join(self.tmp, "reports/report-summary.log")) as f: + return f.read() + + def test_summary_includes_make_failure_before_the_first_stage(self): + result = self.run_helper() + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertRegex(result.stdout, r"No rule to make target .missing\.v.") + for platform in ("nangate45", "ihp-sg13g2"): + self.write_file(f"logs/{platform}/gcd/base/6_report.log", "Finished\n") + self.write_file(f"reports/{platform}/gcd/base/metadata-check.log", "") + self.write_file( + "reports/nangate45/gcd/base/metadata-check.log", + "[ERROR] nangate45/gcd/base: FAIL\n", + ) + + summary = self.summary() + + self.assertIn("Number of designs: 3", summary) + self.assertIn("nangate45 gcd (base)", summary) + self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) + self.assertRegex(summary, r"No rule to make target .missing\.v.") + self.assertNotIn("ihp-sg13g2 gcd (base)", summary) + + def test_early_failure_uses_the_configured_variant_and_design_nickname(self): + self.write_file( + "designs/sky130hd/gcd/config.mk", + "PLATFORM = sky130hd\nDESIGN_NICKNAME = gcd_alias\nFLOW_VARIANT = alternate\n", + ) + result = self.run_helper() + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn( + "sky130hd gcd_alias (alternate)\n Flow did not reach last stage.", + self.summary(), + ) + + def test_successful_rerun_does_not_keep_the_early_failure(self): + result = self.run_helper() + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.write_file("missing.v", "") + result = self.run_helper() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + summary = self.summary() + self.assertIn("Number of designs: 1", summary) + self.assertNotIn("Failing designs", summary) + + def test_finish_log_is_recognized_with_a_later_auxiliary_log(self): + self.write_file("logs/sky130hd/gcd/base/6_report.log", "Finished\n") + self.write_file("logs/sky130hd/gcd/base/6_write_rc.log", "RC values\n") + summary = self.summary() + self.assertIn("Number of designs: 1", summary) + self.assertNotIn("Failing designs", summary) + + def test_empty_log_directory_is_reported_as_incomplete(self): + os.makedirs(os.path.join(self.tmp, "logs/sky130hd/gcd/base")) + summary = self.summary() + self.assertIn("Number of designs: 1", summary) + self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) + self.assertIn("No log file found.", summary) + + class TestGenReport(unittest.TestCase): def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self): name = "nangate45 gcd (base)" diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh index aa961d703d..3c365f92ff 100755 --- a/flow/test/test_helper.sh +++ b/flow/test/test_helper.sh @@ -86,6 +86,10 @@ fi mkdir -p "$(dirname "$LOG_FILE")" "${__make[@]}" clean_all clean_metadata 2>&1 | tee "$LOG_FILE" +REPORT_LOG_DIR=$("${__make[@]}" --no-print-directory -s print-LOG_DIR) +REPORT_LOG_DIR=${REPORT_LOG_DIR#LOG_DIR: } +rm -f "$REPORT_LOG_DIR/0_make.log" + # turn off abort on error so we can always capture the result set +e @@ -96,6 +100,12 @@ read -r -a TARGET_ARR <<< "$TARGET" # the results ret=$? +# Make can fail on a prerequisite before any stage creates a log. +if [ "$ret" -ne 0 ] && ! compgen -G "$REPORT_LOG_DIR/*.log" > /dev/null; then + mkdir -p "$REPORT_LOG_DIR" + command cp "$LOG_FILE" "$REPORT_LOG_DIR/0_make.log" +fi + if [ "${TARGET}" != "finish metadata" ]; then exit $ret fi diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 5fe55ef31b..52ec6115dd 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -151,7 +151,10 @@ def gen_report(name, data): output += " Flow reached last stage.\n" else: output += " Flow did not reach last stage.\n" - output += f" Last log file {data['last_log']}\n" + if data["last_log"]: + output += f" Last log file {data['last_log']}\n" + else: + output += " No log file found.\n" else: output = "" @@ -303,7 +306,7 @@ def write_summary(): d["log_warnings"] += temp_w if name_.endswith(".log"): d["last_log"] = name_ - d["finished"] = d["last_log"] in LAST_EXPECTED_LOG + d["finished"] = any(name in files for name in LAST_EXPECTED_LOG) # check if metrics generation had issues d["metrics_logs_errors"], d["metrics_logs_warnings"] = parse_messages( @@ -346,9 +349,10 @@ def write_summary(): if not d["finished"]: d["status"] = STATUS_RED last_lines = "Could not find last lines of the log file." - with open(os.path.join(log_dir, d["last_log"]), "r") as last_file: - last_lines = last_file.readlines()[-10:] - last_lines = " ".join(last_lines) + if d["last_log"]: + with open(os.path.join(log_dir, d["last_log"]), "r") as last_file: + last_lines = last_file.readlines()[-10:] + last_lines = " ".join(last_lines) message = "[ERROR CI-0000] No error message found. " message += "Here are the last 10 lines of the log.\n" d["log_errors"] = [message + last_lines]