diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index 38dafe4643..ffc171b556 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -2,8 +2,11 @@ import importlib.util import os +import shutil +import subprocess import sys import tempfile +import textwrap import unittest from unittest.mock import patch @@ -16,7 +19,156 @@ SPEC.loader.exec_module(genReport) +class TestReportCollection(unittest.TestCase): + def setUp(self): + self.tmp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp_dir.cleanup) + self.tmp = self.tmp_dir.name + for folder in ("test", "util", "reports"): + os.makedirs(os.path.join(self.tmp, folder)) + shutil.copy2(GEN_REPORT_PATH, os.path.join(self.tmp, "util/genReport.py")) + self.helper = os.path.join(self.tmp, "test/test_helper.sh") + shutil.copy2( + os.path.join(os.path.dirname(GEN_REPORT_PATH), "../test/test_helper.sh"), + self.helper, + ) + self.write_file( + "designs/sky130hd/gcd/config.mk", + "PLATFORM = sky130hd\nDESIGN_NICKNAME = gcd\n", + ) + # Real Make resolves the run directory and fails on the missing input. + self.write_file( + "Makefile", + textwrap.dedent("""\ + include $(DESIGN_CONFIG) + FLOW_VARIANT ?= base + LOG_DIR = $(WORK_HOME)/logs/$(PLATFORM)/$(DESIGN_NICKNAME)/$(FLOW_VARIANT) + .PHONY: clean_all clean_metadata finish metadata + clean_all clean_metadata: + \t@rm -f $(LOG_DIR)/[1-6]_*.log + finish: missing.v + \t@mkdir -p $(LOG_DIR) + \t@echo Finished > $(LOG_DIR)/6_report.log + metadata: + \t@true + print-%: + \t$(info $*: $($*)) + \t@true + """), + ) + self.env = dict(os.environ, WORK_HOME=self.tmp) + for key in ("FLOW_VARIANT", "MAKE_ISSUE", "SAVE_TO_DB", "RUN_CALIBRE"): + self.env.pop(key, None) + + def write_file(self, path, content): + path = os.path.join(self.tmp, path) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(content) + + def run_helper(self): + return subprocess.run( + ["bash", self.helper, "gcd", "sky130hd"], + env=self.env, + capture_output=True, + text=True, + ) + + def summary(self): + result = subprocess.run( + [sys.executable, os.path.join(self.tmp, "util/genReport.py"), "-sv"], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + with open(os.path.join(self.tmp, "reports/report-summary.log")) as f: + return f.read() + + def test_summary_includes_make_failure_before_the_first_stage(self): + result = self.run_helper() + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertRegex(result.stdout, r"No rule to make target .missing\.v.") + for platform in ("nangate45", "ihp-sg13g2"): + self.write_file(f"logs/{platform}/gcd/base/6_report.log", "Finished\n") + self.write_file(f"reports/{platform}/gcd/base/metadata-check.log", "") + self.write_file( + "reports/nangate45/gcd/base/metadata-check.log", + "[ERROR] nangate45/gcd/base: FAIL\n", + ) + + summary = self.summary() + + self.assertIn("Number of designs: 3", summary) + self.assertIn("nangate45 gcd (base)", summary) + self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) + self.assertRegex(summary, r"No rule to make target .missing\.v.") + self.assertNotIn("ihp-sg13g2 gcd (base)", summary) + + def test_early_failure_uses_the_configured_variant_and_design_nickname(self): + self.write_file( + "designs/sky130hd/gcd/config.mk", + "PLATFORM = sky130hd\nDESIGN_NICKNAME = gcd_alias\nFLOW_VARIANT = alternate\n", + ) + result = self.run_helper() + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn( + "sky130hd gcd_alias (alternate)\n Flow did not reach last stage.", + self.summary(), + ) + + def test_successful_rerun_does_not_keep_the_early_failure(self): + result = self.run_helper() + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.write_file("missing.v", "") + result = self.run_helper() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + summary = self.summary() + self.assertIn("Number of designs: 1", summary) + self.assertNotIn("Failing designs", summary) + + def test_finish_log_is_recognized_with_a_later_auxiliary_log(self): + self.write_file("logs/sky130hd/gcd/base/6_report.log", "Finished\n") + self.write_file("logs/sky130hd/gcd/base/6_write_rc.log", "RC values\n") + summary = self.summary() + self.assertIn("Number of designs: 1", summary) + self.assertNotIn("Failing designs", summary) + + def test_empty_log_directory_is_reported_as_incomplete(self): + os.makedirs(os.path.join(self.tmp, "logs/sky130hd/gcd/base")) + summary = self.summary() + self.assertIn("Number of designs: 1", summary) + self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) + self.assertIn("No log file found.", summary) + + class TestGenReport(unittest.TestCase): + def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self): + name = "nangate45 gcd (base)" + data = { + "status": genReport.STATUS_RED, + "finished": False, + "last_log": "5_route.log", + "log_errors": [], + "metrics_logs_errors": [], + "metrics_errors": [], + "metrics_regressions": [], + "calibre_errors": [], + "drcs": {}, + } + + report = genReport.gen_report(name, data) + + self.assertEqual( + report, + "\n".join( + [ + name, + " Flow did not reach last stage.", + " Last log file 5_route.log", + ] + ), + ) + def test_parse_qor_regressions_omits_the_optional_heading(self): with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log: log.write("Ignored UTF-8 text: ยต\n") diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh index aa961d703d..3c365f92ff 100755 --- a/flow/test/test_helper.sh +++ b/flow/test/test_helper.sh @@ -86,6 +86,10 @@ fi mkdir -p "$(dirname "$LOG_FILE")" "${__make[@]}" clean_all clean_metadata 2>&1 | tee "$LOG_FILE" +REPORT_LOG_DIR=$("${__make[@]}" --no-print-directory -s print-LOG_DIR) +REPORT_LOG_DIR=${REPORT_LOG_DIR#LOG_DIR: } +rm -f "$REPORT_LOG_DIR/0_make.log" + # turn off abort on error so we can always capture the result set +e @@ -96,6 +100,12 @@ read -r -a TARGET_ARR <<< "$TARGET" # the results ret=$? +# Make can fail on a prerequisite before any stage creates a log. +if [ "$ret" -ne 0 ] && ! compgen -G "$REPORT_LOG_DIR/*.log" > /dev/null; then + mkdir -p "$REPORT_LOG_DIR" + command cp "$LOG_FILE" "$REPORT_LOG_DIR/0_make.log" +fi + if [ "${TARGET}" != "finish metadata" ]; then exit $ret fi diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 923469d949..52ec6115dd 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -150,7 +150,11 @@ def gen_report(name, data): if data["finished"]: output += " Flow reached last stage.\n" else: - output += f" Last log file {data['last_log']}\n" + output += " Flow did not reach last stage.\n" + if data["last_log"]: + output += f" Last log file {data['last_log']}\n" + else: + output += " No log file found.\n" else: output = "" @@ -302,7 +306,7 @@ def write_summary(): d["log_warnings"] += temp_w if name_.endswith(".log"): d["last_log"] = name_ - d["finished"] = d["last_log"] in LAST_EXPECTED_LOG + d["finished"] = any(name in files for name in LAST_EXPECTED_LOG) # check if metrics generation had issues d["metrics_logs_errors"], d["metrics_logs_warnings"] = parse_messages( @@ -345,9 +349,10 @@ def write_summary(): if not d["finished"]: d["status"] = STATUS_RED last_lines = "Could not find last lines of the log file." - with open(os.path.join(log_dir, d["last_log"]), "r") as last_file: - last_lines = last_file.readlines()[-10:] - last_lines = " ".join(last_lines) + if d["last_log"]: + with open(os.path.join(log_dir, d["last_log"]), "r") as last_file: + last_lines = last_file.readlines()[-10:] + last_lines = " ".join(last_lines) message = "[ERROR CI-0000] No error message found. " message += "Here are the last 10 lines of the log.\n" d["log_errors"] = [message + last_lines]