diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py new file mode 100644 index 0000000000..38dafe4643 --- /dev/null +++ b/flow/test/test_genReport.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +import importlib.util +import os +import sys +import tempfile +import unittest +from unittest.mock import patch + +GEN_REPORT_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "util", "genReport.py" +) +with patch("sys.argv", [GEN_REPORT_PATH]), patch("os.chdir"): + SPEC = importlib.util.spec_from_file_location("genReport", GEN_REPORT_PATH) + genReport = importlib.util.module_from_spec(SPEC) + SPEC.loader.exec_module(genReport) + + +class TestGenReport(unittest.TestCase): + def test_parse_qor_regressions_omits_the_optional_heading(self): + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log: + log.write("Ignored UTF-8 text: µ\n") + log.write( + "QoR regression detected for nangate45/gcd:\n" + " cts__timing__setup__tns -12.2456 " + "(base -6.95522, limit -8.346264)\n" + ) + log.flush() + + self.assertEqual( + genReport.parse_qor_regressions(log.name), + [ + "cts__timing__setup__tns -12.2456 " + "(base -6.95522, limit -8.346264)" + ], + ) + + def test_qor_metrics_are_nested_under_the_failing_run(self): + name = "nangate45 gcd (base)" + data = { + "status": genReport.STATUS_RED, + "finished": True, + "last_log": "", + "log_errors": [], + "metrics_logs_errors": [], + "metrics_errors": [ + "[ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, " + "0 missing (478 metrics sent)", + "[ERROR] Failing runs: nangate45/gcd/base", + ], + "metrics_regressions": [ + "cts__timing__setup__tns -12.2456 " "(base -6.95522, limit -8.346264)", + "cts__timing__setup__ws -0.261172 " "(base -0.156758, limit -0.169258)", + "finish__timing__setup__tns -12.4146 " + "(base -7.19445, limit -8.63334)", + "finish__timing__setup__ws -0.271991 " + "(base -0.163541, limit -0.176041)", + "globalroute__timing__setup__tns -12.7365 " + "(base -7.5407, limit -9.04884)", + "globalroute__timing__setup__ws -0.275208 " + "(base -0.171167, limit -0.183667)", + ], + "calibre_errors": [], + "drcs": {}, + } + + report = genReport.gen_report(name, data) + + self.assertEqual( + report, + "\n".join( + [ + name, + " Flow reached last stage.", + " Found 2 metrics failures.", + " [ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, " + "0 missing (478 metrics sent)", + " cts__timing__setup__tns -12.2456 " + "(base -6.95522, limit -8.346264)", + " cts__timing__setup__ws -0.261172 " + "(base -0.156758, limit -0.169258)", + " finish__timing__setup__tns -12.4146 " + "(base -7.19445, limit -8.63334)", + " finish__timing__setup__ws -0.271991 " + "(base -0.163541, limit -0.176041)", + " globalroute__timing__setup__tns -12.7365 " + "(base -7.5407, limit -9.04884)", + " globalroute__timing__setup__ws -0.275208 " + "(base -0.171167, limit -0.183667)", + " [ERROR] Failing runs: nangate45/gcd/base", + ] + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/flow/util/checkQorMetrics.py b/flow/util/checkQorMetrics.py index 7146c53e50..de16525b58 100755 --- a/flow/util/checkQorMetrics.py +++ b/flow/util/checkQorMetrics.py @@ -99,6 +99,11 @@ def fmt_value(value): return str(value) +def fmt_exact_value(value): + """Render a metric value without rounding away dashboard precision.""" + return "n/a" if value is None else str(value) + + def fmt_delta(value): """Same tolerance as fmt_value, with the sign the delta column wants.""" if value is None: @@ -366,6 +371,19 @@ def render_run(result, verbose): ) ) + failed_metrics = [m for m in metrics if m.get("status") == "fail"] + if failed_metrics: + lines.append( + f"QoR regression detected for {result['platform']}/{result['design']}:" + ) + for metric in sorted(failed_metrics, key=lambda m: m.get("metricName", "")): + lines.append( + f" {metric.get('metricName', '?')} " + f"{fmt_exact_value(metric.get('targetValue'))} " + f"(base {fmt_exact_value(metric.get('baseValue'))}, " + f"limit {fmt_exact_value(metric.get('thresholdValue'))})" + ) + # A design can have hundreds of absent metrics, so name them only on request. absent = sorted( metric.get("metricName", "?") diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 02b898775b..923469d949 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -80,6 +80,37 @@ def parse_messages(filename, print_missing=True): return errors, warnings +def parse_qor_regressions(filename): + """Return failed metric details from a QoR check log, if present.""" + regressions = [] + in_regressions = False + try: + with open(filename, "r", encoding="utf-8") as file: + for line in file: + if line.startswith("QoR regression detected for "): + in_regressions = True + elif in_regressions and line.startswith(" ") and line.strip(): + regressions.append(line.strip()) + elif in_regressions: + break + except OSError: + pass + return regressions + + +def append_metrics_errors(errors, regressions, text): + if not errors: + return text + + text += f" Found {len(errors)} metrics failures.\n" + for error in errors: + text += f" {error}\n" + if error.startswith("[ERROR] ") and ": FAIL " in error: + for regression in regressions: + text += f" {regression}\n" + return text + + def append_text(list_, text, sup, regex, verbose): """ TODO @@ -133,8 +164,8 @@ def gen_report(name, data): REGEX_ERROR, args.verbose, ) - output = append_text( - data["metrics_errors"], output, "metrics failures", REGEX_ERROR, args.verbose + output = append_metrics_errors( + data["metrics_errors"], data["metrics_regressions"], output ) output = append_text( data["calibre_errors"], output, "calibre failures", REGEX_ERROR, args.verbose @@ -283,6 +314,9 @@ def write_summary(): d["metrics_errors"], d["metrics_warnings"] = parse_messages( METRICS_CHECK_FMT.format(report_dir), print_missing=d["finished"] ) + d["metrics_regressions"] = parse_qor_regressions( + METRICS_CHECK_FMT.format(report_dir) + ) # check if calibre was run and if drc check passed d["calibre_errors"], d["calibre_warnings"] = parse_messages(