From 1ac06e74ef6458e91d648762c7835d514edb15ba Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 8 Oct 2026 16:01:41 +0800 Subject: [PATCH] Exit 1 from the legacy CLI flags when an action failed The run command already exits 1 when an action failed; the legacy -e, -d and --execute_str flags ended with 0, so TestPioneer and a CI step could not tell a failed run from a clean one. They now read the same failure count. With TEST_PIONEER_ARTIFACT_DIR set, the HTML, JSON and XML reports asked for under a relative name are written below that directory. Without it nothing changes. --- README.md | 6 +- README/README_zh-CN.md | 2 +- README/README_zh-TW.md | 2 +- architecture.md | 2 +- architecture_explore.md | 16 ++-- docs/updates/2026-10.md | 10 +++ docs/updates/README.md | 3 +- je_auto_control/__main__.py | 11 +++ .../generate_report/generate_html_report.py | 3 +- .../generate_report/generate_json_report.py | 3 +- .../generate_report/generate_xml_report.py | 3 +- .../utils/generate_report/report_path.py | 39 +++++++++ .../headless/test_cross_project_contracts.py | 33 +++++++ test/unit_test/headless/test_report_path.py | 85 +++++++++++++++++++ 14 files changed, 202 insertions(+), 16 deletions(-) create mode 100644 je_auto_control/utils/generate_report/report_path.py create mode 100644 test/unit_test/headless/test_report_path.py diff --git a/README.md b/README.md index 69442eea5..5a5ecfd25 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,11 @@ je_auto_control version `--var name=value` is parsed as JSON when possible (`count=10` becomes an int), otherwise kept as a string. `run` exits 1 when any action failed (the run still goes on to the end), so a CI step fails with it. The legacy -`python -m je_auto_control -e file.json` entry point still works. +`python -m je_auto_control -e file.json` entry point still works, and its +`-e`, `-d` and `--execute_str` flags exit 1 the same way. Under +[TestPioneer](https://github.com/Integration-Automation/TestPioneer), which +sets `TEST_PIONEER_ARTIFACT_DIR`, the HTML, JSON and XML reports asked for +under a relative name are written below that directory. --- diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 508c2bc5b..93420b79b 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -183,7 +183,7 @@ je_auto_control version ``` `--var name=value` 会尽量以 JSON 解析（`count=10` 会变成整数），否则视为字符串。 -`run` 只要有任何动作失败就以 1 退出（仍会跑完整份脚本），CI 步骤会随之失败。旧版 `python -m je_auto_control -e file.json` 入口仍然可用。 +`run` 只要有任何动作失败就以 1 退出（仍会跑完整份脚本），CI 步骤会随之失败。旧版 `python -m je_auto_control -e file.json` 入口仍然可用，它的 `-e`、`-d` 与 `--execute_str` 标志也同样在有动作失败时以 1 退出。由 [TestPioneer](https://github.com/Integration-Automation/TestPioneer) 启动时（它会设置 `TEST_PIONEER_ARTIFACT_DIR`），以相对名称要求的 HTML、JSON、XML 报告会写到该目录底下。 --- diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 404537ccd..05ff2e452 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -183,7 +183,7 @@ je_auto_control version ``` `--var name=value` 會盡量以 JSON 解析（`count=10` 會變成整數），否則視為字串。 -`run` 只要有任何動作失敗就以 1 結束（仍會跑完整份腳本），CI 步驟會跟著失敗。舊版 `python -m je_auto_control -e file.json` 進入點仍然可用。 +`run` 只要有任何動作失敗就以 1 結束（仍會跑完整份腳本），CI 步驟會跟著失敗。舊版 `python -m je_auto_control -e file.json` 進入點仍然可用，它的 `-e`、`-d` 與 `--execute_str` 旗標也同樣在有動作失敗時以 1 結束。由 [TestPioneer](https://github.com/Integration-Automation/TestPioneer) 啟動時（它會設定 `TEST_PIONEER_ARTIFACT_DIR`），以相對名稱要求的 HTML、JSON、XML 報告會寫到該目錄底下。 --- diff --git a/architecture.md b/architecture.md index 92c255dd0..11244c6a2 100644 --- a/architecture.md +++ b/architecture.md @@ -131,7 +131,7 @@ wrapper/auto_control_record.record → OS listener (e.g. windows/record/win32_in | --- | --- | --- | | Jeffrey_RPA | Editable install of **this working tree**: uncommitted changes here reach it immediately. Single facade `JeffreyRPA/_gui_control.py`. | Top-level names (e.g. `click_mouse`, `hotkey`, `write`, `screen_size`, `get_pixel`, `post_click_to_window`) and internal paths `je_auto_control.wrapper.auto_control_window`, `je_auto_control.wrapper.auto_control_keyboard.WRITE_CONTROL_KEYS`, `je_auto_control.utils.monitor_layout` (`logical_virtual_rect`, `enumerate_monitors`), and `wrapper.platform_wrapper.keyboard_keys_table` / `mouse_keys_table` — it validates every key name a user types against the keyboard table and reverse-looks-up recorded virtual keys through it, so a name removed there becomes a rejected hotkey over in that repo. | | PyBreeze | Subprocess `python -m je_auto_control --execute_str ` / `--execute_file `; on Windows the JSON string arrives double-encoded. | Legacy CLI flags; also embeds `je_auto_control.gui.main_widget.AutoControlGUIWidget` and calls `record` / `stop_record` in-process. | -| TestPioneer | Optional extra `gui = ["je_auto_control"]`; `parallel_run` starts `python -m je_auto_control --execute_file `. | `execute_action`, `execute_files`, `RecordingThread`; the `--execute_file` flag. | +| TestPioneer | Optional extra `gui = ["je_auto_control"]`; `parallel_run` starts `python -m je_auto_control --execute_file `. It sets `TEST_PIONEER_ARTIFACT_DIR` for the runner. | `execute_action`, `execute_files`, `RecordingThread`; the `--execute_file` flag. | The flag's exit status, 1 when an action failed (`recorded_failures()` in `__main__.py`); the `_success.json` / `_failure.json` pair, written below the artifact directory for a relative name (`utils/generate_report/report_path.py`). | WebRunner | Optional extra `autocontrol = ["je_auto_control>=0.0.224"]`; `je_web_runner/utils/autocontrol_bridge/` (`WR_ac_*`) imports this package only when one of those commands runs. | `je_auto_control.utils.executor.action_executor.executor`: `execute_action(actions, raise_on_error=True)`, whose record values it reads in action order, and `known_commands()`; its native commands send `AC_write` (`write_string`), `AC_write_secret` (`secret`, from `WR_ac_basic_auth`), `AC_type_keyboard` (`keycode`), `AC_get_keyboard_keys_table` (the `enter` or `return` key), `AC_locate_image_center` (`image`, `detect_threshold`) and `AC_click_mouse` (`mouse_keycode`, `x`, `y` in this DPI-aware process's coordinates). It refuses `AC_shell_command`, `AC_execute_process`, `AC_add_package_*`, `AC_execute_action`, `AC_execute_files`, `AC_run_agent` and `AC_web_*` by name, so renaming one of them would let it through WebRunner's bridge. | **Guarded by** `test/unit_test/headless/test_cross_project_contracts.py`: every legacy CLI flag (short and long, run as a diff --git a/architecture_explore.md b/architecture_explore.md index 5d98e0e70..e749bb6d2 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,8 +19,8 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,063 | -| 程式碼總行數 | 157,101 | +| Python 模組總數（含周邊子專案） | 1,064 | +| 程式碼總行數 | 157,154 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 776 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -155,7 +155,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `je_auto_control/__init__.py` | 1,971 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | -| `je_auto_control/__main__.py` | 87 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | +| `je_auto_control/__main__.py` | 98 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | | `je_auto_control/cli.py` | 353 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | | `je_auto_control/api/__init__.py` | 22 | 版本化整合進入點。 | | `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約以此為起點，現已擴到整包（見「設定基線」）。 | @@ -558,7 +558,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 7,674 行。 +> 34 個套件、約 7,716 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -573,7 +573,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/failure_signature/` | 76 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | | `utils/flake_cluster/` | 112 | 以共同失敗 Jaccard 相似度為易碎測試分群 | | `utils/flakiness/` | 151 | 以執行歷史分析不穩定測試 | -| `utils/generate_report/` | 294 | HTML／JSON／XML 三種報表產生器（Template Method） | +| `utils/generate_report/` | 336 | HTML／JSON／XML 三種報表產生器（Template Method） | | `utils/media_assert/` | 273 | 媒體斷言：音訊活動與影片動態檢查 | | `utils/observability/` | 710 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | | `utils/otlp_export/` | 114 | OTLP/JSON span 匯出 | @@ -1077,7 +1077,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/remote_desktop/` | 56 | 13,014 | | `utils/executor/` | 8 | 9,560 | | `utils/usb/` | 17 | 4,572 | -| `je_auto_control/`（頂層 3 檔） | 3 | 2,411 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,422 | | `utils/accessibility/` | 14 | 3,143 | | `wrapper/` | 19 | 3,667 | | `windows/` | 23 | 1,959 | @@ -1092,6 +1092,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 925 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 852 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 681 | 57,038 | -| **總計** | **1,057** | **157,036** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 682 | 57,080 | +| **總計** | **1,058** | **157,089** | diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 344fd9335..aef575f54 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -103,3 +103,13 @@ WebRunner's native commands (its U-20261001-28 and -29), `WR_ac_fill_native_file - **Docs**: `architecture.md` (§2 `scripts/` row, §3 PyPI packages row), `architecture_explore.md` (§5.6 `scripts/dev_release.py` row, §7 `dev.yml` row), `CLAUDE.md` › Key Conventions, `CHANGELOG.md` (Changed). No README or installation page mentions the dev package, so none changed. - **Files**: `.github/workflows/dev.yml`, `scripts/dev_release.py`, `dev.toml`, `test/unit_test/headless/{test_dev_release,test_dev_toml_parity}.py`, `architecture.md`, `architecture_explore.md`, `CLAUDE.md`, `CHANGELOG.md`. - **Open items**: the points under **Not running yet**. `Progress.md` is not part of this change, so it does not list them. + +## U-20261008-01 · 2026-10-08 · The legacy CLI flags exit 1 when an action failed; reports go below TestPioneer's artifact directory · #feature #cli #report #testpioneer + +- **What**: two things TestPioneer needs from a runner (its `docs/artifacts.rst`, "Native Support"). + - **Exit status**: `python -m je_auto_control` with `-e`, `-d` or `--execute_str` ended with status 0 when an action of the script failed. `__main__.py` now resets and reads `recorded_failures()`, the count the `run` command already uses, and exits 1 with `error: N action(s) failed`. The rest of the script still runs first. + - **Report location**: with `TEST_PIONEER_ARTIFACT_DIR` set, the HTML, JSON and XML reports asked for under a relative name are written below that directory (`utils/generate_report/report_path.py`). An absolute name, a name that leaves the directory and a directory that cannot be created fall back to the name as given. Without the variable nothing changes. +- **Effect on callers**: a script with a failing action ends with status 1 where it ended with 0. PyBreeze only displays the status; TestPioneer's `parallel_run` reports the runner as failed. +- **Tests**: `test/unit_test/headless/test_cross_project_contracts.py` +3 (real child processes), `test/unit_test/headless/test_report_path.py`. +- **Docs**: `README.md`, `README/README_zh-TW.md`, `README/README_zh-CN.md`, `architecture.md`. +- **Files**: `je_auto_control/__main__.py`, `je_auto_control/utils/generate_report/` (`report_path.py` new; three writers), the tests and docs above. diff --git a/docs/updates/README.md b/docs/updates/README.md index 08d1cd29c..8064d8222 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261008-01 | 2026-10-08 | The legacy CLI flags exit 1 when an action failed; reports go below TestPioneer's artifact directory | #feature #cli #report #testpioneer | [2026-10](2026-10.md) | | U-20261001-09 | 2026-10-01 | CI publishes je_auto_control_dev from the dev branch; dev.toml says what pyproject.toml says | #release #ci #X-13 | [2026-10](2026-10.md) | | U-20261001-08 | 2026-10-01 | Package gate in front of AC_add_package_to_executor | #security #X-12 | [2026-10](2026-10.md) | | U-20261001-07 | 2026-10-01 | write_secret / AC_write_secret: type a password without logging, recording or returning it | #done #keyboard #security #webrunner | [2026-10](2026-10.md) | @@ -341,7 +342,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-10.md](2026-10.md) | 2026-10 | 5 | +| [2026-10.md](2026-10.md) | 2026-10 | 6 | | [2026-09-e.md](2026-09-e.md) | 2026-09 | 7 | | [2026-09-d.md](2026-09-d.md) | 2026-09 | 55 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | diff --git a/je_auto_control/__main__.py b/je_auto_control/__main__.py index ecb48f963..a5b98339d 100644 --- a/je_auto_control/__main__.py +++ b/je_auto_control/__main__.py @@ -11,6 +11,9 @@ ) from je_auto_control.utils.executor.action_executor import execute_action from je_auto_control.utils.executor.action_executor import execute_files +from je_auto_control.utils.executor.action_executor import ( + recorded_failures, reset_recorded_failures, +) from je_auto_control.utils.file_process.get_dir_file_list import \ get_dir_files_as_list from je_auto_control.utils.json.json_file import read_executable_action_json @@ -63,6 +66,7 @@ def preprocess_read_str_execute_action(execute_str: str): type=str, help="execute json str" ) parsed = vars(parser.parse_args()) + reset_recorded_failures() for key, value in parsed.items(): if value is None: continue @@ -73,6 +77,13 @@ def preprocess_read_str_execute_action(execute_str: str): handler(value) if all(value is None for value in parsed.values()): raise AutoControlArgparseException(argparse_get_wrong_data_error_message) + # Failed actions are recorded and the run goes on, as with ``run``; + # exiting 0 anyway left TestPioneer and a CI step unable to tell a + # failed run from a clean one. + failures = recorded_failures() + if failures: + sys.stderr.write(f"error: {failures} action(s) failed\n") + sys.exit(1) # The message also goes to stderr: it went only to the log file, so a # failed -e printed nothing at all. except AutoControlArgparseException as error: diff --git a/je_auto_control/utils/generate_report/generate_html_report.py b/je_auto_control/utils/generate_report/generate_html_report.py index db0d4dc96..e70962a08 100644 --- a/je_auto_control/utils/generate_report/generate_html_report.py +++ b/je_auto_control/utils/generate_report/generate_html_report.py @@ -3,6 +3,7 @@ from je_auto_control.utils.exception.exception_tags import html_generate_no_data_tag_error_message from je_auto_control.utils.exception.exceptions import AutoControlHTMLException +from je_auto_control.utils.generate_report.report_path import report_path from je_auto_control.utils.json_store.json_store import atomic_write_text from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.test_record.record_test_class import test_record_instance @@ -166,7 +167,7 @@ def generate_html_report(html_name: str = "default_name") -> None: with _lock: # 使用 with 確保 Lock 正確釋放 Ensure lock is released properly try: - atomic_write_text(html_name + ".html", new_html_string) + atomic_write_text(report_path(html_name + ".html"), new_html_string) except OSError as error: # Only logging it made a missing report look like a written one. raise AutoControlHTMLException( diff --git a/je_auto_control/utils/generate_report/generate_json_report.py b/je_auto_control/utils/generate_report/generate_json_report.py index 2c552c8bd..357014c47 100644 --- a/je_auto_control/utils/generate_report/generate_json_report.py +++ b/je_auto_control/utils/generate_report/generate_json_report.py @@ -3,6 +3,7 @@ from je_auto_control.utils.exception.exception_tags import cant_generate_json_report_error_message from je_auto_control.utils.exception.exceptions import AutoControlGenerateJsonReportException +from je_auto_control.utils.generate_report.report_path import report_path from je_auto_control.utils.json_store.json_store import atomic_write_text from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.test_record.record_test_class import test_record_instance @@ -44,7 +45,7 @@ def generate_json() -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, str] def _write_json_file(file_name: str, data: Dict[str, Dict[str, str]]) -> None: """Write ``data`` atomically; raise on failure instead of only logging it.""" try: - atomic_write_text(file_name, json.dumps(data, indent=4, ensure_ascii=False)) + atomic_write_text(report_path(file_name), json.dumps(data, indent=4, ensure_ascii=False)) except (OSError, TypeError, ValueError) as error: raise AutoControlGenerateJsonReportException( f"cannot write report {file_name!r}: {error!r}") from error diff --git a/je_auto_control/utils/generate_report/generate_xml_report.py b/je_auto_control/utils/generate_report/generate_xml_report.py index 1e250e5ea..51a0a9ffc 100644 --- a/je_auto_control/utils/generate_report/generate_xml_report.py +++ b/je_auto_control/utils/generate_report/generate_xml_report.py @@ -4,6 +4,7 @@ from je_auto_control.utils.exception.exceptions import XMLException from je_auto_control.utils.generate_report.generate_json_report import generate_json +from je_auto_control.utils.generate_report.report_path import report_path from je_auto_control.utils.json_store.json_store import atomic_write_text from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.xml.change_xml_structure.change_xml_structure import dict_to_elements_tree @@ -33,7 +34,7 @@ def _write_xml_file(file_name: str, xml_content: str) -> None: on as if the report existed. """ try: - atomic_write_text(file_name, xml_content) + atomic_write_text(report_path(file_name), xml_content) except OSError as error: raise XMLException(f"cannot write report {file_name!r}: {error!r}") from error diff --git a/je_auto_control/utils/generate_report/report_path.py b/je_auto_control/utils/generate_report/report_path.py new file mode 100644 index 000000000..009f28567 --- /dev/null +++ b/je_auto_control/utils/generate_report/report_path.py @@ -0,0 +1,39 @@ +"""Where a report file is written. + +TestPioneer starts a runner with ``TEST_PIONEER_ARTIFACT_DIR`` set to the +directory that run's output belongs in. A report asked for under a relative +name goes there, so the workflow does not have to say which files to collect. +Outside TestPioneer nothing changes. +""" +import os +from pathlib import Path + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +# Set by TestPioneer for every runner it starts, in a child process and in-process alike. +ARTIFACT_DIR_VARIABLE: str = "TEST_PIONEER_ARTIFACT_DIR" + + +def report_path(name: str) -> str: + """Return where the report called ``name`` is written. + + With ``TEST_PIONEER_ARTIFACT_DIR`` set, a relative ``name`` is placed below + that directory and the folders it needs are created. ``name`` is returned + as given when the variable is not set, when ``name`` is absolute, when it + would leave the directory (``../x``), and when the folders cannot be + created. + """ + directory = os.environ.get(ARTIFACT_DIR_VARIABLE, "").strip() + if not directory or Path(name).is_absolute(): + return name + base = Path(directory).resolve() + target = (base / name).resolve() + if base not in target.parents: + return name + try: + target.parent.mkdir(parents=True, exist_ok=True) + except OSError as error: + autocontrol_logger.error( + "report_path: %r, the artifact directory is not usable: %r", name, error) + return name + return str(target) diff --git a/test/unit_test/headless/test_cross_project_contracts.py b/test/unit_test/headless/test_cross_project_contracts.py index 3b5b949e6..99330aa25 100644 --- a/test/unit_test/headless/test_cross_project_contracts.py +++ b/test/unit_test/headless/test_cross_project_contracts.py @@ -104,6 +104,39 @@ def test_create_project(tmp_path, flag): assert keywords, result.stderr[-2000:] +def _failing_actions(target: Path) -> list: + """An action that fails when it runs, then one that writes ``target``. + + The first writes below a path that is a file, so it passes the check of the + script and raises only at run time. + """ + not_a_directory = target.with_name(target.name + ".blocker") + not_a_directory.write_text("x", encoding="utf-8") + return [["AC_export_sarif", {"findings": [], "path": str(not_a_directory / "out.sarif")}], + *_actions(target)] + + +@pytest.mark.parametrize("flag", ["-e", "--execute_file"]) +def test_a_failed_action_exits_1_after_the_rest_ran(tmp_path, flag): + target = tmp_path / "out.sarif" + action_file = tmp_path / "actions.json" + action_file.write_text(json.dumps(_failing_actions(target)), encoding="utf-8") + result = _run_cli(tmp_path, flag, str(action_file)) + assert result.returncode == 1, result.stderr[-2000:] + assert "error: 1 action(s) failed" in result.stderr + assert target.is_file(), result.stderr[-2000:] + + +def test_a_failed_action_in_a_directory_exits_1(tmp_path): + action_dir = tmp_path / "actions" + action_dir.mkdir() + (action_dir / "a_ok.json").write_text( + json.dumps(_actions(tmp_path / "a.sarif")), encoding="utf-8") + (action_dir / "b_bad.json").write_text( + json.dumps(_failing_actions(tmp_path / "b.sarif")), encoding="utf-8") + assert _run_cli(tmp_path, "--execute_dir", str(action_dir)).returncode == 1 + + def test_no_flag_exits_non_zero(tmp_path): """A caller that forgot its flag must see a failure, not a silent 0.""" assert _run_cli(tmp_path).returncode != 0 diff --git a/test/unit_test/headless/test_report_path.py b/test/unit_test/headless/test_report_path.py new file mode 100644 index 000000000..a47716f4e --- /dev/null +++ b/test/unit_test/headless/test_report_path.py @@ -0,0 +1,85 @@ +"""Under TestPioneer a report asked for under a relative name goes below ``TEST_PIONEER_ARTIFACT_DIR``.""" +from pathlib import Path + +import pytest + +from je_auto_control.utils.generate_report.generate_html_report import generate_html_report +from je_auto_control.utils.generate_report.generate_json_report import generate_json_report +from je_auto_control.utils.generate_report.generate_xml_report import generate_xml_report +from je_auto_control.utils.generate_report.report_path import ARTIFACT_DIR_VARIABLE, report_path +from je_auto_control.utils.test_record.record_test_class import test_record_instance + + +@pytest.fixture(autouse=True) +def _records(): + saved = list(test_record_instance.test_record_list) + test_record_instance.test_record_list[:] = [ + {"function_name": "ok_func", "local_param": None, "time": "t", "program_exception": "None"}, + {"function_name": "bad_func", "local_param": None, "time": "t", "program_exception": "boom"}, + ] + yield + test_record_instance.test_record_list[:] = saved + + +@pytest.fixture +def artifact_dir(tmp_path, monkeypatch): + """A TestPioneer run: the variable is set, and the working directory is somewhere else.""" + directory = tmp_path / "artifacts" / "run-1" / "runners" / "gui-runner" / "01-gui" + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + monkeypatch.setenv(ARTIFACT_DIR_VARIABLE, str(directory)) + return directory + + +def test_a_name_is_returned_as_given_outside_testpioneer(monkeypatch): + monkeypatch.delenv(ARTIFACT_DIR_VARIABLE, raising=False) + assert report_path("reports/gui") == "reports/gui" + + +def test_an_empty_variable_counts_as_unset(monkeypatch): + monkeypatch.setenv(ARTIFACT_DIR_VARIABLE, " ") + assert report_path("reports/gui") == "reports/gui" + + +def test_a_relative_name_goes_below_the_artifact_directory(artifact_dir): + target = Path(report_path("reports/gui_success.json")) + assert target == artifact_dir.resolve() / "reports" / "gui_success.json" + assert target.parent.is_dir() + + +def test_an_absolute_name_is_left_alone(artifact_dir, tmp_path): + name = str(tmp_path / "elsewhere" / "gui.json") + assert report_path(name) == name + assert not artifact_dir.exists() + + +def test_a_name_that_leaves_the_directory_is_left_alone(artifact_dir): + assert report_path("../outside.json") == "../outside.json" + assert not artifact_dir.exists() + + +def test_a_directory_that_cannot_be_created_falls_back_to_the_name(tmp_path, monkeypatch): + not_a_directory = tmp_path / "file" + not_a_directory.write_text("x", encoding="utf-8") + monkeypatch.setenv(ARTIFACT_DIR_VARIABLE, str(not_a_directory)) + assert report_path("reports/gui.json") == "reports/gui.json" + + +@pytest.mark.parametrize("generate, written", [ + (generate_json_report, ["reports/gui_success.json", "reports/gui_failure.json"]), + (generate_xml_report, ["reports/gui_success.xml", "reports/gui_failure.xml"]), + (generate_html_report, ["reports/gui.html"]), +], ids=["json", "xml", "html"]) +def test_a_report_is_written_below_the_artifact_directory(artifact_dir, generate, written): + generate("reports/gui") + assert [(artifact_dir / path).is_file() for path in written] == [True] * len(written) + # Nothing is left in the working directory, where the name used to point. + assert not Path("reports").exists() + + +def test_outside_testpioneer_a_report_stays_in_the_working_directory(tmp_path, monkeypatch): + monkeypatch.delenv(ARTIFACT_DIR_VARIABLE, raising=False) + monkeypatch.chdir(tmp_path) + generate_json_report("gui") + assert (tmp_path / "gui_success.json").is_file()