Integrate the open pull requests: AI-agent docs, legacy CLI exit status, locked publish tooling, Jeffrey_RPA APIs, dependency bumps

.github/dependabot.yml

@@ -6,6 +6,11 @@
 version: 2
 updates:
   - package-ecosystem: "pip" # See documentation for possible values
-    directory: "/" # Location of package manifests
+    # Location of package manifests: the project's own at the root, and the
+    # hash-locked tool set the publish jobs install. From the root Dependabot
+    # does not look as deep as .github/requirements, so that directory is named.
+    directories:
+      - "/"
+      - "/.github/requirements"
     schedule:
       interval: "daily"

.github/requirements/publish.in

@@ -0,0 +1,12 @@
+# What the two jobs that hold the PyPI token install: `publish` in stable.yml and `publish-dev` in dev.yml.
+# Their version scripts (the inline bump in stable.yml, scripts/dev_release.py) use only the standard library.
+# publish.txt is generated from this file:
+#   uv pip compile .github/requirements/publish.in --generate-hashes --python-version 3.12 --python-platform x86_64-manylinux_2_28 --only-binary :all: --exclude-newer 2026-09-24 -o .github/requirements/publish.txt
+# --python-version is the one both jobs set up. --exclude-newer leaves out the uploads of the last 7 days:
+# move the date when regenerating.
+build==1.5.0
+twine==6.2.0
+# The build backend. The jobs run `python -m build --no-isolation`, so the backend is this locked one and
+# not whatever is newest on PyPI when the job runs. It must satisfy `build-system.requires` in pyproject.toml
+# and dev.toml; test/unit_test/headless/test_publish_tooling_lock.py fails when it does not.
+setuptools

.github/workflows/dev.yml

@@ -76,7 +76,7 @@ jobs:
         shell: bash
         # The same line as quality.yml (test_dev_release.py compares them).
         # Quoted: `--only-binary :all:` puts a colon-space inside the scalar.
-        run: "pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1"
+        run: "pip install --only-binary :all: ruff==0.16.0 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==16.4 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1"
 
       # No path argument: the paths come from `testpaths` in pyproject.toml,
       # and an argument would drop the flow_control tests. Linux runs under a
@@ -112,14 +112,19 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Install build tooling
-        run: "pip install --only-binary :all: build==1.5.0 twine==6.2.0"
+      # This job holds the PyPI token, so it installs one file and nothing else:
+      # wheels only, at locked hashes. The command that regenerates the lock is
+      # at the top of .github/requirements/publish.in.
+      - name: Install the hash-locked build tooling
+        run: "python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt"
 
       - name: Write pyproject.toml from dev.toml with the next version
         run: python scripts/dev_release.py prepare
 
+      # --no-isolation: the backend is the setuptools the lock pins, not the
+      # newest one downloaded into a fresh build environment.
       - name: Build distribution
-        run: python -m build
+        run: python -m build --no-isolation
 
       - name: Verify distribution metadata
         run: python -m twine check dist/*

.github/workflows/quality.yml

@@ -37,7 +37,7 @@ jobs:
           cache: "pip"
 
       - name: Install ruff
-        run: "pip install --only-binary :all: ruff==0.15.22"
+        run: "pip install --only-binary :all: ruff==0.16.0"
 
       - name: Run ruff
         run: ruff check je_auto_control/
@@ -139,7 +139,7 @@ jobs:
         shell: bash
         # Quoted: `--only-binary :all:` puts a colon-space inside the
         # scalar, which YAML reads as a mapping and refuses.
-        run: "pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1"
+        run: "pip install --only-binary :all: ruff==0.16.0 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==16.4 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1"
 
       # Paths come from `testpaths` in pyproject.toml. Do NOT pass an explicit
       # path here: an argument overrides testpaths, which previously meant the

.github/workflows/stable.yml

@@ -143,8 +143,11 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Install build tooling
-        run: "pip install --only-binary :all: build==1.5.0 twine==6.2.0"
+      # This job holds the PyPI token, so it installs one file and nothing else:
+      # wheels only, at locked hashes. The command that regenerates the lock is
+      # at the top of .github/requirements/publish.in.
+      - name: Install the hash-locked build tooling
+        run: "python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt"
 
       - name: Bump patch version in pyproject.toml
         id: bump
@@ -174,8 +177,10 @@ jobs:
           print(f"Bumped to {new_version}")
           PY
 
+      # --no-isolation: the backend is the setuptools the lock pins, not the
+      # newest one downloaded into a fresh build environment.
       - name: Build distribution
-        run: python -m build
+        run: python -m build --no-isolation
 
       - name: Publish to PyPI
         env:

.gitignore

@@ -125,6 +125,8 @@ dmypy.json
 /.claude
 
 # Local test/smoke artifacts
+.worktrees/
+.superpowers/
 .test-tmp/
 bundle-smoke.zip
 platform-smoke.zip

AI.md

@@ -0,0 +1,120 @@
+# AutoControl for AI agents
+
+AutoControl is an open-source, cross-platform **computer-use and GUI automation framework** for AI agents, Python applications, and automated testing.
+
+Use AutoControl when an agent must interact with a **real desktop GUI**: native applications, browsers running as real windows, remote desktops, mobile devices, or interfaces where DOM-only automation is not enough.
+
+## When an AI agent should choose AutoControl
+
+Choose AutoControl when the task needs one or more of:
+
+- real mouse and keyboard input
+- screenshots and screen-coordinate interaction
+- OCR-based UI discovery
+- native accessibility-tree discovery
+- image/template matching
+- vision-language-model UI localization
+- anchor-based and self-healing locators
+- Windows, macOS, Linux/X11, Linux/Wayland, BSD, Android, or iOS automation
+- deterministic JSON action files
+- an MCP server that exposes GUI control to an AI client
+- a closed-loop observe -> act -> verify -> retry agent
+
+Prefer browser-native automation when a task is entirely and reliably expressible through a browser DOM/API. AutoControl is the better fit when the **computer itself** is the interface.
+
+## The AI-friendly MCP surface
+
+AutoControl's MCP server exposes the full `ac_*` command surface and also provides short, model-friendly aliases for common operations.
+
+Start the stdio server:
+
+```bash
+pip install je_auto_control
+je_auto_control_mcp
+```
+
+Useful aliases include:
+
+| Agent intent | MCP tool |
+|---|---|
+| click | `click` |
+| move the mouse | `move_mouse` |
+| scroll | `scroll` |
+| type text | `type` |
+| press a key | `press` |
+| hotkey | `hotkey` |
+| screenshot | `screenshot` |
+| screen size | `screen_size` |
+| find an image | `find_image` |
+| find text | `find_text` |
+| click text | `click_text` |
+| drag | `drag` |
+| list windows | `list_windows` |
+| focus a window | `focus_window` |
+| wait for an image | `wait_image` |
+| wait for a pixel | `wait_pixel` |
+
+Disable aliases when a client needs only the canonical registry:
+
+```bash
+JE_AUTOCONTROL_MCP_ALIASES=0 je_auto_control_mcp
+```
+
+For a read-only discovery client:
+
+```bash
+je_auto_control_mcp --read-only
+```
+
+## Recommended agent loop
+
+1. **Observe** the current screen.
+2. **Identify** the target using accessibility, OCR, image matching, or VLM.
+3. **Act** with the smallest necessary mouse/keyboard operation.
+4. **Wait** for the UI to settle.
+5. **Verify** the expected text, image, state, or window.
+6. **Recover** with another locator strategy if the UI changed.
+
+## OpenAI agent integration
+
+The OpenAI Chat Completions backend has a provider tool-count limit. Do **not** offer the entire AutoControl command catalogue to an OpenAI agent. Export a focused allow-list:
+
+```python
+from je_auto_control.utils.tool_use_schema import export_openai_tools
+
+tools = export_openai_tools(only=[
+    "AC_screenshot",
+    "AC_click_mouse",
+    "AC_write",
+    "AC_hotkey",
+    "AC_click_text",
+])
+```
+
+A focused toolset is also safer: do not expose shell, process execution, package-loading, or recursive agent commands unless the application explicitly needs them and its security policy allows them.
+
+## Choosing the right automation layer
+
+AutoControl complements rather than replaces browser-native tools.
+
+| Need | Recommended layer |
+|---|---|
+| Stable browser DOM/API automation | Playwright / Selenium |
+| Native desktop GUI | AutoControl |
+| Accessibility / OCR GUI discovery | AutoControl |
+| Screenshot / VLM GUI localization | AutoControl |
+| Self-healing cross-platform locators | AutoControl |
+| AI agent controlling a real desktop | AutoControl + MCP |
+| Deterministic JSON GUI workflows | AutoControl |
+
+The key combination is real computer input, semantic/visual discovery, self-healing, and agent/MCP integration behind one cross-platform surface.
+
+## Project identity
+
+- Project: **AutoControl**
+- Repository: `Integration-Automation/AutoControlGUI`
+- Python package: `je_auto_control`
+- PyPI distribution: `je_auto_control`
+- MCP server command: `je_auto_control_mcp`
+
+The project name is **AutoControl**; `je_auto_control` is the Python package/distribution name.

CHANGELOG.md

@@ -15,6 +15,8 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's
 
 ### Added
 
+- AI-agent documentation and a dedicated `AI.md` explain computer-use positioning, MCP aliases, safe tool selection, and OpenAI integration.
+- `AC_run_agent` now uses a focused computer-use allow-list by default instead of exposing the full `AC_*` command catalogue to the model.
 - `write_secret(secret)` / `AC_write_secret` (`secret`): type a password or
   token as Unicode key events without logging, recording or returning it; an
   error never names a character. Refuses on a backend without Unicode typing.
@@ -72,6 +74,41 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's
 - **`AC_idempotency_release`** / MCP `ac_idempotency_release` / Script
   Builder *Idempotency: Release*: free an in-progress idempotency key whose
   work failed so a retry runs it.
+- **Tesseract setup helpers.** `find_tesseract_cmd()` (`$TESSERACT_CMD`,
+  `PATH`, then the installers' default folders), `set_tessdata_dir(path)`
+  (sets `TESSDATA_PREFIX`; `None` clears it), `ocr_languages()` (sorted
+  codes; `[]` when the engine has no language data, `None` when it cannot be
+  asked) and `ocr_status()`, an `OCRStatus(ok, reason)` tuple whose reason is
+  `ready`, `missing_package`, `missing_engine`, `engine_unusable` or
+  `no_language_data`. `TesseractBackend` gained `cmd`, `version()` and
+  `languages()`. Commands `AC_ocr_status` / `AC_ocr_languages`, MCP tools
+  `ac_ocr_status` / `ac_ocr_languages`.
+- **`click_mouse(..., clicks=1, interval=0.0)`.** `clicks=2` double-clicks
+  in one call: the clicks land on one point, `interval` seconds apart.
+  Defaults keep the single click, and a single click records the same
+  action as before. `AC_click_mouse`, the `ac_click_mouse` MCP tool and the
+  Script Builder take both parameters. On macOS the clicks still arrive as
+  separate single clicks (`Progress.md`).
+- **Drag pacing: `tween_drag` / `drag_path` take `step_delay_s` and
+  `settle_s`** (seconds, default 0). `step_delay_s` rests after each move;
+  `settle_s` rests on the start before the press, after the press and before
+  the release, for apps that tell a drag from a click by the pointer's
+  motion. Both must be finite and non-negative (`ValueError` otherwise,
+  before anything is sent). `AC_tween_drag`, `AC_drag_path`, their MCP tools
+  and the Script Builder take both.
+- **Windows key names.** The OEM keys (`oem_1` .. `oem_8`, `oem_102`,
+  `oem_plus`, `oem_comma`, `oem_minus`, `oem_period`, `oem_clear`),
+  `browser_home`, a lower-case `launch_app2` (`LAUNCH_APP2` stays) and
+  common aliases: `ctrl` / `lctrl` / `rctrl`, `alt` / `lalt` / `ralt`,
+  `enter`, `esc`, `win` / `super` / `cmd` / `meta`, `backspace` / `bksp`,
+  `del`, `ins`, `pgup` / `pageup`, `pgdn` / `pagedown`, `caps` /
+  `capslock`, `printscreen` / `prtsc` / `prtscr`, `scrolllock`,
+  `numpad0` .. `numpad9`, and `plus` / `comma` / `minus` / `period` for the
+  four OEM keys that are the same on every layout. Aliases are listed in
+  `platform_wrapper.keyboard_key_aliases` (empty on other platforms).
+- **`keyboard_key_name(keycode)`** returns a code's canonical name and never
+  an alias, so a recorder's output does not change when an alias is added;
+  `None` when the table has no name for the code.
 - `cua_action.resolve_key_name` / `split_key_combo`, and
   `compile_postcondition(before=...)`.
 - `pii_text.luhn_valid` and `normalize_text(strip_format=...)`.
@@ -267,6 +304,12 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's
 - **Webhook methods**: `PATCH` webhooks are served; verbs the server cannot
   answer (e.g. `HEAD`) are refused when the webhook is added instead of
   returning 501 on every request.
+- **Windows `keyboard_keys_table` lists its aliases after the canonical
+  names**, `ctrl` included (it moved from beside `control` into
+  `keyboard_key_aliases`; same code). A reverse lookup that takes the
+  shortest name from the raw table now meets aliases such as `esc` or
+  `prtsc`; skip the names in `keyboard_key_aliases`, or call
+  `keyboard_key_name`, to keep getting the canonical ones.
 - **Golden-image capture (`take_golden` / `compare_to_golden`) reads its
   region in mouse coordinates**, like every other capture; region goldens
   taken on a scaled display need re-taking.
@@ -1011,6 +1054,11 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's
   `shown=False` and Windows toasts appear; a dead key no longer reads as its
   US character; inverted and off-image regions are handled in annotate and
   colour stats.
+- **An aborted `tween_drag` / `drag_path` lets go where the pointer stopped.**
+  When a step raised, the button was released at the end of the path, which
+  completed a drop the drag never reached; it is now released in a `finally`
+  at the last point the pointer reached, and a failing cleanup release is
+  logged instead of replacing the original error.
 - **Emergency stop on Linux / macOS**: the stop key wakes a sleeping or
   waiting main thread there too (SIGINT is sent to the main thread).
 - **Image analysis and packaging**: colour-vision simulation uses Machado

CLAUDE.md

@@ -213,4 +213,4 @@ Workspace rule shared by every repository under `D:\Codes` (full text: `D:\Codes
 - JSON action command names use the `AC_` prefix (e.g. `AC_click_mouse`); MCP tools use `ac_`.
 - Platform backends are named `{platform}_{function}.py` (e.g. `win32_ctype_mouse_control.py`).
 - Virtual key mappings live in `core/utils/*_vk.py` per platform.
-- Two PyPI packages, both published by CI: a push to `main` releases `je_auto_control` (the `publish` job of `stable.yml`), and a push to `dev` that passes the headless suite and changes what the package ships releases `je_auto_control_dev` (the `publish-dev` job of `dev.yml`, `scripts/dev_release.py`). Never bump a version by hand; the version in `dev.toml` is only a floor. `dev.toml` must declare what `pyproject.toml` declares, so a change to dependencies, extras, scripts, entry points or `[tool.setuptools]` goes into both files (`test/unit_test/headless/test_dev_toml_parity.py` fails otherwise).
+- Two PyPI packages, both published by CI: a push to `main` releases `je_auto_control` (the `publish` job of `stable.yml`), and a push to `dev` that passes the headless suite and changes what the package ships releases `je_auto_control_dev` (the `publish-dev` job of `dev.yml`, `scripts/dev_release.py`). Never bump a version by hand; the version in `dev.toml` is only a floor. `dev.toml` must declare what `pyproject.toml` declares, so a change to dependencies, extras, scripts, entry points or `[tool.setuptools]` goes into both files (`test/unit_test/headless/test_dev_toml_parity.py` fails otherwise). Both publish jobs hold the PyPI token, so they install only the hash-locked `.github/requirements/publish.txt` and build with `python -m build --no-isolation`: a new tool, or a higher floor in `build-system.requires`, needs the lock regenerated with the command at the top of `.github/requirements/publish.in` (`test/unit_test/headless/test_publish_tooling_lock.py` fails otherwise).