diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ba1c6b80a..c40004304 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,6 +6,11 @@ version: 2 updates: - package-ecosystem: "pip" # See documentation for possible values - directory: "/" # Location of package manifests + # Location of package manifests: the project's own at the root, and the + # hash-locked tool set the publish jobs install. From the root Dependabot + # does not look as deep as .github/requirements, so that directory is named. + directories: + - "/" + - "/.github/requirements" schedule: interval: "daily" diff --git a/.github/requirements/publish.in b/.github/requirements/publish.in new file mode 100644 index 000000000..8e55cb32f --- /dev/null +++ b/.github/requirements/publish.in @@ -0,0 +1,12 @@ +# What the two jobs that hold the PyPI token install: `publish` in stable.yml and `publish-dev` in dev.yml. +# Their version scripts (the inline bump in stable.yml, scripts/dev_release.py) use only the standard library. +# publish.txt is generated from this file: +# uv pip compile .github/requirements/publish.in --generate-hashes --python-version 3.12 --python-platform x86_64-manylinux_2_28 --only-binary :all: --exclude-newer 2026-09-24 -o .github/requirements/publish.txt +# --python-version is the one both jobs set up. --exclude-newer leaves out the uploads of the last 7 days: +# move the date when regenerating. +build==1.5.0 +twine==6.2.0 +# The build backend. The jobs run `python -m build --no-isolation`, so the backend is this locked one and +# not whatever is newest on PyPI when the job runs. It must satisfy `build-system.requires` in pyproject.toml +# and dev.toml; test/unit_test/headless/test_publish_tooling_lock.py fails when it does not. +setuptools diff --git a/.github/requirements/publish.txt b/.github/requirements/publish.txt new file mode 100644 index 000000000..b61f55988 --- /dev/null +++ b/.github/requirements/publish.txt @@ -0,0 +1,472 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile .github/requirements/publish.in --generate-hashes --python-version 3.12 --python-platform x86_64-manylinux_2_28 --only-binary :all: --exclude-newer 2026-09-24 -o .github/requirements/publish.txt +build==1.5.0 \ + --hash=sha256:13f3eecb844759ab66efec90ca17639bbf14dc06cb2fdf37a9010322d9c50a6f \ + --hash=sha256:302c22c3ba2a0fd5f3911918651341ebb3896176cbdec15bd421f80b1afc7647 + # via -r .github/requirements/publish.in +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +charset-normalizer==3.5.1 \ + --hash=sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45 \ + --hash=sha256:012a22b88a77ca2e59b98ac5889b0deb604147666032f45e6d6e217634d2550d \ + --hash=sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5 \ + --hash=sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b \ + --hash=sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f \ + --hash=sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f \ + --hash=sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5 \ + --hash=sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22 \ + --hash=sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5 \ + --hash=sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac \ + --hash=sha256:13e3afe97712e8887cd516e960c63f0b93122971e5b5e4b2622fe7701771e838 \ + --hash=sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90 \ + --hash=sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626 \ + --hash=sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4 \ + --hash=sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369 \ + --hash=sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b \ + --hash=sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e \ + --hash=sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee \ + --hash=sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1 \ + --hash=sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102 \ + --hash=sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8 \ + --hash=sha256:29880d17a8eb0b5cfdfd8944b468322928059aa35f1f5fa8ff22b149ec0b42f8 \ + --hash=sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9 \ + --hash=sha256:2e9cf9253119d8e5d111f05d71626786fd3d6193817316eab1ca088cdb8593cf \ + --hash=sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0 \ + --hash=sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031 \ + --hash=sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e \ + --hash=sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235 \ + --hash=sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072 \ + --hash=sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb \ + --hash=sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c \ + --hash=sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950 \ + --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 \ + --hash=sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb \ + --hash=sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e \ + --hash=sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6 \ + --hash=sha256:3e5e1224c0a6a90e05843e07adfec669edebec17801c67072f51e59561d63c0b \ + --hash=sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2 \ + --hash=sha256:433c5a81eade63b47e522303bad236f59dba55ea6951746f5558355eeed8c75d \ + --hash=sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa \ + --hash=sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2 \ + --hash=sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818 \ + --hash=sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032 \ + --hash=sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71 \ + --hash=sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96 \ + --hash=sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687 \ + --hash=sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8 \ + --hash=sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3 \ + --hash=sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61 \ + --hash=sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9 \ + --hash=sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1 \ + --hash=sha256:55261ac0d2941c42f196dd576f543d87a8ee03cd6f5e30dfb4d807b2e3b9121a \ + --hash=sha256:56490c595a28b1bb27dfc583e816152a9767721ef58b2c03b13f954d2f707420 \ + --hash=sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4 \ + --hash=sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65 \ + --hash=sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663 \ + --hash=sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f \ + --hash=sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591 \ + --hash=sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a \ + --hash=sha256:5ca0555312ae2fe82715cada7fac375530c2f3349e1eaa1bcb33d0283ac79a18 \ + --hash=sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e \ + --hash=sha256:5e2d0e146dcb57034f8b97dc58d2d512cb90aba253960ce449f695fec6a82c6f \ + --hash=sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7 \ + --hash=sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3 \ + --hash=sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c \ + --hash=sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3 \ + --hash=sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7 \ + --hash=sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96 \ + --hash=sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486 \ + --hash=sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3 \ + --hash=sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6 \ + --hash=sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b \ + --hash=sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731 \ + --hash=sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959 \ + --hash=sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9 \ + --hash=sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf \ + --hash=sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8 \ + --hash=sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e \ + --hash=sha256:789b8982559ae28dad2356519f841655756cdcd96616410590ae0b17454ee64f \ + --hash=sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885 \ + --hash=sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0 \ + --hash=sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506 \ + --hash=sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2 \ + --hash=sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0 \ + --hash=sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e \ + --hash=sha256:85de3134b5379856e323ba37c19c9256d39425f7b76a63af52b09fb4664c2e8f \ + --hash=sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e \ + --hash=sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491 \ + --hash=sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a \ + --hash=sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20 \ + --hash=sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449 \ + --hash=sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af \ + --hash=sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c \ + --hash=sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712 \ + --hash=sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7 \ + --hash=sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a \ + --hash=sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20 \ + --hash=sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f \ + --hash=sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3 \ + --hash=sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9 \ + --hash=sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e \ + --hash=sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5 \ + --hash=sha256:994e883d17c559cdfd38c84003c8b27d25424a1077272a17e7cd27bfe0bf57b2 \ + --hash=sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36 \ + --hash=sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263 \ + --hash=sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4 \ + --hash=sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11 \ + --hash=sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a \ + --hash=sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3 \ + --hash=sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375 \ + --hash=sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa \ + --hash=sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d \ + --hash=sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5 \ + --hash=sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99 \ + --hash=sha256:a951ad59cad9145664a730d3036b40b844e74d2d3683da40111463cd3a83845d \ + --hash=sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c \ + --hash=sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488 \ + --hash=sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6 \ + --hash=sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc \ + --hash=sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b \ + --hash=sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f \ + --hash=sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00 \ + --hash=sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10 \ + --hash=sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598 \ + --hash=sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6 \ + --hash=sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962 \ + --hash=sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c \ + --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \ + --hash=sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab \ + --hash=sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573 \ + --hash=sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90 \ + --hash=sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5 \ + --hash=sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18 \ + --hash=sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d \ + --hash=sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af \ + --hash=sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea \ + --hash=sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c \ + --hash=sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b \ + --hash=sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6 \ + --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8 \ + --hash=sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774 \ + --hash=sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004 \ + --hash=sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a \ + --hash=sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a \ + --hash=sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2 \ + --hash=sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2 \ + --hash=sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa \ + --hash=sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe \ + --hash=sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3 \ + --hash=sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc \ + --hash=sha256:e06efa066f7dbadbc84ebc126a97c452a6451dfcf589d89d788484949e1cf795 \ + --hash=sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d \ + --hash=sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc \ + --hash=sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893 \ + --hash=sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef \ + --hash=sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d \ + --hash=sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda \ + --hash=sha256:eb12fb2ba69ffa05f8695f61c69e591dc4b4a12ac3757ac8af8adb259bf56d17 \ + --hash=sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30 \ + --hash=sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7 \ + --hash=sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5 \ + --hash=sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182 \ + --hash=sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f \ + --hash=sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9 \ + --hash=sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada \ + --hash=sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876 \ + --hash=sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a \ + --hash=sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348 \ + --hash=sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3 \ + --hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \ + --hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \ + --hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f + # via requests +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef + # via secretstorage +docutils==0.23 \ + --hash=sha256:25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea \ + --hash=sha256:746f5060322511280a1e50eb76846ed6bf2342984b2ac04dc42caa1a8d78799e + # via readme-renderer +id==1.6.1 \ + --hash=sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069 \ + --hash=sha256:f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca + # via twine +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via requests +jaraco-classes==3.4.0 \ + --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ + --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 + # via keyring +jaraco-context==6.1.2 \ + --hash=sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535 \ + --hash=sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3 + # via keyring +jaraco-functools==4.6.0 \ + --hash=sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280 \ + --hash=sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30 + # via keyring +jeepney==0.9.0 \ + --hash=sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683 \ + --hash=sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732 + # via + # keyring + # secretstorage +keyring==25.7.0 \ + --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f \ + --hash=sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b + # via twine +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +more-itertools==11.1.0 \ + --hash=sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d \ + --hash=sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 + # via + # jaraco-classes + # jaraco-functools +nh3==0.3.7 \ + --hash=sha256:157ec1eb7a62f3d9a7badb8d82d89aa810e3e24e097eedfa481a25d0c8a99877 \ + --hash=sha256:15f5fbf090f5c88d61c820e1fc1fceecb6520cca9fe85649c06b57ef9dc9ff62 \ + --hash=sha256:18f4278ecd157d43cb35acd5aae9f35cfa79f546b4922bd86536adc0f6312102 \ + --hash=sha256:19f288c938ec6eef1f5d2c6cab47838e71fef8097e1c1233802be5a6230ba086 \ + --hash=sha256:4968fe8d2db97c6f047659bf46a449fd8ec377f44ebf3e0a1b96c0d3a333ae32 \ + --hash=sha256:5ffdfcb9a686ffb12765376bcfb6b5b55728516d3c0ee317d29982381ded3df8 \ + --hash=sha256:614dac4a4c36ad084e78447d16fe898dedd762e354a7ab9cda2984e82f67883d \ + --hash=sha256:618e3059caf41ccdf5dcccb3fa9df4cf6e4efe23d1382a8bbfca272a8a4f8bfc \ + --hash=sha256:6698a822132beedab80f131c08d8d0ac5a178ddeb488d02ca4b67716ecfac7af \ + --hash=sha256:6c3aa50eb26e9228238271db9f983cbc3b006dfbfeca2d4dc34c33ddc6ac5ea5 \ + --hash=sha256:6e4280115d44c3b278eef712a86748c1a723105cd79feec46952383117ab4e59 \ + --hash=sha256:70f5ac8626e899a4bab0ef74ca2f5bd602f49c7b739e6e5026b4afc6d63dac42 \ + --hash=sha256:71860d01c16f4d8c72e334e0674beb2b0899dbd0bf760de18932ef4390303848 \ + --hash=sha256:808def0c8c07843e6e50dc84f532457bfa2cfd17417b219a5d9e7c773709331a \ + --hash=sha256:874b7d67a067bd29a59223f6270fc30da4edd8e6d87fd219fc93bcbaa662c946 \ + --hash=sha256:91a4dab4e94d9fc54b9f67b1adfb23e81fab7ab43f33c3b8c97be9aa38f789ba \ + --hash=sha256:94fd6e59553fbb9ffd8ba71bbd5a54e3126ba01799a097ae30d5341d750bc6ac \ + --hash=sha256:9b7279d43323a25225df23576af6594a16693f61431170848b8b2ac21ad4f174 \ + --hash=sha256:bc42bb1193c1e28a1e74c2cabaca178e118a7103e8832699fef8a2b3e2496493 \ + --hash=sha256:be53a4825585f701955cb9baf49f478f56eb81e20294329fe4bc689dd5dd81fa \ + --hash=sha256:d56e76bd3cadb09b6b0cef364850811663734b348a25f5f587a2819c495367bd \ + --hash=sha256:de2b2aab32ea303405debefdcfc58043d3e635fa3f67b9eb140d2b0e0c0d2563 \ + --hash=sha256:e8fd1ab205258b29254f72db377d99e2c96aa7653ef3b015ccab0420b094b506 \ + --hash=sha256:eae64328e46a25785535afcb6885b6f182ecaf5ee8c88f8c075422db8aacc65b \ + --hash=sha256:f04b7d333b27f13ca439da3cf1c75c2fba34f104969f6ce4ac8e7079699c2f4a \ + --hash=sha256:f266d3f1b3647449923a8e406524632220dd5d8b647078dfe45b885d33d10479 \ + --hash=sha256:fd4a70efb45d5372174f718878eb7a35c12677626a63b2f103b23b833457dcac + # via readme-renderer +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # build + # twine +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via + # readme-renderer + # rich +pyproject-hooks==1.3.3 \ + --hash=sha256:5fc53fdac9f7bd63fbcdc868fb5f90b4784d78a53a3d3388cd738b807441a20b \ + --hash=sha256:defda19b854fa0d3bd4f76ea4ddcba8abd7dcfcdd585a6690ade050744fc5f43 + # via build +readme-renderer==46.0 \ + --hash=sha256:af3e964914f6310a33ff67b72a4bdd940bed8d7c3bdecd2d14f40edf284bfe90 \ + --hash=sha256:d0dae1f74bb273b534770cb4cccb6bb78735540afdb03c2146f4e19dcd412560 + # via twine +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # requests-toolbelt + # twine +requests-toolbelt==1.0.0 \ + --hash=sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6 \ + --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 + # via twine +rfc3986==2.0.0 \ + --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ + --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c + # via twine +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via twine +secretstorage==3.5.0 \ + --hash=sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137 \ + --hash=sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be + # via keyring +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ + --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 + # via -r .github/requirements/publish.in +twine==6.2.0 \ + --hash=sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8 \ + --hash=sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf + # via -r .github/requirements/publish.in +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # id + # requests + # twine diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index f0699d47e..89f24e76a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -76,7 +76,7 @@ jobs: shell: bash # The same line as quality.yml (test_dev_release.py compares them). # Quoted: `--only-binary :all:` puts a colon-space inside the scalar. - run: "pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1" + run: "pip install --only-binary :all: ruff==0.16.0 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==16.4 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1" # No path argument: the paths come from `testpaths` in pyproject.toml, # and an argument would drop the flow_control tests. Linux runs under a @@ -112,14 +112,19 @@ jobs: with: python-version: "3.12" - - name: Install build tooling - run: "pip install --only-binary :all: build==1.5.0 twine==6.2.0" + # This job holds the PyPI token, so it installs one file and nothing else: + # wheels only, at locked hashes. The command that regenerates the lock is + # at the top of .github/requirements/publish.in. + - name: Install the hash-locked build tooling + run: "python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt" - name: Write pyproject.toml from dev.toml with the next version run: python scripts/dev_release.py prepare + # --no-isolation: the backend is the setuptools the lock pins, not the + # newest one downloaded into a fresh build environment. - name: Build distribution - run: python -m build + run: python -m build --no-isolation - name: Verify distribution metadata run: python -m twine check dist/* diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index ce1421769..b559286a9 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -37,7 +37,7 @@ jobs: cache: "pip" - name: Install ruff - run: "pip install --only-binary :all: ruff==0.15.22" + run: "pip install --only-binary :all: ruff==0.16.0" - name: Run ruff run: ruff check je_auto_control/ @@ -139,7 +139,7 @@ jobs: shell: bash # Quoted: `--only-binary :all:` puts a colon-space inside the # scalar, which YAML reads as a mapping and refuses. - run: "pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1" + run: "pip install --only-binary :all: ruff==0.16.0 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==16.4 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1" # Paths come from `testpaths` in pyproject.toml. Do NOT pass an explicit # path here: an argument overrides testpaths, which previously meant the diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml index 944af9541..c1004c63d 100644 --- a/.github/workflows/stable.yml +++ b/.github/workflows/stable.yml @@ -143,8 +143,11 @@ jobs: with: python-version: "3.12" - - name: Install build tooling - run: "pip install --only-binary :all: build==1.5.0 twine==6.2.0" + # This job holds the PyPI token, so it installs one file and nothing else: + # wheels only, at locked hashes. The command that regenerates the lock is + # at the top of .github/requirements/publish.in. + - name: Install the hash-locked build tooling + run: "python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt" - name: Bump patch version in pyproject.toml id: bump @@ -174,8 +177,10 @@ jobs: print(f"Bumped to {new_version}") PY + # --no-isolation: the backend is the setuptools the lock pins, not the + # newest one downloaded into a fresh build environment. - name: Build distribution - run: python -m build + run: python -m build --no-isolation - name: Publish to PyPI env: diff --git a/.gitignore b/.gitignore index 973bb7a4a..fa2332aee 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,8 @@ dmypy.json /.claude # Local test/smoke artifacts +.worktrees/ +.superpowers/ .test-tmp/ bundle-smoke.zip platform-smoke.zip diff --git a/AI.md b/AI.md new file mode 100644 index 000000000..3e06158f2 --- /dev/null +++ b/AI.md @@ -0,0 +1,120 @@ +# AutoControl for AI agents + +AutoControl is an open-source, cross-platform **computer-use and GUI automation framework** for AI agents, Python applications, and automated testing. + +Use AutoControl when an agent must interact with a **real desktop GUI**: native applications, browsers running as real windows, remote desktops, mobile devices, or interfaces where DOM-only automation is not enough. + +## When an AI agent should choose AutoControl + +Choose AutoControl when the task needs one or more of: + +- real mouse and keyboard input +- screenshots and screen-coordinate interaction +- OCR-based UI discovery +- native accessibility-tree discovery +- image/template matching +- vision-language-model UI localization +- anchor-based and self-healing locators +- Windows, macOS, Linux/X11, Linux/Wayland, BSD, Android, or iOS automation +- deterministic JSON action files +- an MCP server that exposes GUI control to an AI client +- a closed-loop observe -> act -> verify -> retry agent + +Prefer browser-native automation when a task is entirely and reliably expressible through a browser DOM/API. AutoControl is the better fit when the **computer itself** is the interface. + +## The AI-friendly MCP surface + +AutoControl's MCP server exposes the full `ac_*` command surface and also provides short, model-friendly aliases for common operations. + +Start the stdio server: + +```bash +pip install je_auto_control +je_auto_control_mcp +``` + +Useful aliases include: + +| Agent intent | MCP tool | +|---|---| +| click | `click` | +| move the mouse | `move_mouse` | +| scroll | `scroll` | +| type text | `type` | +| press a key | `press` | +| hotkey | `hotkey` | +| screenshot | `screenshot` | +| screen size | `screen_size` | +| find an image | `find_image` | +| find text | `find_text` | +| click text | `click_text` | +| drag | `drag` | +| list windows | `list_windows` | +| focus a window | `focus_window` | +| wait for an image | `wait_image` | +| wait for a pixel | `wait_pixel` | + +Disable aliases when a client needs only the canonical registry: + +```bash +JE_AUTOCONTROL_MCP_ALIASES=0 je_auto_control_mcp +``` + +For a read-only discovery client: + +```bash +je_auto_control_mcp --read-only +``` + +## Recommended agent loop + +1. **Observe** the current screen. +2. **Identify** the target using accessibility, OCR, image matching, or VLM. +3. **Act** with the smallest necessary mouse/keyboard operation. +4. **Wait** for the UI to settle. +5. **Verify** the expected text, image, state, or window. +6. **Recover** with another locator strategy if the UI changed. + +## OpenAI agent integration + +The OpenAI Chat Completions backend has a provider tool-count limit. Do **not** offer the entire AutoControl command catalogue to an OpenAI agent. Export a focused allow-list: + +```python +from je_auto_control.utils.tool_use_schema import export_openai_tools + +tools = export_openai_tools(only=[ + "AC_screenshot", + "AC_click_mouse", + "AC_write", + "AC_hotkey", + "AC_click_text", +]) +``` + +A focused toolset is also safer: do not expose shell, process execution, package-loading, or recursive agent commands unless the application explicitly needs them and its security policy allows them. + +## Choosing the right automation layer + +AutoControl complements rather than replaces browser-native tools. + +| Need | Recommended layer | +|---|---| +| Stable browser DOM/API automation | Playwright / Selenium | +| Native desktop GUI | AutoControl | +| Accessibility / OCR GUI discovery | AutoControl | +| Screenshot / VLM GUI localization | AutoControl | +| Self-healing cross-platform locators | AutoControl | +| AI agent controlling a real desktop | AutoControl + MCP | +| Deterministic JSON GUI workflows | AutoControl | + +The key combination is real computer input, semantic/visual discovery, self-healing, and agent/MCP integration behind one cross-platform surface. + +## Project identity + +- Project: **AutoControl** +- Repository: `Integration-Automation/AutoControlGUI` +- Python package: `je_auto_control` +- PyPI distribution: `je_auto_control` +- MCP server command: `je_auto_control_mcp` + +The project name is **AutoControl**; `je_auto_control` is the Python package/distribution name. diff --git a/CHANGELOG.md b/CHANGELOG.md index f5fe83ee6..062ccdf3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's ### Added +- AI-agent documentation and a dedicated `AI.md` explain computer-use positioning, MCP aliases, safe tool selection, and OpenAI integration. +- `AC_run_agent` now uses a focused computer-use allow-list by default instead of exposing the full `AC_*` command catalogue to the model. - `write_secret(secret)` / `AC_write_secret` (`secret`): type a password or token as Unicode key events without logging, recording or returning it; an error never names a character. Refuses on a backend without Unicode typing. @@ -72,6 +74,41 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's - **`AC_idempotency_release`** / MCP `ac_idempotency_release` / Script Builder *Idempotency: Release*: free an in-progress idempotency key whose work failed so a retry runs it. +- **Tesseract setup helpers.** `find_tesseract_cmd()` (`$TESSERACT_CMD`, + `PATH`, then the installers' default folders), `set_tessdata_dir(path)` + (sets `TESSDATA_PREFIX`; `None` clears it), `ocr_languages()` (sorted + codes; `[]` when the engine has no language data, `None` when it cannot be + asked) and `ocr_status()`, an `OCRStatus(ok, reason)` tuple whose reason is + `ready`, `missing_package`, `missing_engine`, `engine_unusable` or + `no_language_data`. `TesseractBackend` gained `cmd`, `version()` and + `languages()`. Commands `AC_ocr_status` / `AC_ocr_languages`, MCP tools + `ac_ocr_status` / `ac_ocr_languages`. +- **`click_mouse(..., clicks=1, interval=0.0)`.** `clicks=2` double-clicks + in one call: the clicks land on one point, `interval` seconds apart. + Defaults keep the single click, and a single click records the same + action as before. `AC_click_mouse`, the `ac_click_mouse` MCP tool and the + Script Builder take both parameters. On macOS the clicks still arrive as + separate single clicks (`Progress.md`). +- **Drag pacing: `tween_drag` / `drag_path` take `step_delay_s` and + `settle_s`** (seconds, default 0). `step_delay_s` rests after each move; + `settle_s` rests on the start before the press, after the press and before + the release, for apps that tell a drag from a click by the pointer's + motion. Both must be finite and non-negative (`ValueError` otherwise, + before anything is sent). `AC_tween_drag`, `AC_drag_path`, their MCP tools + and the Script Builder take both. +- **Windows key names.** The OEM keys (`oem_1` .. `oem_8`, `oem_102`, + `oem_plus`, `oem_comma`, `oem_minus`, `oem_period`, `oem_clear`), + `browser_home`, a lower-case `launch_app2` (`LAUNCH_APP2` stays) and + common aliases: `ctrl` / `lctrl` / `rctrl`, `alt` / `lalt` / `ralt`, + `enter`, `esc`, `win` / `super` / `cmd` / `meta`, `backspace` / `bksp`, + `del`, `ins`, `pgup` / `pageup`, `pgdn` / `pagedown`, `caps` / + `capslock`, `printscreen` / `prtsc` / `prtscr`, `scrolllock`, + `numpad0` .. `numpad9`, and `plus` / `comma` / `minus` / `period` for the + four OEM keys that are the same on every layout. Aliases are listed in + `platform_wrapper.keyboard_key_aliases` (empty on other platforms). +- **`keyboard_key_name(keycode)`** returns a code's canonical name and never + an alias, so a recorder's output does not change when an alias is added; + `None` when the table has no name for the code. - `cua_action.resolve_key_name` / `split_key_combo`, and `compile_postcondition(before=...)`. - `pii_text.luhn_valid` and `normalize_text(strip_format=...)`. @@ -267,6 +304,12 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's - **Webhook methods**: `PATCH` webhooks are served; verbs the server cannot answer (e.g. `HEAD`) are refused when the webhook is added instead of returning 501 on every request. +- **Windows `keyboard_keys_table` lists its aliases after the canonical + names**, `ctrl` included (it moved from beside `control` into + `keyboard_key_aliases`; same code). A reverse lookup that takes the + shortest name from the raw table now meets aliases such as `esc` or + `prtsc`; skip the names in `keyboard_key_aliases`, or call + `keyboard_key_name`, to keep getting the canonical ones. - **Golden-image capture (`take_golden` / `compare_to_golden`) reads its region in mouse coordinates**, like every other capture; region goldens taken on a scaled display need re-taking. @@ -1011,6 +1054,11 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's `shown=False` and Windows toasts appear; a dead key no longer reads as its US character; inverted and off-image regions are handled in annotate and colour stats. +- **An aborted `tween_drag` / `drag_path` lets go where the pointer stopped.** + When a step raised, the button was released at the end of the path, which + completed a drop the drag never reached; it is now released in a `finally` + at the last point the pointer reached, and a failing cleanup release is + logged instead of replacing the original error. - **Emergency stop on Linux / macOS**: the stop key wakes a sleeping or waiting main thread there too (SIGINT is sent to the main thread). - **Image analysis and packaging**: colour-vision simulation uses Machado diff --git a/CLAUDE.md b/CLAUDE.md index 4b155da56..458a3cb6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -213,4 +213,4 @@ Workspace rule shared by every repository under `D:\Codes` (full text: `D:\Codes - JSON action command names use the `AC_` prefix (e.g. `AC_click_mouse`); MCP tools use `ac_`. - Platform backends are named `{platform}_{function}.py` (e.g. `win32_ctype_mouse_control.py`). - Virtual key mappings live in `core/utils/*_vk.py` per platform. -- Two PyPI packages, both published by CI: a push to `main` releases `je_auto_control` (the `publish` job of `stable.yml`), and a push to `dev` that passes the headless suite and changes what the package ships releases `je_auto_control_dev` (the `publish-dev` job of `dev.yml`, `scripts/dev_release.py`). Never bump a version by hand; the version in `dev.toml` is only a floor. `dev.toml` must declare what `pyproject.toml` declares, so a change to dependencies, extras, scripts, entry points or `[tool.setuptools]` goes into both files (`test/unit_test/headless/test_dev_toml_parity.py` fails otherwise). +- Two PyPI packages, both published by CI: a push to `main` releases `je_auto_control` (the `publish` job of `stable.yml`), and a push to `dev` that passes the headless suite and changes what the package ships releases `je_auto_control_dev` (the `publish-dev` job of `dev.yml`, `scripts/dev_release.py`). Never bump a version by hand; the version in `dev.toml` is only a floor. `dev.toml` must declare what `pyproject.toml` declares, so a change to dependencies, extras, scripts, entry points or `[tool.setuptools]` goes into both files (`test/unit_test/headless/test_dev_toml_parity.py` fails otherwise). Both publish jobs hold the PyPI token, so they install only the hash-locked `.github/requirements/publish.txt` and build with `python -m build --no-isolation`: a new tool, or a higher floor in `build-system.requires`, needs the lock regenerated with the command at the top of `.github/requirements/publish.in` (`test/unit_test/headless/test_publish_tooling_lock.py` fails otherwise). diff --git a/Progress.md b/Progress.md index be164bd33..276069c77 100644 --- a/Progress.md +++ b/Progress.md @@ -1,5 +1,15 @@ # Progress +## 跨平台與 GUI 全面改版 + +`TODO` — 重設 UI、重寫並優化 GUI、修正 Wayland 與函式庫問題、跨機器同步、 +深化全套 mypy、補齊 iOS／Android、MCP 逐步揭露、自愈定位器量測、動作日誌 codegen、 +完整範例與文件。涉及 `gui/`、`linux_wayland/`、`android/`、`ios/`、 +`utils/{config_sync,remote_desktop,mcp_server,self_healing,codegen,executor}/` 與型別／文件驗證。 +核准設計：[跨平台自動化與 GUI 改版](docs/superpowers/specs/2026-10-02-platform-gui-modernization-design.md)。 +實作計畫：[分階段交付計畫](docs/superpowers/plans/2026-10-02-modernization-index.md)，待審閱。 +現有 `[Answer]` 決策沿用；產品實作尚未開始。 + **只記未完成的事。** 完成的工作記在 [docs/updates/](docs/updates/README.md)（每月一個批次檔， 索引與查詢指令在它的 README），相容性變更寫進 [CHANGELOG.md](CHANGELOG.md)；完成的項目 從本檔移除，同一個 commit 在 `docs/updates/` 補一筆 `#done` 條目，不在這裡累積歷史。 @@ -184,6 +194,14 @@ pip install --dry-run --only-binary=:all: --platform win_arm64 --python-version --- +## macOS 上 `click_mouse(clicks=2)` 不是雙擊 + +`TODO` — `wrapper/auto_control_mouse.py` 的 `click_mouse` 在同一點連點 `clicks` 次。Windows 與 X11 依兩次點擊的時間差與位移判定雙擊，所以這樣就夠；macOS 的應用程式讀的是事件上的點擊次數欄位（`kCGMouseEventClickState`），`osx/mouse/osx_mouse.py` 的 `mouse_event` 從不設定它（一律是 1），所以 macOS 收到的是兩次單擊。文件（`docs/source/API/wrapper/mouse.rst`、`docs/source/{Eng,Zh}/doc/mouse/mouse_doc.rst`、`click_mouse` 的 docstring）照實寫了這個限制。 + +做法：讓 osx 後端的按下／放開帶點擊次數（第 n 次點擊設成 n），wrapper 在 macOS 分支把序號傳下去；要在 macOS 上實測（`quality.yml` 的 `macos-14` 可以跑 CI，但雙擊是否被應用程式認得要真機看），完成後把文件裡的限制拿掉。 + +--- + ## RBAC 還沒接到 REST API 與 MCP server `DECIDE` — 要不要把 `utils/rbac` 接上兩個伺服器，以及現有單一共用 token 怎麼過渡（維護者拍板） @@ -208,6 +226,7 @@ MCP 的 bearer 比對同理；稽核寫入帶上 `user_id`。 **選項**：簽章指令在強制模式下只准本機 CLI 使用；或簽章金鑰與執行權限分開保存（簽章端不在執行端）。 +[Answer] 簽章金鑰與執行權限分開保存 --- ## USB passthrough viewer 以種類配對回覆，逾時的回覆會交給下一個請求 @@ -222,6 +241,7 @@ MCP 的 bearer 比對同理；稽核寫入帶上 `user_id`。 **選項**：在 payload 加一個由 client 產生、host 原樣帶回的請求編號（舊 host 不帶就退回現在的配對）；或逾時後把該 claim 標成需排空，丟掉下一個回覆——但 host 若根本沒回，會丟掉正確的回覆。 +[Answer] 加，但是想辦法解決可能丟掉正確回復的問題 --- ## 全域 executor 的變數會留到下一次執行 @@ -239,18 +259,7 @@ socket server 的執行也都用同一個 `executor`；`for_each` 的迴圈變 **附帶**：`AC_circuit_call`、`AC_bulkhead_run`、`AC_run_chaos`、`AC_run_dag` 的巢狀動作跑在全域 `executor` 上， 在 `AC_parallel` 分支裡因此用到父層的變數範圍，而不是分支自己的。 ---- - -## 舊式 CLI（`-e`／`-d`／`--execute_str`）在動作失敗時仍然結束碼 0 - -`DECIDE` — 要不要讓舊式入口也以結束碼 1 回報動作失敗（跨專案契約，PyBreeze 與 TestPioneer 以子程序呼叫） - -`je_auto_control/__main__.py` 執行完不看 `recorded_failures()`；`je_auto_control run` 在 `cli.py` 已經會回 1。 -同一個會失敗的腳本，`run` 回 1，`-e`、`-d`、`--execute_str` 回 0（2026-09-24 稽核重現）。 - -**要先確認**：PyBreeze（`AI_CONTEXT.md` §5）與 TestPioneer 的 `parallel_run` 怎麼解讀這個結束碼——若把非 0 當成 -「無法執行」而非「有動作失敗」，改了會讓它們把一次有失敗步驟的執行回報成錯誤。改的話兩邊的 `architecture.md` §6 -與相容性測試要一起更新。 +[Answer] `execute_action_with_vars` 與各伺服器入口每次開一個新的 `VariableScope`（`AC_set_var` 在單次執行內照舊 --- @@ -262,22 +271,22 @@ socket server 的執行也都用同一個 `executor`；`for_each` 的迴圈變 一回合多個呼叫逐一執行後一次回覆、每個 `tool_result` 帶 `toolset_name`、截圖縮到高解析度層級的 2576 px／4784 visual tokens 內並換算座標、`zoom` 以全解析度裁切回覆）， `claude-opus-5-5` 自動使用它；其他模型仍預設 beta 形式，因為 toolset 只以假 client 測過、還沒對真的 API 跑過。 -**附帶**：`AC_run_agent backend="openai"` 送出全部約 740 個工具，超過 OpenAI Chat Completions 的 128 個上限， -所以一定失敗——與「`AC_run_agent` 預設工具集」那一條 DECIDE 一起決定。 +**附帶**：`AC_run_agent` 現在預設只提供一組聚焦的 computer-use 工具，OpenAI 不再收到整個命令目錄；需要更大的工具集時，應由應用程式明確用 `export_openai_tools(only=[...])` 建立 agent。 --- -## MCP registry 的 server 名稱與專案網址還是舊組織 -`DECIDE` — 要發布到 MCP registry 前得先定名稱，改名會影響已發布的項目 +## mypy 2.4.0 在三個平台模組回報 `has-type` + +`TODO` — 讓型別契約在 mypy 2.4 也過，再把 CI 的 pin 往上提 -`utils/mcp_registry/registry.py` 的 `_SERVER_NAME` 是 `io.github.intergration-automation-testing/autocontrol`， -`_REPO_URL` 與 `pyproject.toml` 的 Homepage / Code、`README.md` 的 clone 網址都還是 -`Intergration-Automation-Testing/AutoControl`；repo 現在在 `Integration-Automation/AutoControlGUI`（舊網址只是轉址）。 -registry 以 GitHub 帳號驗證 `io.github./` 命名空間，舊組織名發布不了。 +`quality.yml` 的 `typing-stable-api` 固定 `mypy==2.3.0`，`test/verify/typing_contract_verify.py` 在它上面是 0 個失敗模組； +`dev_requirements.txt` 只寫下限 `mypy>=2.3.0`，新環境會裝到 2.4.0，同一個指令就多出三個不在豁免清單上的模組 +（2026-10-08 實測）：`wrapper/_platform_osx.py`（非 darwin 目標下 `osx_key_*` 全部 `Cannot determine type`）、 +`wrapper/_platform_windows.py:363`（`win32_recorder`）、`windows/message/window_message.py:7-8`（`user32`）。 -**做法**：決定正式名稱（例如 `io.github.integration-automation/autocontrol`），在同一輪改 `registry.py`、 -`pyproject.toml`、三份 README 的網址。 +**做法**：替這些跨平台名稱補上明確型別註記（或把平台分支改成 mypy 看得懂的 `sys.platform` 判斷）， +兩個版本都驗過後把 `quality.yml` 的 pin 提到 2.4.x。 --- @@ -302,6 +311,8 @@ registry 以 GitHub 帳號驗證 `io.github./` 命名空間，舊組織名 「外掛載入時門面已經在 `sys.modules` 裡」，改完就不成立，那份說明與測試要一起改寫 （CI 仍可繼續用 `coverage run -m pytest`）。 +[Answer] 照你建議改寫 + --- ## libei 的 `ei_unref` 在半開交握上會 SIGSEGV @@ -340,6 +351,8 @@ pip install --dry-run --only-binary=:all: --platform macosx_10_9_x86_64 \ --python-version 3.12 --target /tmp/probe 'cryptography>=50' ``` +[Answer] 可以 + --- ## Viewer 端要不要把 host 推來的檔案關在一個目錄裡 @@ -358,27 +371,7 @@ viewer 端的 `FileReceiver`（`utils/remote_desktop/file_transfer.py`）照單 **為什麼要拍板**：`dest_path` 的語意會從「viewer 上的絕對路徑」變成「viewer 下載目錄裡的相對路徑」， 現有腳本與文件範例都要跟著改。 ---- - -## `AC_run_agent` 預設把每個 AC_* 指令都交給模型 - -`DECIDE` — 預設工具集要不要排除高風險指令 - -`utils/executor/action_executor.py` 的 `_run_agent` 以 `export_anthropic_tools()` / `export_openai_tools()` -不帶 `only=` 建立 backend，所以模型拿得到 `AC_shell_command`、`AC_execute_process`、`AC_android_shell`、 -`AC_add_package_to_executor`、`AC_run_agent`、`AC_computer_use`、`AC_execute_action` 等指令。 -2026-09-23 已讓 backend 拒絕「沒有提供的工具」，但提供的清單本身就包含這些； -螢幕上的內容（網頁、文件）若誘導模型呼叫 shell，目前不會被擋。 - -**做法**：`_run_agent` 預設排除上述類別，另加一個 opt-in 參數（例如 `allow_system_commands`） -讓需要的人明確打開；MCP `ac_run_agent` 與 Script Builder 的欄位同步。 - -**為什麼要拍板**：這會縮小既有的 agent 能力，依賴它跑 shell 的腳本會改變行為。 - -實測數字（2026-09-25）：預設清單有 741 個指令，含 `AC_run_agent` 本身（模型可以遞迴開 agent）； -`backend="openai"` 超過 Chat Completions 的 128 個工具上限，現在建 backend 時就明確拒絕； -Anthropic 每一步送約 202 KB 的工具 schema、沒有 `cache_control`。拍板後一併決定上限與快取。 - +[Answer] 可以改 --- @@ -436,6 +429,8 @@ MCP 工具的檔案參數（`path`、`file_path`、`db`、`image_path`、`golden `base_dir` 限制，`env://` 可讀任何環境變數，包括放 API 金鑰的那些，結果直接回給模型。`secret://` 已經拒絕； `env://` 要不要改成允許清單、`file://` 要不要套同一個根目錄，跟上面一起決定。 +[Answer] 兩者，不要預設開啟唯獨 + --- ## Windows 的 DPI 感知是系統層級，混合 DPI 的螢幕座標被虛擬化 @@ -453,6 +448,8 @@ per-monitor。DPI 與主螢幕不同的螢幕會被 Windows 虛擬化：本機 **為什麼要拍板**：這個檔在 Jeffrey_RPA 正在跑的截圖路徑上。換成 per-monitor 之後，縮放螢幕上的座標與截圖 尺寸都會變，既有的樣板和錄好的座標在那些螢幕上會失準。 +[Answer] 換並修好 Jeffrey_RPA + --- ## `pil_screenshot`／`screenshot` 的區域擷取在 Windows 只看得到主螢幕 @@ -504,6 +501,8 @@ be at 2x if on a Retina screen」，`scale_down=True` 只在帶 `bbox` 時生效 **為什麼要拍板**：`AC_remote_*` 指令與 MCP 工具依賴「registry 裡就是那一個 viewer」，改成多槽位要一起改它們的語意。 +[Answer] 一起改沒問題 + --- ## `test_usb_acl_prompt.py` 讓 Python 3.10 的 headless 測試間歇 segfault diff --git a/README.md b/README.md index 69442eea5..ec9c28d20 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![Documentation](https://readthedocs.org/projects/autocontrol/badge/?version=latest)](https://autocontrol.readthedocs.io/en/latest/?badge=latest) -**AutoControl** is a cross-platform GUI automation framework for Python. It drives the -mouse and keyboard, finds things on screen (template matching, OCR, the OS accessibility -tree, or a vision model), records and replays flows, and runs them from JSON action -files — on Windows, macOS, Linux (X11 and Wayland), the BSDs, Android, -and iOS. +**AutoControl** is an open-source, cross-platform **computer-use and GUI automation framework** for AI agents, Python applications, and automated testing. It drives the mouse and keyboard, finds UI targets through template matching, OCR, the OS accessibility tree, or a vision model, records and replays flows, and runs deterministic JSON actions across Windows, macOS, Linux (X11 and Wayland), the BSDs, Android, and iOS. Every capability ships three ways: a **Python API**, an **`AC_*` action command** usable from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. @@ -22,7 +18,7 @@ from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. - **One API, seven platforms.** `wrapper/platform_wrapper.py` picks the backend at import time; your script does not change between Windows, macOS, X11, and Wayland. -- **Scriptable without Python.** 776 `AC_*` commands cover the whole feature set, so a +- **Scriptable without Python.** 778 `AC_*` commands cover the whole feature set, so a JSON file can do anything the library can — including loops, branches, try/catch, macros, and variables. - **Headless by default.** `import je_auto_control` never loads Qt. The GUI is an @@ -33,8 +29,86 @@ from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. WebSocket framing, ACME client, USB/IP protocol, and Prometheus metrics are all standard-library implementations. Heavy things are opt-in extras. + +## AI agents and computer use + +**AutoControl is an open-source, cross-platform computer-use and GUI automation framework for AI agents.** + +Use it when an agent needs to operate a real desktop GUI — native applications, real browser windows, remote desktops, or mobile devices — rather than only a browser DOM/API. + +### When an AI agent should choose AutoControl + +- Real mouse and keyboard input +- Screenshots and screen-coordinate interaction +- OCR and native accessibility-tree discovery +- Image/template matching +- Vision-language-model UI localization +- Anchor and self-healing locators +- Cross-platform desktop/mobile automation +- Deterministic JSON action files +- MCP-based computer use + +A robust agent loop is: + +**observe → identify → act → wait → verify → recover** + +The repository also has a dedicated [AI agent guide](AI.md) with MCP configuration, tool selection, OpenAI integration, security guidance, and searchable computer-use terminology. + +### AI-friendly MCP + +Start the stdio server with: + +```bash +pip install je_auto_control +je_auto_control_mcp +``` + +The MCP server exposes the canonical `ac_*` surface plus short, model-friendly aliases such as `click`, `type`, `screenshot`, `find_text`, `click_text`, `drag`, `focus_window`, and `wait_image`. + +For inspection-only clients: + +```bash +je_auto_control_mcp --read-only +``` + +If a client needs only canonical `ac_*` names: + +```bash +JE_AUTOCONTROL_MCP_ALIASES=0 je_auto_control_mcp +``` + +For OpenAI agent integrations, expose a focused allow-list with `export_openai_tools(only=[...])` instead of passing the complete AutoControl command catalogue. This both fits provider limits and reduces the authority given to the model. + +### Project identity + +**Project:** AutoControl +**Repository:** `Integration-Automation/AutoControlGUI` +**Python package / PyPI:** `je_auto_control` +**MCP command:** `je_auto_control_mcp` + + --- + +## Choosing the right automation layer + +AutoControl is not intended to replace every automation tool. Use the smallest layer that matches the interface: + +| Need | Good fit | +|---|---| +| Stable browser DOM/API automation | Playwright / Selenium | +| Simple Python mouse and keyboard scripting | PyAutoGUI or AutoControl | +| Native desktop application automation | **AutoControl** | +| Accessibility-tree GUI automation | **AutoControl** | +| OCR-driven GUI automation | **AutoControl** | +| Screenshot / vision-model GUI localization | **AutoControl** | +| Self-healing cross-platform GUI locators | **AutoControl** | +| AI agent controlling a real desktop | **AutoControl + MCP** | +| Deterministic JSON GUI workflows | **AutoControl** | + +The differentiator is the combination of **real computer input + semantic/visual discovery + self-healing + agent/MCP integration** behind one cross-platform automation surface. + + ## Installation ```bash @@ -73,7 +147,10 @@ sudo apt-get install cmake libssl-dev ``` OCR, VLM, and LLM backends (`pytesseract`, `easyocr`, `paddleocr`, `anthropic`, -`openai`) are loaded on demand — install whichever you actually use. +`openai`) are loaded on demand — install whichever you actually use. For Tesseract, +`find_tesseract_cmd()` locates the executable (`$TESSERACT_CMD`, then `PATH`, then the +installers' default folders), `set_tessdata_dir()` points it at a language-data folder, +and `ocr_status()` / `ocr_languages()` say what is missing before the first OCR call. **Log file:** the library logs to `~/.je_auto_control/logs/AutoControlGUI.log`, created on the first record (importing alone writes nothing) and shared by every @@ -142,11 +219,11 @@ desktop app; tab commands live in the window's **Actions** menu. | Capability | Python API | `AC_*` command | GUI tab | |---|---|---|---| -| Mouse | `click_mouse`, `set_mouse_position`, `mouse_scroll` | `AC_click_mouse` | Auto Click | -| Keyboard | `write`, `write_secret`, `hotkey`, `type_keyboard` | `AC_write`, `AC_write_secret`, `AC_hotkey` | Auto Click | +| Mouse | `click_mouse` (`clicks=2` double-clicks), `set_mouse_position`, `mouse_scroll`, `tween_drag` / `drag_path` (paced by `step_delay_s`, `settle_s`) | `AC_click_mouse`, `AC_tween_drag`, `AC_drag_path` | Auto Click | +| Keyboard | `write`, `write_secret`, `hotkey`, `type_keyboard`, `keyboard_key_name` (code → canonical name; Windows also takes aliases such as `ctrl`, `esc`, `enter`) | `AC_write`, `AC_write_secret`, `AC_hotkey` | Auto Click | | Screen & pixels | `screenshot`, `screen_size`, `get_pixel` | `AC_screenshot` | Screenshot | | Image matching | `locate_image_center`, `locate_and_click` | `AC_locate_and_click` | Image Detect | -| OCR text | `click_text`, `wait_for_text`, `read_text_in_region` | `AC_click_text`, `AC_wait_text` | OCR Reader | +| OCR text | `click_text`, `wait_for_text`, `read_text_in_region`, `ocr_status` | `AC_click_text`, `AC_wait_text`, `AC_ocr_status` | OCR Reader | | Accessibility tree | `find_accessibility_element`, `click_accessibility_element` | `AC_a11y_find`, `AC_a11y_click` | Accessibility | | Vision-model locator | `locate_by_description`, `click_by_description` | `AC_vlm_locate`, `AC_vlm_click` | VLM | | Anchor locator | — | `AC_anchor_click`, `AC_anchor_locate` | — | @@ -154,7 +231,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Natural-language planner | `plan_actions`, `run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`, `run_agent` | `AC_run_agent` | Computer Use | | Record & replay | `record`, `stop_record` | `AC_record`, `AC_stop_record` | Record | -| JSON scripting | `execute_action`, `execute_files` | all 776 commands | Script, Script Builder | +| JSON scripting | `execute_action`, `execute_files` | all 778 commands | Script, Script Builder | | Variables & flow control | `execute_action_with_vars` | `AC_set_var`, `AC_loop`, `AC_for_each`, `AC_try`, `AC_retry` | Variables | | Data-driven runs | — | `AC_for_each_row` (CSV / JSON / SQLite / Excel) | Data Sources | | Assertions | `assert_text`, `assert_image` | `AC_assert_text` + 20 more | Assertions | @@ -198,7 +275,11 @@ je_auto_control version `--var name=value` is parsed as JSON when possible (`count=10` becomes an int), otherwise kept as a string. `run` exits 1 when any action failed (the run still goes on to the end), so a CI step fails with it. The legacy -`python -m je_auto_control -e file.json` entry point still works. +`python -m je_auto_control -e file.json` entry point still works, and its +`-e`, `-d` and `--execute_str` flags exit 1 the same way. Under +[TestPioneer](https://github.com/Integration-Automation/TestPioneer), which +sets `TEST_PIONEER_ARTIFACT_DIR`, the HTML, JSON and XML reports asked for +under a relative name are written below that directory. --- @@ -206,7 +287,7 @@ still goes on to the end), so a CI step fails with it. The legacy | Surface | Start it with | Notes | |---|---|---| -| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 678 tools for Claude Desktop / Claude Code / custom tool loops. Speaks the stateless MCP 2026-07-28 beside the `initialize`-based revisions. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | +| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 680 tools for Claude Desktop / Claude Code / custom tool loops: the full `ac_*` surface plus short model-friendly aliases for common GUI actions. Speaks the stateless MCP 2026-07-28 beside the `initialize`-based revisions. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | | **REST API** | `je_auto_control start-rest` | Bearer token, per-IP rate limit + lockout, SQLite audit hook, `/metrics`, `/openapi.json`, `/docs` Swagger UI, `/dashboard`. | | **TCP socket server** | `je_auto_control start-server` | Newline-framed JSON action lists. Binds `127.0.0.1` by default. | | **pytest plugin** | installed automatically | Fixtures plus a Gherkin step library for pytest-bdd / behave. | @@ -366,7 +447,7 @@ ignore synthetic input, and fall back silently when the driver is absent. ## Development ```bash -git clone https://github.com/Intergration-Automation-Testing/AutoControl.git +git clone https://github.com/Integration-Automation/AutoControlGUI.git cd AutoControl pip install -r dev_requirements.txt uv sync # or: reproducible install from the committed uv.lock @@ -394,6 +475,6 @@ API and a GUI surface. See [Third_Party_License.md](Third_Party_License.md) for the licenses of bundled and optional third-party components. -- **Homepage**: https://github.com/Intergration-Automation-Testing/AutoControl +- **Homepage**: https://github.com/Integration-Automation/AutoControlGUI - **PyPI**: https://pypi.org/project/je_auto_control/ - **Documentation**: https://autocontrol.readthedocs.io/en/latest/ diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 508c2bc5b..a8eb3e953 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -5,8 +5,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](../LICENSE) [![Documentation](https://readthedocs.org/projects/autocontrol/badge/?version=latest)](https://autocontrol.readthedocs.io/en/latest/?badge=latest) -**AutoControl** 是一套跨平台的 Python GUI 自动化框架。它能驱动鼠标与键盘、在画面上找到目标 -（模板匹配、OCR、操作系统无障碍树，或视觉模型）、录制与回放操作流程，并以 JSON 动作文件执行—— +**AutoControl** 是一套开源、跨平台的 **computer-use 与 GUI 自动化框架**，面向 AI agent、Python 应用程序与自动化测试。它能驱动鼠标与键盘、通过模板匹配、OCR、操作系统无障碍树或视觉模型找到 UI 目标、录制与回放操作流程，并以 JSON 动作文件执行—— 支持 Windows、macOS、Linux（X11 与 Wayland）、BSD、Android 与 iOS。 每项能力都以三种形式提供：**Python API**、可在 JSON 文件／CLI／服务器使用的 **`AC_*` 动作命令**， @@ -20,15 +19,91 @@ - **一套 API，七个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 Windows、macOS、X11 与 Wayland 上都不需要改写。 -- **不写 Python 也能脚本化。** 776 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 +- **不写 Python 也能脚本化。** 778 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 能做的任何事——包含循环、分支、try/catch、宏与变量。 - **默认无头运行。** `import je_auto_control` 绝不会加载 Qt。GUI 是可选包，包在同一个无头内核之外。 - **四种定位方式。** 模板匹配、OCR、无障碍树、视觉语言模型——可通过锚点定位器与自愈回退串接组合。 - **依赖基线轻量。** REST 服务器、JSON Schema 校验、JWT、TOTP、WebSocket 帧、ACME 客户端、 USB/IP 协议与 Prometheus 指标全部以标准库实现；较重的依赖都是可选项。 + +## AI Agent 与 Computer Use + +**AutoControl 是开源、跨平台的 computer-use 与 GUI 自动化框架。** + +当 AI agent 需要操作真实桌面、原生应用程序、真实浏览器窗口、远程桌面或移动设备，而不只是操作浏览器 DOM/API 时，使用 AutoControl。 + +### 何时选择 AutoControl + +- 真实鼠标与键盘输入 +- 屏幕截图与坐标操作 +- OCR 与原生无障碍树 +- 图像／模板匹配 +- Vision-Language Model UI 定位 +- Anchor 与 self-healing locator +- 跨平台桌面／移动设备自动化 +- JSON 动作文件 +- MCP computer use + +建议的 agent loop： + +**observe → identify → act → wait → verify → recover** + +完整的 [AI Agent 指南](../AI.md) 提供 MCP 配置、工具选择、OpenAI 集成、安全性与 computer-use 关键词。 + +### AI-friendly MCP + +```bash +pip install je_auto_control +je_auto_control_mcp +``` + +MCP 提供完整的 `ac_*` 工具，以及 `click`、`type`、`screenshot`、`find_text`、`click_text`、`drag`、`focus_window`、`wait_image` 等短名称 alias。 + +只做检查： + +```bash +je_auto_control_mcp --read-only +``` + +只需要 canonical `ac_*` 名称： + +```bash +JE_AUTOCONTROL_MCP_ALIASES=0 je_auto_control_mcp +``` + +OpenAI agent 请使用 `export_openai_tools(only=[...])` 提供聚焦工具集，不要一次提供整个 AutoControl 命令目录。 + +### 项目识别 + +**项目：** AutoControl +**Repository：** `Integration-Automation/AutoControlGUI` +**Python package / PyPI：** `je_auto_control` +**MCP command：** `je_auto_control_mcp` + + --- + +## 如何选择自动化层 + +AutoControl 不需要取代所有自动化工具；应选择最符合界面的那一层： + +| 需求 | 适合工具 | +|---|---| +| 稳定的浏览器 DOM／API 自动化 | Playwright／Selenium | +| 简单的 Python 鼠标键盘脚本 | PyAutoGUI 或 AutoControl | +| 原生桌面应用程序自动化 | **AutoControl** | +| 无障碍树 GUI 自动化 | **AutoControl** | +| OCR GUI 自动化 | **AutoControl** | +| 截图／视觉模型 GUI 定位 | **AutoControl** | +| Self-healing 跨平台 GUI locator | **AutoControl** | +| AI agent 操作真实桌面 | **AutoControl + MCP** | +| 确定性的 JSON GUI workflow | **AutoControl** | + +AutoControl 的差异在于把 **真实电脑输入 + 语义／视觉定位 + self-healing + agent/MCP 集成** 放在同一个跨平台自动化接口中。 + + ## 安装 ```bash @@ -63,7 +138,9 @@ sudo apt-get install cmake libssl-dev ``` OCR、VLM 与 LLM 后端（`pytesseract`、`easyocr`、`paddleocr`、`anthropic`、`openai`） -都是按需加载——只装你实际会用到的。 +都是按需加载——只装你实际会用到的。Tesseract 方面，`find_tesseract_cmd()` 找出可执行文件 +（依次是 `$TESSERACT_CMD`、`PATH`、安装程序的默认文件夹），`set_tessdata_dir()` 把它指向 +语言数据文件夹，`ocr_status()`／`ocr_languages()` 在第一次 OCR 之前就说出缺了什么。 **日志文件：** 库写到 `~/.je_auto_control/logs/AutoControlGUI.log`，第一条记录时 才创建（只 import 不会写任何文件），同一个账户的所有进程共用（追加写入、每行带进程 ID， @@ -130,11 +207,11 @@ python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" | 能力 | Python API | `AC_*` 命令 | GUI 标签页 | |---|---|---|---| -| 鼠标 | `click_mouse`、`set_mouse_position`、`mouse_scroll` | `AC_click_mouse` | Auto Click | -| 键盘 | `write`、`write_secret`、`hotkey`、`type_keyboard` | `AC_write`、`AC_write_secret`、`AC_hotkey` | Auto Click | +| 鼠标 | `click_mouse`（`clicks=2` 即双击）、`set_mouse_position`、`mouse_scroll`、`tween_drag`／`drag_path`（以 `step_delay_s`、`settle_s` 控制节奏） | `AC_click_mouse`、`AC_tween_drag`、`AC_drag_path` | Auto Click | +| 键盘 | `write`、`write_secret`、`hotkey`、`type_keyboard`、`keyboard_key_name`（键码 → 标准键名；Windows 也接受 `ctrl`、`esc`、`enter` 等别名） | `AC_write`、`AC_write_secret`、`AC_hotkey` | Auto Click | | 屏幕与像素 | `screenshot`、`screen_size`、`get_pixel` | `AC_screenshot` | Screenshot | | 图像匹配 | `locate_image_center`、`locate_and_click` | `AC_locate_and_click` | Image Detect | -| OCR 文字 | `click_text`、`wait_for_text`、`read_text_in_region` | `AC_click_text`、`AC_wait_text` | OCR Reader | +| OCR 文字 | `click_text`、`wait_for_text`、`read_text_in_region`、`ocr_status` | `AC_click_text`、`AC_wait_text`、`AC_ocr_status` | OCR Reader | | 无障碍树 | `find_accessibility_element`、`click_accessibility_element` | `AC_a11y_find`、`AC_a11y_click` | Accessibility | | 视觉模型定位 | `locate_by_description`、`click_by_description` | `AC_vlm_locate`、`AC_vlm_click` | VLM | | 锚点定位 | — | `AC_anchor_click`、`AC_anchor_locate` | — | @@ -142,7 +219,7 @@ python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" | 自然语言规划 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | | 录制与回放 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | -| JSON 脚本 | `execute_action`、`execute_files` | 全部 776 个命令 | Script、Script Builder | +| JSON 脚本 | `execute_action`、`execute_files` | 全部 778 个命令 | Script、Script Builder | | 变量与流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | | 数据驱动执行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | | 断言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 个 | Assertions | @@ -183,7 +260,7 @@ je_auto_control version ``` `--var name=value` 会尽量以 JSON 解析（`count=10` 会变成整数），否则视为字符串。 -`run` 只要有任何动作失败就以 1 退出（仍会跑完整份脚本），CI 步骤会随之失败。旧版 `python -m je_auto_control -e file.json` 入口仍然可用。 +`run` 只要有任何动作失败就以 1 退出（仍会跑完整份脚本），CI 步骤会随之失败。旧版 `python -m je_auto_control -e file.json` 入口仍然可用，它的 `-e`、`-d` 与 `--execute_str` 标志也同样在有动作失败时以 1 退出。由 [TestPioneer](https://github.com/Integration-Automation/TestPioneer) 启动时（它会设置 `TEST_PIONEER_ARTIFACT_DIR`），以相对名称要求的 HTML、JSON、XML 报告会写到该目录底下。 --- @@ -191,7 +268,7 @@ je_auto_control version | 接口 | 启动方式 | 说明 | |---|---|---| -| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 678 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用。除了以 `initialize` 握手的各版协议，也支持无状态的 MCP 2026-07-28。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | +| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 680 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用：完整的 `ac_*` 工具接口，以及常用 GUI 操作的短名称 alias。除了以 `initialize` 握手的各版协议，也支持无状态的 MCP 2026-07-28。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | | **REST API** | `je_auto_control start-rest` | Bearer token、按 IP 限流与锁定、SQLite 审计 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | | **TCP socket 服务器** | `je_auto_control start-server` | 以换行分隔的 JSON 动作列表。默认绑定 `127.0.0.1`。 | | **pytest 插件** | 安装后自动生效 | 提供 fixture 与供 pytest-bdd／behave 使用的 Gherkin step library。 | @@ -332,7 +409,7 @@ Windows、macOS（pyobjc）与 X11（含 XWayland）；纯 Wayland 会话的协 ## 开发 ```bash -git clone https://github.com/Intergration-Automation-Testing/AutoControl.git +git clone https://github.com/Integration-Automation/AutoControlGUI.git cd AutoControl pip install -r dev_requirements.txt uv sync # 或：以已提交的 uv.lock 做可重现安装 @@ -358,6 +435,6 @@ bandit -c pyproject.toml -r je_auto_control/ [MIT License](../LICENSE) © JE-Chen。 内含与可选第三方组件的许可请见 [Third_Party_License.md](../Third_Party_License.md)。 -- **主页**：https://github.com/Intergration-Automation-Testing/AutoControl +- **主页**：https://github.com/Integration-Automation/AutoControlGUI - **PyPI**：https://pypi.org/project/je_auto_control/ - **文档**：https://autocontrol.readthedocs.io/en/latest/ diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 404537ccd..88674e276 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -5,9 +5,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](../LICENSE) [![Documentation](https://readthedocs.org/projects/autocontrol/badge/?version=latest)](https://autocontrol.readthedocs.io/en/latest/?badge=latest) -**AutoControl** 是一套跨平台的 Python GUI 自動化框架。它能驅動滑鼠與鍵盤、在畫面上找到目標 -（樣板比對、OCR、作業系統無障礙樹，或視覺模型）、錄製與重播操作流程，並以 JSON 動作檔執行—— -支援 Windows、macOS、Linux（X11 與 Wayland）、BSD、Android 與 iOS。 +**AutoControl** 是一套開源、跨平台的 **computer-use 與 GUI 自動化框架**，面向 AI agent、Python 應用程式與自動化測試。它能驅動滑鼠與鍵盤、透過樣板比對、OCR、作業系統無障礙樹或視覺模型找到 UI 目標、錄製與重播流程，並以 JSON 動作檔執行——支援 Windows、macOS、Linux（X11 與 Wayland）、BSD、Android 與 iOS。 每項能力都以三種形式提供：**Python API**、可在 JSON 檔／CLI／伺服器使用的 **`AC_*` 動作指令**， 以及 **GUI 分頁**。沒有任何功能只存在於 GUI。 @@ -20,15 +18,91 @@ - **一套 API，七個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 Windows、macOS、X11 與 Wayland 上都不需要改寫。 -- **不寫 Python 也能腳本化。** 776 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 +- **不寫 Python 也能腳本化。** 778 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 能做的任何事——包含迴圈、分支、try/catch、巨集與變數。 - **預設無頭執行。** `import je_auto_control` 絕不會載入 Qt。GUI 是選用套件，包在同一個無頭核心之外。 - **四種定位方式。** 樣板比對、OCR、無障礙樹、視覺語言模型——可透過錨點定位器與自癒後備串接組合。 - **相依基線輕薄。** REST 伺服器、JSON Schema 驗證、JWT、TOTP、WebSocket 框架、ACME 用戶端、 USB/IP 協定與 Prometheus 指標全部以標準庫實作；較重的相依都是選用。 + +## AI Agent 與 Computer Use + +**AutoControl 是開源、跨平台的 computer-use 與 GUI 自動化框架。** + +當 AI agent 需要操作真實桌面、原生應用程式、真正的瀏覽器視窗、遠端桌面或行動裝置，而不只是操作瀏覽器 DOM/API 時，使用 AutoControl。 + +### 何時選擇 AutoControl + +- 真實滑鼠與鍵盤輸入 +- 螢幕擷取與座標操作 +- OCR 與原生無障礙樹 +- 影像／樣板比對 +- Vision-Language Model UI 定位 +- Anchor 與 self-healing locator +- 跨平台桌面／行動裝置自動化 +- JSON 動作檔 +- MCP computer use + +建議的 agent loop： + +**observe → identify → act → wait → verify → recover** + +完整的 [AI Agent 指南](../AI.md) 提供 MCP 設定、工具選擇、OpenAI 整合、安全性與 computer-use 關鍵字。 + +### AI-friendly MCP + +```bash +pip install je_auto_control +je_auto_control_mcp +``` + +MCP 提供完整的 `ac_*` 工具，以及 `click`、`type`、`screenshot`、`find_text`、`click_text`、`drag`、`focus_window`、`wait_image` 等短名稱 alias。 + +只做檢查： + +```bash +je_auto_control_mcp --read-only +``` + +只需要 canonical `ac_*` 名稱： + +```bash +JE_AUTOCONTROL_MCP_ALIASES=0 je_auto_control_mcp +``` + +OpenAI agent 請使用 `export_openai_tools(only=[...])` 提供聚焦工具集，不要一次提供整個 AutoControl 命令目錄。 + +### 專案識別 + +**專案：** AutoControl +**Repository：** `Integration-Automation/AutoControlGUI` +**Python package / PyPI：** `je_auto_control` +**MCP command：** `je_auto_control_mcp` + + --- + +## 如何選擇自動化層 + +AutoControl 不需要取代所有自動化工具；應該選擇最符合介面的那一層： + +| 需求 | 適合工具 | +|---|---| +| 穩定的瀏覽器 DOM／API 自動化 | Playwright／Selenium | +| 簡單的 Python 滑鼠鍵盤腳本 | PyAutoGUI 或 AutoControl | +| 原生桌面應用程式自動化 | **AutoControl** | +| 無障礙樹 GUI 自動化 | **AutoControl** | +| OCR GUI 自動化 | **AutoControl** | +| 截圖／視覺模型 GUI 定位 | **AutoControl** | +| Self-healing 跨平台 GUI locator | **AutoControl** | +| AI agent 操作真實桌面 | **AutoControl + MCP** | +| 確定性的 JSON GUI workflow | **AutoControl** | + +AutoControl 的差異在於把 **真實電腦輸入 + 語意／視覺定位 + self-healing + agent/MCP 整合** 放在同一個跨平台自動化介面。 + + ## 安裝 ```bash @@ -63,7 +137,9 @@ sudo apt-get install cmake libssl-dev ``` OCR、VLM 與 LLM 後端（`pytesseract`、`easyocr`、`paddleocr`、`anthropic`、`openai`） -都是按需載入——只裝你實際會用到的。 +都是按需載入——只裝你實際會用到的。Tesseract 的部分，`find_tesseract_cmd()` 找出執行檔 +（依序是 `$TESSERACT_CMD`、`PATH`、安裝程式的預設資料夾），`set_tessdata_dir()` 把它指到 +語言資料夾，`ocr_status()`／`ocr_languages()` 在第一次 OCR 之前就說出缺了什麼。 **記錄檔：** 函式庫寫到 `~/.je_auto_control/logs/AutoControlGUI.log`，第一筆記錄時 才建立（只 import 不會寫任何檔），同一個帳號的所有行程共用（附加寫入、每行帶行程 ID， @@ -130,11 +206,11 @@ python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" | 能力 | Python API | `AC_*` 指令 | GUI 分頁 | |---|---|---|---| -| 滑鼠 | `click_mouse`、`set_mouse_position`、`mouse_scroll` | `AC_click_mouse` | Auto Click | -| 鍵盤 | `write`、`write_secret`、`hotkey`、`type_keyboard` | `AC_write`、`AC_write_secret`、`AC_hotkey` | Auto Click | +| 滑鼠 | `click_mouse`（`clicks=2` 就是雙擊）、`set_mouse_position`、`mouse_scroll`、`tween_drag`／`drag_path`（以 `step_delay_s`、`settle_s` 控制節奏） | `AC_click_mouse`、`AC_tween_drag`、`AC_drag_path` | Auto Click | +| 鍵盤 | `write`、`write_secret`、`hotkey`、`type_keyboard`、`keyboard_key_name`（鍵碼 → 標準鍵名；Windows 也收 `ctrl`、`esc`、`enter` 等別名） | `AC_write`、`AC_write_secret`、`AC_hotkey` | Auto Click | | 螢幕與像素 | `screenshot`、`screen_size`、`get_pixel` | `AC_screenshot` | Screenshot | | 影像比對 | `locate_image_center`、`locate_and_click` | `AC_locate_and_click` | Image Detect | -| OCR 文字 | `click_text`、`wait_for_text`、`read_text_in_region` | `AC_click_text`、`AC_wait_text` | OCR Reader | +| OCR 文字 | `click_text`、`wait_for_text`、`read_text_in_region`、`ocr_status` | `AC_click_text`、`AC_wait_text`、`AC_ocr_status` | OCR Reader | | 無障礙樹 | `find_accessibility_element`、`click_accessibility_element` | `AC_a11y_find`、`AC_a11y_click` | Accessibility | | 視覺模型定位 | `locate_by_description`、`click_by_description` | `AC_vlm_locate`、`AC_vlm_click` | VLM | | 錨點定位 | — | `AC_anchor_click`、`AC_anchor_locate` | — | @@ -142,7 +218,7 @@ python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" | 自然語言規劃 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | | 錄製與重播 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | -| JSON 腳本 | `execute_action`、`execute_files` | 全部 776 個指令 | Script、Script Builder | +| JSON 腳本 | `execute_action`、`execute_files` | 全部 778 個指令 | Script、Script Builder | | 變數與流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | | 資料驅動執行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | | 斷言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 個 | Assertions | @@ -183,7 +259,7 @@ je_auto_control version ``` `--var name=value` 會盡量以 JSON 解析（`count=10` 會變成整數），否則視為字串。 -`run` 只要有任何動作失敗就以 1 結束（仍會跑完整份腳本），CI 步驟會跟著失敗。舊版 `python -m je_auto_control -e file.json` 進入點仍然可用。 +`run` 只要有任何動作失敗就以 1 結束（仍會跑完整份腳本），CI 步驟會跟著失敗。舊版 `python -m je_auto_control -e file.json` 進入點仍然可用，它的 `-e`、`-d` 與 `--execute_str` 旗標也同樣在有動作失敗時以 1 結束。由 [TestPioneer](https://github.com/Integration-Automation/TestPioneer) 啟動時（它會設定 `TEST_PIONEER_ARTIFACT_DIR`），以相對名稱要求的 HTML、JSON、XML 報告會寫到該目錄底下。 --- @@ -191,7 +267,7 @@ je_auto_control version | 介面 | 啟動方式 | 說明 | |---|---|---| -| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 678 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用。除了以 `initialize` 握手的各版協定，也支援無狀態的 MCP 2026-07-28。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | +| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 680 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用：完整的 `ac_*` 工具介面，以及常用 GUI 操作的短名稱 alias。除了以 `initialize` 握手的各版協定，也支援無狀態的 MCP 2026-07-28。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | | **REST API** | `je_auto_control start-rest` | Bearer token、逐 IP 限流與鎖定、SQLite 稽核 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | | **TCP socket 伺服器** | `je_auto_control start-server` | 以換行分隔的 JSON 動作清單。預設綁 `127.0.0.1`。 | | **pytest 外掛** | 安裝後自動生效 | 提供 fixture 與供 pytest-bdd／behave 使用的 Gherkin step library。 | @@ -333,7 +409,7 @@ Windows、macOS（pyobjc）與 X11（含 XWayland）；純 Wayland session 的 ## 開發 ```bash -git clone https://github.com/Intergration-Automation-Testing/AutoControl.git +git clone https://github.com/Integration-Automation/AutoControlGUI.git cd AutoControl pip install -r dev_requirements.txt uv sync # 或：以已提交的 uv.lock 做可重現安裝 @@ -359,6 +435,6 @@ bandit -c pyproject.toml -r je_auto_control/ [MIT License](../LICENSE) © JE-Chen。 內含與選用第三方元件的授權請見 [Third_Party_License.md](../Third_Party_License.md)。 -- **首頁**：https://github.com/Intergration-Automation-Testing/AutoControl +- **首頁**：https://github.com/Integration-Automation/AutoControlGUI - **PyPI**：https://pypi.org/project/je_auto_control/ - **文件**：https://autocontrol.readthedocs.io/en/latest/ diff --git a/architecture.md b/architecture.md index 92c255dd0..c4ef2ffe6 100644 --- a/architecture.md +++ b/architecture.md @@ -53,7 +53,7 @@ entry points → execution core (`utils/executor/`) → headless capabilities (` | LSP | `autocontrol-lsp` → `autocontrol_lsp.server.server:run`; `python -m autocontrol_lsp.server` | Command list is read from the live executor. | | GUI | `start_autocontrol_gui()` in `gui/__init__.py`; `exe/start_autocontrol_gui.py` | Needs `pip install je_auto_control[gui]`; PySide6 is imported only under `gui/`. | | Action lint | `python -m je_auto_control.utils.action_lint` | Used by `.github/workflows/action-json-lint.yml`. | -| PyPI packages | `je_auto_control` (stable), `je_auto_control_dev` (dev channel) | Both ship the same `je_auto_control` import package. Stable: a push to `main` runs the `publish` job of `stable.yml`, which bumps `pyproject.toml`, uploads and tags. Dev: the `publish-dev` job of `dev.yml` runs after the headless suite on a push to `dev`, builds from `dev.toml` and uploads when the commit is still the tip of `dev` and the wheel differs from the newest published one; `scripts/dev_release.py` takes the version from PyPI (newest release plus one patch), so nothing is committed back. `dev.toml` declares what `pyproject.toml` declares (`test_dev_toml_parity.py`). | +| PyPI packages | `je_auto_control` (stable), `je_auto_control_dev` (dev channel) | Both ship the same `je_auto_control` import package. Stable: a push to `main` runs the `publish` job of `stable.yml`, which bumps `pyproject.toml`, uploads and tags. Dev: the `publish-dev` job of `dev.yml` runs after the headless suite on a push to `dev`, builds from `dev.toml` and uploads when the commit is still the tip of `dev` and the wheel differs from the newest published one; `scripts/dev_release.py` takes the version from PyPI (newest release plus one patch), so nothing is committed back. `dev.toml` declares what `pyproject.toml` declares (`test_dev_toml_parity.py`). Both jobs hold the PyPI token and install nothing but the hash-locked `.github/requirements/publish.txt` (`build`, `twine` and the `setuptools` backend, generated from `publish.in` beside it), then build with `python -m build --no-isolation`, so the backend is the locked one (`test_publish_tooling_lock.py`). | ## 4. Main flows @@ -129,14 +129,15 @@ wrapper/auto_control_record.record → OS listener (e.g. windows/record/win32_in | Consumer | How it uses this repo | What it relies on | | --- | --- | --- | -| Jeffrey_RPA | Editable install of **this working tree**: uncommitted changes here reach it immediately. Single facade `JeffreyRPA/_gui_control.py`. | Top-level names (e.g. `click_mouse`, `hotkey`, `write`, `screen_size`, `get_pixel`, `post_click_to_window`) and internal paths `je_auto_control.wrapper.auto_control_window`, `je_auto_control.wrapper.auto_control_keyboard.WRITE_CONTROL_KEYS`, `je_auto_control.utils.monitor_layout` (`logical_virtual_rect`, `enumerate_monitors`), and `wrapper.platform_wrapper.keyboard_keys_table` / `mouse_keys_table` — it validates every key name a user types against the keyboard table and reverse-looks-up recorded virtual keys through it, so a name removed there becomes a rejected hotkey over in that repo. | +| Jeffrey_RPA | Editable install of **this working tree**: uncommitted changes here reach it immediately. Single facade `JeffreyRPA/_gui_control.py`. | Top-level names (e.g. `click_mouse`, `hotkey`, `write`, `screen_size`, `get_pixel`, `post_click_to_window`) and internal paths `je_auto_control.wrapper.auto_control_window`, `je_auto_control.wrapper.auto_control_keyboard.WRITE_CONTROL_KEYS`, `je_auto_control.utils.monitor_layout` (`logical_virtual_rect`, `enumerate_monitors`), and `wrapper.platform_wrapper.keyboard_keys_table` / `mouse_keys_table` — it validates every key name a user types against the keyboard table and reverse-looks-up recorded virtual keys through it, so a name removed there becomes a rejected hotkey over in that repo. Added so it can drop its own copies: the Tesseract setup helpers `find_tesseract_cmd`, `set_tessdata_dir`, `ocr_languages` (`None` when the engine cannot be asked, `[]` when it has no language data) and `ocr_status`; `click_mouse(clicks=, interval=)` for a double-click in one call; `tween_drag` / `drag_path` pacing (`step_delay_s`, `settle_s`); `keyboard_key_name` and `wrapper.platform_wrapper.keyboard_key_aliases` for reverse lookups that never answer with an alias, plus the Windows table's OEM keys, `browser_home`, lower-case `launch_app2` and its aliases (`ctrl`, `esc`, `enter`, `win`, `pgup`, `numpad0`, ...), which match the names and codes of its own `KEY_ALIASES` / `_EXTRA_KEY_CODES`. | | PyBreeze | Subprocess `python -m je_auto_control --execute_str ` / `--execute_file `; on Windows the JSON string arrives double-encoded. | Legacy CLI flags; also embeds `je_auto_control.gui.main_widget.AutoControlGUIWidget` and calls `record` / `stop_record` in-process. | -| TestPioneer | Optional extra `gui = ["je_auto_control"]`; `parallel_run` starts `python -m je_auto_control --execute_file `. | `execute_action`, `execute_files`, `RecordingThread`; the `--execute_file` flag. | +| TestPioneer | Optional extra `gui = ["je_auto_control"]`; `parallel_run` starts `python -m je_auto_control --execute_file `. It sets `TEST_PIONEER_ARTIFACT_DIR` for the runner. | `execute_action`, `execute_files`, `RecordingThread`; the `--execute_file` flag. | The flag's exit status, 1 when an action failed (`recorded_failures()` in `__main__.py`); the `_success.json` / `_failure.json` pair, written below the artifact directory for a relative name (`utils/generate_report/report_path.py`). | WebRunner | Optional extra `autocontrol = ["je_auto_control>=0.0.224"]`; `je_web_runner/utils/autocontrol_bridge/` (`WR_ac_*`) imports this package only when one of those commands runs. | `je_auto_control.utils.executor.action_executor.executor`: `execute_action(actions, raise_on_error=True)`, whose record values it reads in action order, and `known_commands()`; its native commands send `AC_write` (`write_string`), `AC_write_secret` (`secret`, from `WR_ac_basic_auth`), `AC_type_keyboard` (`keycode`), `AC_get_keyboard_keys_table` (the `enter` or `return` key), `AC_locate_image_center` (`image`, `detect_threshold`) and `AC_click_mouse` (`mouse_keycode`, `x`, `y` in this DPI-aware process's coordinates). It refuses `AC_shell_command`, `AC_execute_process`, `AC_add_package_*`, `AC_execute_action`, `AC_execute_files`, `AC_run_agent` and `AC_web_*` by name, so renaming one of them would let it through WebRunner's bridge. | **Guarded by** `test/unit_test/headless/test_cross_project_contracts.py`: every legacy CLI flag (short and long, run as a real child process, including PyBreeze's double-encoded `--execute_str`), the facade names in the rows above -(Jeffrey_RPA's list is every `ac.` in `_gui_control.py`), the `auto_control_window` functions Jeffrey_RPA calls, +(Jeffrey_RPA's list is every `ac.` in `_gui_control.py`, plus the names and keyword parameters added so it +can drop its own copies), the `auto_control_window` functions Jeffrey_RPA calls, its three internal imports, the two key tables (shape everywhere, Windows key names on Windows), `AutoControlGUIWidget`, and the executor API and refused command names WebRunner's bridge relies on. The test only knows what this table knows: when a consumer starts relying on something new, add it to both. diff --git a/architecture_explore.md b/architecture_explore.md index 5d98e0e70..de1263480 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -6,7 +6,7 @@ > 擷取每個模組的 docstring 與頂層公開名稱；統計數字取自實際檔案，非估算。 > 指令數與公開 API 數以 `executor.known_commands()` 與 `je_auto_control.__all__` 在工作樹上實測取得。 > -> **掃描時間**：2026-10-01　**版本**：`pyproject.toml` version `0.0.221`　**分支**：`feat/coverage-to-80` +> **掃描時間**：2026-10-08　**版本**：`pyproject.toml` version `0.0.225`　**分支**：`feat/coverage-to-80` --- @@ -19,13 +19,13 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,063 | -| 程式碼總行數 | 157,101 | +| Python 模組總數（含周邊子專案） | 1,065 | +| 程式碼總行數 | 157,741 | | `je_auto_control/utils/` 子套件數 | 310 | -| `AC_*` 動作指令數（`known_commands()` 實測） | 776 | +| `AC_*` 動作指令數（`known_commands()` 實測） | 778 | | 套件門面 `__all__` 公開名稱數 | 1,244 | | GUI 分頁數（`main_widget` 註冊） | 48 | -| MCP 工具數（`build_default_tool_registry()` 實測） | 678 | +| MCP 工具數（`build_default_tool_registry()` 實測） | 680 | | `test_*.py` 測試檔／測試函式 | 478 / 4,654 | | 範例腳本 | 27 | @@ -92,7 +92,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 | 模式 | 落點 | 說明 | | --- | --- | --- | | **Strategy** | `wrapper/platform_wrapper.py` | 依 `sys.platform` 只匯入當前 OS 的 `keyboard`／`mouse`／`screen`／`recorder` 實作；Linux 再細分 Wayland／X11，Wayland 後端不可用時自動退回 XWayland 並記警告。新增平台不需要改 wrapper。 | -| **Facade** | `je_auto_control/__init__.py` | 把 1,200 個公開名稱集中再匯出，使用者只 `import je_auto_control`。`api/core.py` 另提供一個小而穩定的版本化門面（7 個名稱）給新整合使用。 | +| **Facade** | `je_auto_control/__init__.py` | 把 1,251 個公開名稱集中再匯出，使用者只 `import je_auto_control`。`api/core.py` 另提供一個小而穩定的版本化門面（7 個名稱）給新整合使用。 | | **Command** | `utils/executor/action_executor.py` | `Executor.event_dict` 是字串 → callable 的分派表；JSON 動作檔即指令序列，因此可錄製、序列化、重播、簽章。 | | **Observer** | `utils/callback/`、`utils/observer/`、`utils/triggers/` | 動作完成後觸發回呼；畫面出現／消失／變化與外部事件（webhook／IMAP／檔案）驅動腳本。 | | **Template Method** | `utils/generate_report/` | HTML／JSON／XML 三個產生器共用「收集紀錄 → 格式化 → 寫檔」骨架，各自實作渲染。 | @@ -154,8 +154,8 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `je_auto_control/__init__.py` | 1,971 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | -| `je_auto_control/__main__.py` | 87 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | +| `je_auto_control/__init__.py` | 1,978 | **套件門面**。集中匯入並再匯出 1,251 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | +| `je_auto_control/__main__.py` | 98 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | | `je_auto_control/cli.py` | 353 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | | `je_auto_control/api/__init__.py` | 22 | 版本化整合進入點。 | | `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約以此為起點，現已擴到整包（見「設定基線」）。 | @@ -171,14 +171,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `wrapper/platform_wrapper.py` | 116 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；八個名稱都帶著 `backend_contract` 的型別出去，其中 `keyboard`／`mouse` 因為四個分支綁的是三種互不相容的形狀，先落在私有的 `_keyboard`／`_mouse`（`Any`）上再標合約；載入失敗直接拋 `AutoControlException`（fail fast）。 | +| `wrapper/platform_wrapper.py` | 119 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；八個名稱都帶著 `backend_contract` 的型別出去，其中 `keyboard`／`mouse` 因為四個分支綁的是三種互不相容的形狀，先落在私有的 `_keyboard`／`_mouse`（`Any`）上再標合約；載入失敗直接拋 `AutoControlException`（fail fast）。 | | `wrapper/backend_contract.py` | 238 | 平台縫的型別合約：`ScreenBackend`／`KeyboardCheckBackend`／`RecorderBackend` 三個跨平台 Protocol，加上 `keyboard`／`mouse` 各自的三份——`Win32*`（SendInput 與 Interception）、`Darwin*`（Quartz）、`X11Unix*`（XTest／uinput／Wayland／BSD），因為這兩個名稱的呼叫形狀真的因平台而異；`KeyboardBackend`／`MouseBackend` 依 `sys.platform` 別名到其中一組，所以呼叫端被檢查的是它真的會走到的簽章。四個 `_platform_*` 組裝模組各自標注自己綁的是什麼，少一個成員就在該後端自己的檔案裡紅掉，而不是在三層之上的呼叫點。 | -| `wrapper/_platform_windows.py` | 318 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | -| `wrapper/_platform_osx.py` | 157 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | -| `wrapper/_platform_linux.py` | 275 | X11 後端組裝（python-Xlib + 選用 uinput）。 | -| `wrapper/_platform_wayland.py` | 58 | Wayland 後端組裝（libei／ydotool／grim）。 | -| `wrapper/auto_control_mouse.py` | 491 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | -| `wrapper/auto_control_keyboard.py` | 404 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。**`type_keyboard` 與 `hotkey` 的放開走 `finally`**（見下）。 | +| `wrapper/_platform_windows.py` | 366 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表（含 OEM 鍵與 `keyboard_key_aliases` 別名表）+ 選用 Interception 驅動。 | +| `wrapper/_platform_osx.py` | 162 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | +| `wrapper/_platform_linux.py` | 278 | X11 後端組裝（python-Xlib + 選用 uinput）。 | +| `wrapper/_platform_wayland.py` | 63 | Wayland 後端組裝（libei／ydotool／grim）。 | +| `wrapper/auto_control_mouse.py` | 537 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | +| `wrapper/auto_control_keyboard.py` | 432 | 鍵盤 API：鍵表查詢、`keyboard_key_name` 反查標準鍵名（不回別名）、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。**`type_keyboard` 與 `hotkey` 的放開走 `finally`**（見下）。 | | `wrapper/auto_control_screen.py` | 111 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | | `wrapper/auto_control_record.py` | 124 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | @@ -187,12 +187,12 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.3 平台後端 -#### Windows（`windows/`，23 檔／1,959 行） +#### Windows（`windows/`，23 檔／1,976 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | | `core/utils/win32_ctype_input.py` | 73 | `SendInput` 的 ctypes 結構定義與送出。 | -| `core/utils/win32_vk.py` | 188 | Windows 虛擬鍵碼對照表。 | +| `core/utils/win32_vk.py` | 205 | Windows 虛擬鍵碼對照表。 | | `core/utils/win32_keypress_check.py` | 22 | `GetAsyncKeyState` 按鍵狀態查詢。 | | `mouse/win32_ctype_mouse_control.py` | 220 | 滑鼠事件產生（含多螢幕絕對座標換算）。 | | `keyboard/win32_ctype_keyboard_control.py` | 98 | 鍵盤事件產生。 | @@ -272,7 +272,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 14,689 行。 +> 24 個套件、約 14,735 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -283,7 +283,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/dag/` | 536 | 跨主機 DAG 編排器（圖模型 + runner） | | `utils/decision_table/` | 112 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | | `utils/deterministic/` | 116 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | -| `utils/executor/` | 9,560 | **核心**。`Executor` 指令分派表（776 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/executor/` | 9,606 | **核心**。`Executor` 指令分派表（778 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | | `utils/flow_debugger/` | 166 | action list 的單步除錯器與追蹤器 | | `utils/input_macro/` | 462 | 定時輸入事件：錄製結果的整形（`timeline`／`InputRecorder`，Windows 與 macOS 共用）、重播與宣告式輸入序列 DSL | | `utils/json/` | 99 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | @@ -342,7 +342,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.4 輸入模擬與動作品質 -> 22 個套件、約 2,843 行。 +> 22 個套件、約 2,919 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -358,7 +358,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/ime_state/` | 149 | 讀取即時 IME 組字／轉換狀態，確保 CJK 輸入安全 | | `utils/key_hold/` | 109 | 按住按鍵一段時間，或以固定頻率自動重複 | | `utils/modifier_state/` | 81 | 跨一組動作按住修飾鍵，並保證安全釋放 | -| `utils/mouse_path/` | 115 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | +| `utils/mouse_path/` | 110 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | | `utils/mouse_relative/` | 59 | 相對位移滑鼠移動 | | `utils/postcondition/` | 157 | 宣告式的動作預期結果規格，對照畫面驗證 | | `utils/step_repair/` | 136 | 失敗／無效動作的修復策略（自我修正迴圈） | @@ -366,7 +366,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/input_reach/` | 111 | 送出去的輸入到不到得了：桌面鎖定查詢（免費）＋ 實際送一個 F13 確認沒有被過濾（有副作用，只給診斷用） | | `utils/keyboard_layout/` | 152 | 向系統問「這個鍵盤配置下每個鍵印出什麼字」（`ToUnicodeEx`），問不到退回 US 對照表 | | `utils/text_unicode/` | 151 | 輸入任意 Unicode（emoji／CJK／重音字）：優先送字元按鍵事件，不支援時退回剪貼簿貼上 | -| `utils/tween_drag/` | 101 | 沿曲線的緩動插值拖曳 | +| `utils/tween_drag/` | 182 | 沿曲線的緩動插值拖曳；`_drag_through` 是它與 `drag_path` 共用的按下／移動／放開（`step_delay_s`、`settle_s` 節奏，失敗時在 `finally` 裡於最後到達點放開） | | `utils/verify_field/` | 118 | 打字後讀回欄位，確認內容確實落地 | ### 5.4.5 影像辨識與畫面分析 @@ -415,7 +415,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.6 OCR 與文字理解 -> 19 個套件、約 3,576 行。 +> 19 個套件、約 3,803 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -428,7 +428,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/guardrail/` | 117 | 針對畫面／OCR 文字的啟發式 prompt-injection 防護 | | `utils/heading_segment/` | 71 | 判定 OCR 行是標題或內文，建出文件大綱 | | `utils/near_dup/` | 108 | 近似重複文字偵測（SimHash／MinHash） | -| `utils/ocr/` | 1,140 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`） | +| `utils/ocr/` | 1,367 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`）、Tesseract 的尋找／設定／健康檢查（`tesseract_setup`） | | `utils/pii_text/` | 141 | 自由文字中的 PII 偵測與遮蔽（email／電話／SSN／卡號／IP／IBAN） | | `utils/readability/` | 140 | 可讀性評分（Flesch、Flesch-Kincaid、Gunning Fog、SMOG、ARI） | | `utils/reading_flow/` | 165 | 以遞迴 XY-cut 推導欄位感知的閱讀順序 | @@ -494,7 +494,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 23,194 行。 +> 13 個套件、約 23,246 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -507,7 +507,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/cua_action/` | 204 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | | `utils/llm/` | 365 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 97 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 18,846 | **無頭 MCP 伺服器**（16K LOC，預設註冊 678 個工具＝659 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/mcp_server/` | 18,898 | **無頭 MCP 伺服器**（16K LOC，預設註冊 680 個工具＝661 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | | `utils/tool_use_schema/` | 195 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | | `utils/trajectory_eval/` | 132 | agent 軌跡評估：依評分規準為一次執行打分 | | `utils/vision/` | 538 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | @@ -558,7 +558,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 7,674 行。 +> 34 個套件、約 7,716 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -573,7 +573,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/failure_signature/` | 76 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | | `utils/flake_cluster/` | 112 | 以共同失敗 Jaccard 相似度為易碎測試分群 | | `utils/flakiness/` | 151 | 以執行歷史分析不穩定測試 | -| `utils/generate_report/` | 294 | HTML／JSON／XML 三種報表產生器（Template Method） | +| `utils/generate_report/` | 336 | HTML／JSON／XML 三種報表產生器（Template Method） | | `utils/media_assert/` | 273 | 媒體斷言：音訊活動與影片動態檢查 | | `utils/observability/` | 710 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | | `utils/otlp_export/` | 114 | OTLP/JSON span 匯出 | @@ -696,11 +696,11 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 上表以子套件為單位；以下把行數最大的幾個子系統展開到檔案層。 -#### `utils/executor/`（9,560 行）— 執行核心 +#### `utils/executor/`（9,606 行）— 執行核心 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `action_executor.py` | 8,341 | `Executor` 類別與 `event_dict` 分派表（776 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | +| `action_executor.py` | 8,387 | `Executor` 類別與 `event_dict` 分派表（778 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | | `flow_control.py` | 644 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | | `flow_data_commands.py` | 272 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | | `action_schema.py` | 159 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | @@ -708,15 +708,15 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | | `flags.py` | 19 | 旗標讀取 `as_bool`：依拼法（`"true"`／`"yes"`／`"on"`／`"1"`）而非 Python 真值判斷，所以 JSON、CLI、MCP 傳來的 `"false"` 是關；所有 adapter 與流程指令共用。 | -#### `utils/mcp_server/`（18,846 行，678 個工具）— 最大子系統 +#### `utils/mcp_server/`（18,898 行，680 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `tools/_factories.py` | 9,029 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | -| `tools/_handlers.py` | 545 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter；主題模組拆完之後這裡留的是資料／文字／HTTP 那一類與 WebRunner 橋接。 | +| `tools/_factories.py` | 9,060 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | +| `tools/_handlers.py` | 547 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter；主題模組拆完之後這裡留的是資料／文字／HTTP 那一類與 WebRunner 橋接。 | | `tools/_handlers_qa.py` | 419 | 同一種 adapter，QA 主題：斷言 DSL、資料驅動、SQL／PDF／郵件／HTTP 步驟、codegen、視覺回歸、狀態機、flaky 偵測與隔離、suite runner、無障礙稽核、裝置矩陣、媒體斷言。從 `_handlers.py` 依主題拆出的第一塊（750 行上限）；兩者互不引用。 | -| `tools/_handlers_input.py` | 218 | 同一種 adapter，輸入主題：滑鼠、鍵盤、虛擬手把（ViGEm）。 | -| `tools/_handlers_screen.py` | 331 | 同一種 adapter，螢幕主題：擷取、像素、影像與文字搜尋、螢幕錄影。 | +| `tools/_handlers_input.py` | 221 | 同一種 adapter，輸入主題：滑鼠、鍵盤、虛擬手把（ViGEm）。 | +| `tools/_handlers_screen.py` | 344 | 同一種 adapter，螢幕主題：擷取、像素、影像與文字搜尋、螢幕錄影。 | | `tools/_handlers_system.py` | 575 | 同一種 adapter，桌面工作階段：視窗、行程與 shell、開檔、閒置與睡眠、音量、鎖定、輸入法狀態、欄位驗證與重試、色彩對比、變更排序、元件分類、剪貼簿。 | | `tools/_handlers_runs.py` | 110 | 同一種 adapter，執行主題：executor、執行歷史、錄製、動作檔。 | | `tools/_handlers_scheduling.py` | 200 | 同一種 adapter，排程主題：排程器、觸發器、熱鍵常駐。 | @@ -735,7 +735,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `_protocol.py` | 243 | JSON-RPC 線路格式：版本與識別常數、`_MCPError`、決定失敗工具行為的錯誤 tuple、envelope 產生器、工具回傳值轉 `content` 區塊。不碰伺服器狀態。 | | `resources.py` | 307 | MCP resource 提供者。 | | `prompts.py` | 220 | MCP prompt 目錄。 | -| `fake_backend.py` | 184 | CI／無頭測試用的記憶體內假後端。 | +| `fake_backend.py` | 187 | CI／無頭測試用的記憶體內假後端。 | | `plugin_watcher.py` | 168 | 檔案變更時熱重載外掛工具的背景 watcher。 | | `tools/_base.py` | 146 | 工具註冊表的共用型別與輔助。 | | `tools/_validation.py` | 122 | MCP 工具用到的 JSON Schema 子集驗證器。 | @@ -838,7 +838,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | --- | --- | | `accessibility/` | `accessibility_api.py`（公開 API）、`element.py`（dataclass）、`tree.py`（遞迴樹傾印）、`recorder.py`（輪詢式事件錄製）、`backends/`：`base.py` 330 行抽象、`windows_backend.py` 801 行（comtypes UIA）、`windows_reads.py` 142 行（pattern／文字範圍／表頭／元素屬性的純讀取與 `UIA_READ_ERRORS`）、`windows_query.py` 176 行（UIA 搜尋起點、可中斷走訪、快取請求、NULL COM 指標判定與 `UIA_ERRORS`）、`windows_state.py` 98 行（控制項狀態讀取與密碼欄位判定）、`macos_backend.py` 125 行（pyobjc AX）、`null_backend.py` fallback | | `agent/` | `agent_loop.py`、`computer_use.py`、`backends/`：`anthropic.py`、`anthropic_computer_use.py`（644 行）、`_computer_toolset.py`（141 行，GA `computer_toolset_20260801` 的批次、依高解析度層級上限縮放截圖、座標換算與 `zoom` 裁切）、`openai.py`、`base.py` | -| `ocr/` | `ocr_engine.py`（門面）、`structure.py`（版面）、`backends/`：`tesseract_backend.py`、`easyocr_backend.py`、`paddleocr_backend.py`、`base.py` | +| `ocr/` | `ocr_engine.py`（門面）、`structure.py`（版面）、`tesseract_setup.py`（找執行檔、`TESSDATA_PREFIX`、`ocr_status`／`ocr_languages`；`None` 與 `[]` 分開回報）、`backends/`：`tesseract_backend.py`、`easyocr_backend.py`、`paddleocr_backend.py`、`base.py` | | `vision/` | `vlm_api.py`、`backends/`：`anthropic_backend.py`、`openai_backend.py`、`null_backend.py`、`_parse.py`、`base.py` | | `llm/` | `planner.py`、`backends/`：`anthropic_backend.py`、`null_backend.py`、`base.py` | | `hotkey/` | `hotkey_daemon.py`、`backends/`：`windows_backend.py`（RegisterHotKey + 訊息幫浦）、`linux_backend.py`（XGrabKey）、`macos_backend.py`（CGEventTap）、`base.py` | @@ -994,7 +994,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `scripts/dev_release.py` | 只用標準函式庫的發佈輔助腳本 | dev 通道（PyPI 的 `je_auto_control_dev`）的版本與比對：`prepare` 把 `dev.toml` 寫成 `pyproject.toml`，版本取 PyPI 最新一版加一個 patch（`dev.toml` 的版本只是下限，不回寫 repo）；`changed dist` 把建好的 wheel 跟 PyPI 最新一版逐檔比對，內容沒變就不上傳。只給 `dev.yml` 的 `publish-dev` job 與拋棄式的 checkout 用，因為 `prepare` 會蓋掉 `pyproject.toml`。 | | `docs/` | Sphinx（`API`／`Eng`／`Zh`／`getting_started`） | Read the Docs 文件。 | | `architecture_diagram/` | drawio + png | 既有的架構圖原始檔。 | -| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`verify`、`test_source` | 478 個 `test_*.py`／4,654 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 `verify/macos_verify.py` 是在真的 `macos-14` runner 上量測 TCC 到底允許什麼的探針（macOS 是唯一沒有容器可用的支援平台），不被 pytest 收集。 | +| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`verify`、`test_source` | 628 個 `test_*.py`／6,673 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 `verify/macos_verify.py` 是在真的 `macos-14` runner 上量測 TCC 到底允許什麼的探針（macOS 是唯一沒有容器可用的支援平台），不被 pytest 收集。 | --- @@ -1018,8 +1018,8 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | 檔案 | 用途 | | --- | --- | -| `stable.yml` | 每次 push／PR 到 `main` 與每日排程跑 Windows 五版本的示範腳本；合併到 main 後版本遞增並上傳 PyPI（使用 `PYPI_API_TOKEN`）。 | -| `dev.yml` | 每次 push／PR 到 `dev` 跑 headless pytest（四格：Windows 的 3.10 與 3.14、Ubuntu 3.14、macOS 3.10，都是 `quality.yml` 九格裡的格子，不量 coverage）。push 通過後由 `publish-dev` job 用 `dev.toml` 建置並上傳 `je_auto_control_dev`（同一個 `PYPI_API_TOKEN`）；只有這個 commit 仍是 `dev` 的最新一筆、而且 wheel 內容跟 PyPI 最新一版不同時才上傳，版本由 `scripts/dev_release.py` 向 PyPI 查，不回寫 repo。 | +| `stable.yml` | 每次 push／PR 到 `main` 與每日排程跑 Windows 五版本的示範腳本；合併到 main 後版本遞增並上傳 PyPI（使用 `PYPI_API_TOKEN`）。`publish` job 只安裝雜湊鎖定的 `.github/requirements/publish.txt`（`build`、`twine` 與建置後端 `setuptools`，由同目錄的 `publish.in` 用 `uv pip compile` 產生），並以 `python -m build --no-isolation` 建置，所以建置後端也是鎖定的那一版。 | +| `dev.yml` | 每次 push／PR 到 `dev` 跑 headless pytest（四格：Windows 的 3.10 與 3.14、Ubuntu 3.14、macOS 3.10，都是 `quality.yml` 九格裡的格子，不量 coverage）。push 通過後由 `publish-dev` job 用 `dev.toml` 建置並上傳 `je_auto_control_dev`（同一個 `PYPI_API_TOKEN`）；只有這個 commit 仍是 `dev` 的最新一筆、而且 wheel 內容跟 PyPI 最新一版不同時才上傳，版本由 `scripts/dev_release.py` 向 PyPI 查，不回寫 repo。`publish-dev` 跟 `stable.yml` 的 `publish` 一樣只安裝 `.github/requirements/publish.txt` 並以 `python -m build --no-isolation` 建置；`test_publish_tooling_lock.py` 守住這兩個 job 的安裝行、建置指令，以及鎖定版本滿足 `pyproject.toml`／`dev.toml` 的 `build-system.requires`。 | | `release.yml` | 發佈流程（上傳步驟目前關閉）。 | | `quality.yml` | ruff、bandit、dependency review、九格矩陣的 headless pytest（含 coverage 地板）與 mypy。 | | `platform-smoke.yml` | 跨平台煙霧測試。 | @@ -1066,32 +1066,36 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 其中一支專門釘住「`AutoControlKeyboardException` 不在那五個型別底下」這個**前提**——整個 設計靠它成立，前提哪天變了，這批註解也要跟著改。 +滑鼠拖曳是同一個形狀：`utils/tween_drag/tween_drag.py` 的 `_drag_through`（`tween_drag` 與 `mouse_path.drag_path` +共用）在按下之後用 `held` 旗標加 `finally`，任何一步丟例外都會在**游標最後到達的點**放開——不是終點，那等於替一次 +沒走完的拖曳完成放置——而收尾的放開失敗只記 log、不蓋掉原本的例外。守門：`test_drag_pacing.py`。 + --- ## 8. 附錄：各層規模 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | -| `gui/` | 95 | 27,829 | -| `utils/mcp_server/` | 35 | 18,846 | +| `gui/` | 95 | 27,853 | +| `utils/mcp_server/` | 35 | 18,898 | | `utils/remote_desktop/` | 56 | 13,014 | -| `utils/executor/` | 8 | 9,560 | +| `utils/executor/` | 8 | 9,606 | | `utils/usb/` | 17 | 4,572 | -| `je_auto_control/`（頂層 3 檔） | 3 | 2,411 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,429 | | `utils/accessibility/` | 14 | 3,143 | -| `wrapper/` | 19 | 3,667 | -| `windows/` | 23 | 1,959 | +| `wrapper/` | 19 | 3,805 | +| `windows/` | 23 | 1,976 | | `utils/rest_api/` | 8 | 1,881 | | `utils/agent/` | 9 | 1,975 | | `linux_with_x11/` | 19 | 1,281 | | `linux_wayland/` | 17 | 2,921 | | `utils/triggers/` | 4 | 1,383 | -| `utils/ocr/` | 9 | 1,140 | +| `utils/ocr/` | 10 | 1,367 | | `utils/usbip/` | 5 | 1,008 | | `utils/assertion/` | 3 | 887 | | `osx/` | 17 | 925 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 852 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 681 | 57,038 | -| **總計** | **1,057** | **157,036** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 682 | 57,156 | +| **總計** | **1,059** | **157,676** | diff --git a/autocontrol-lsp/vscode/package-lock.json b/autocontrol-lsp/vscode/package-lock.json index f24654af7..13bce8ce6 100644 --- a/autocontrol-lsp/vscode/package-lock.json +++ b/autocontrol-lsp/vscode/package-lock.json @@ -485,9 +485,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", - "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", "dev": true, "license": "MIT", "dependencies": { diff --git a/dev.toml b/dev.toml index 2bd513bc9..0b331cb40 100644 --- a/dev.toml +++ b/dev.toml @@ -51,9 +51,9 @@ je_auto_control_mcp = "je_auto_control.utils.mcp_server.__main__:main" je_auto_control = "je_auto_control.utils.pytest_plugin.plugin" [project.urls] -Homepage = "https://github.com/Intergration-Automation-Testing/AutoControl" +Homepage = "https://github.com/Integration-Automation/AutoControlGUI" Documentation = "https://autocontrol.readthedocs.io/en/latest/" -Code = "https://github.com/Intergration-Automation-Testing/AutoControl" +Code = "https://github.com/Integration-Automation/AutoControlGUI" [project.readme] file = "README.md" diff --git a/dev_requirements.txt b/dev_requirements.txt index ab10d821f..541244f75 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -21,11 +21,11 @@ python-docx==1.2.0 python-pptx==1.0.2 # Quality tooling — used by .github/workflows/quality.yml and locally. -ruff==0.15.22 +ruff==0.16.0 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 -pytest-rerunfailures==15.1 +pytest-rerunfailures==16.4 # `coverage`, not `pytest-cov`: this package registers a `pytest11` entry # point, so pytest imports the facade before pytest-cov starts measuring and # `pytest --cov` under-reports by ~24 points. Measure with @@ -34,4 +34,4 @@ coverage==7.15.4 # The complexity limit in CLAUDE.md is measured by this, both in the # pre-commit list and by test/unit_test/headless/test_complexity_budget.py. radon==6.0.1 -mypy>=1.15 +mypy>=2.3.0 diff --git a/docs/source/API/special/keyboard_keys.rst b/docs/source/API/special/keyboard_keys.rst index dd7cc6790..16796e83f 100644 --- a/docs/source/API/special/keyboard_keys.rst +++ b/docs/source/API/special/keyboard_keys.rst @@ -14,11 +14,12 @@ in keyboard functions. Windows ======= -Common keys: +Common keys. The first name is the Win32 one a reverse lookup +(``keyboard_key_name``) returns; the others are aliases for the same key: .. list-table:: :header-rows: 1 - :widths: 30 70 + :widths: 40 60 * - Key Name - Description @@ -28,46 +29,55 @@ Common keys: - Number keys * - ``f1`` - ``f24`` - Function keys - * - ``enter``, ``return`` + * - ``return``, ``enter`` - Enter / Return * - ``tab`` - Tab * - ``space`` - Space bar - * - ``back`` + * - ``back``, ``backspace``, ``bksp`` - Backspace - * - ``escape`` + * - ``escape``, ``esc`` - Escape - * - ``lcontrol``, ``rcontrol`` - - Left / Right Control - * - ``lshift``, ``rshift`` - - Left / Right Shift - * - ``lalt``, ``ralt`` - - Left / Right Alt - * - ``lwin``, ``rwin`` - - Left / Right Windows key + * - ``control``, ``ctrl``; ``lcontrol``, ``lctrl``; ``rcontrol``, ``rctrl`` + - Control, left / right Control + * - ``shift``, ``lshift``, ``rshift`` + - Shift, left / right Shift + * - ``menu``, ``alt``; ``lmenu``, ``lalt``; ``rmenu``, ``ralt`` + - Alt, left / right Alt + * - ``lwin``, ``win``, ``super``, ``cmd``, ``meta``; ``rwin`` + - Left / right Windows key * - ``up``, ``down``, ``left``, ``right`` - Arrow keys - * - ``insert``, ``delete`` + * - ``insert``, ``ins``; ``delete``, ``del`` - Insert / Delete * - ``home``, ``end`` - Home / End - * - ``pageup``, ``pagedown`` + * - ``prior``, ``pgup``, ``pageup``; ``next``, ``pgdn``, ``pagedown`` - Page Up / Page Down - * - ``capslock`` + * - ``capital``, ``caps``, ``capslock`` - Caps Lock - * - ``numlock`` - - Num Lock - * - ``print_screen`` + * - ``numlock``; ``scroll``, ``scrolllock`` + - Num Lock / Scroll Lock + * - ``snapshot``, ``printscreen``, ``prtsc``, ``prtscr`` - Print Screen - * - ``numpad0`` - ``numpad9`` + * - ``num0`` - ``num9``, ``numpad0`` - ``numpad9`` - Numpad keys - * - ``add``, ``subtract``, ``multiply``, ``divide`` + * - ``add``, ``subtract``, ``multiply``, ``divide``, ``decimal`` - Numpad operators + * - ``oem_plus``, ``plus``; ``oem_comma``, ``comma``; ``oem_minus``, ``minus``; ``oem_period``, ``period`` + - ``=`` / ``,`` / ``-`` / ``.`` keys, the same key on every layout + * - ``oem_1`` - ``oem_8``, ``oem_102`` + - Punctuation keys whose character depends on the layout (``oem_1`` is + ``;`` on a US layout); they have no character-named alias on purpose + * - ``oem_clear`` + - Clear * - ``apps`` - Application / Menu key - * - ``browser_back``, ``browser_forward`` + * - ``browser_back``, ``browser_forward``, ``browser_home`` - Browser navigation keys + * - ``launch_app1``, ``launch_app2`` + - Launch application keys (``LAUNCH_APP2`` also still works) * - ``volume_mute``, ``volume_up``, ``volume_down`` - Volume control keys @@ -77,8 +87,8 @@ Common keys: .. code-block:: python - from je_auto_control import keys_table - for key_name in sorted(keys_table.keys()): + from je_auto_control import keyboard_keys_table + for key_name in sorted(keyboard_keys_table.keys()): print(key_name) Linux (X11) diff --git a/docs/source/API/wrapper/keyboard.rst b/docs/source/API/wrapper/keyboard.rst index 8850ca4fb..c0ce8f0b1 100644 --- a/docs/source/API/wrapper/keyboard.rst +++ b/docs/source/API/wrapper/keyboard.rst @@ -32,6 +32,26 @@ get_keyboard_keys_table ---- +keyboard_key_name +================= + +.. function:: keyboard_key_name(keycode) + + Returns the canonical name of a key code, for recorders and anything else + that turns a captured virtual key back into a name. A code often has + several names in the table; aliases (``esc`` for ``escape``, listed in + ``platform_wrapper.keyboard_key_aliases``) never answer, so adding one + cannot change the names a recording writes. Among the rest, a name of + lower-case letters, digits and underscores wins, then the shortest, then + the alphabetically first. + + :param int keycode: Platform key code (a Win32 virtual key on Windows). + :returns: The name, e.g. ``"escape"`` for 27 on Windows, or ``None`` when + the table has no name for the code. + :rtype: str or None + +---- + press_keyboard_key ================== diff --git a/docs/source/API/wrapper/mouse.rst b/docs/source/API/wrapper/mouse.rst index 2ec151dc8..a1de4c4a1 100644 --- a/docs/source/API/wrapper/mouse.rst +++ b/docs/source/API/wrapper/mouse.rst @@ -98,14 +98,21 @@ release_mouse click_mouse =========== -.. function:: click_mouse(mouse_keycode, x=None, y=None) +.. function:: click_mouse(mouse_keycode, x=None, y=None, clicks=1, interval=0.0) - Presses and releases a mouse button at the specified position. + Presses and releases a mouse button at the specified position, ``clicks`` + times, ``interval`` seconds apart; ``clicks=2`` double-clicks. Windows and + X11 recognise a double-click from the timing and distance of the clicks, so + keep ``interval`` under the system double-click time. On macOS the clicks + arrive as separate single clicks. :param mouse_keycode: Mouse button name (e.g., ``"mouse_left"``). :type mouse_keycode: int or str :param int x: X position (default: current position). :param int y: Y position (default: current position). + :param int clicks: Number of clicks, at least 1 (default: 1). + :param float interval: Seconds between clicks, at least 0 (default: 0.0). + :raises AutoControlMouseException: on an invalid ``clicks`` or ``interval``. :returns: Tuple of ``(keycode, x, y)``. :rtype: tuple diff --git a/docs/source/Eng/doc/ai_agents/ai_agents_doc.rst b/docs/source/Eng/doc/ai_agents/ai_agents_doc.rst new file mode 100644 index 000000000..3d050b8f2 --- /dev/null +++ b/docs/source/Eng/doc/ai_agents/ai_agents_doc.rst @@ -0,0 +1,120 @@ +AI agents and computer use +=========================== + +AutoControl is an open-source, cross-platform computer-use and GUI automation framework for AI agents. + +Use it when an agent needs to control a real desktop GUI rather than only a browser DOM or API. + +When to use AutoControl +----------------------- + +AutoControl is a good fit for: + +* real mouse and keyboard input; +* screenshots and screen-coordinate interaction; +* OCR and accessibility-tree discovery; +* image/template matching; +* vision-language-model UI localization; +* anchor and self-healing locators; +* native desktop applications and real browsers; +* cross-platform desktop and mobile automation; +* deterministic JSON action files; +* MCP-based computer use. + +MCP for AI agents +----------------- + +Install and start the stdio MCP server: + +.. code-block:: bash + + pip install je_auto_control + je_auto_control_mcp + +The server exposes the canonical ac_* tools and short, model-friendly aliases for common actions: + +* click +* move_mouse +* scroll +* type +* press +* hotkey +* screenshot +* screen_size +* find_image +* find_text +* click_text +* drag +* list_windows +* focus_window +* wait_image +* wait_pixel + +Disable aliases when a client needs only canonical tools: + +.. code-block:: bash + + JE_AUTOCONTROL_MCP_ALIASES=0 je_auto_control_mcp + +Use read-only mode for inspection-only clients: + +.. code-block:: bash + + je_auto_control_mcp --read-only + +Recommended agent loop +---------------------- + +#. Observe the current screen. +#. Identify the target with accessibility, OCR, image matching, or VLM. +#. Perform the smallest required action. +#. Wait for the interface to settle. +#. Verify the expected result. +#. Retry with another locator strategy if the UI changed. + +OpenAI tool selection +--------------------- + +OpenAI Chat Completions has a provider-side tool-count limit. Do not pass the complete AutoControl catalogue to an OpenAI agent. Export a focused allow-list: + +.. code-block:: python + + from je_auto_control.utils.tool_use_schema import export_openai_tools + + tools = export_openai_tools(only=[ + "AC_screenshot", + "AC_click_mouse", + "AC_write", + "AC_hotkey", + "AC_click_text", + ]) + +Focused toolsets are also safer: keep shell, process execution, package loading, and recursive agent commands out of the list unless explicitly required. + +Examples +-------- + +For the closed-loop Python agent see the repository example at examples/05_agent_loop.py. + +A stdio MCP client can launch AutoControl with: + +.. code-block:: json + + { + "mcpServers": { + "autocontrol": { + "command": "je_auto_control_mcp" + } + } + } + +Security +-------- + +An AI-controlled GUI process has control over the host machine. + +* Keep servers on loopback unless remote access is intentional. +* Use --read-only for inspection-only clients. +* Prefer explicit tool allow-lists. +* Do not expose shell/process/package-loading tools to untrusted models. +* Use authentication, audit, rate limiting, and confirmation controls for service deployments. diff --git a/docs/source/Eng/doc/keyboard/keyboard_doc.rst b/docs/source/Eng/doc/keyboard/keyboard_doc.rst index 6de8673c7..ea0468b7b 100644 --- a/docs/source/Eng/doc/keyboard/keyboard_doc.rst +++ b/docs/source/Eng/doc/keyboard/keyboard_doc.rst @@ -24,6 +24,19 @@ Retrieve available key names: See :doc:`/API/special/keyboard_keys` for the full list of available keyboard keys per platform. +On Windows the table also accepts common aliases (``ctrl``, ``alt``, ``enter``, +``esc``, ``win``, ``backspace``, ``del``, ``pgup``, ``capslock``, ``prtsc``, +``numpad0`` ...). To turn a captured key code back into a name, use +``keyboard_key_name``; it always answers with the canonical name, never an +alias: + +.. code-block:: python + + from je_auto_control import keyboard_key_name + + keyboard_key_name(27) # "escape" on Windows + keyboard_key_name(0xBB) # "oem_plus" + Press and Release ================= diff --git a/docs/source/Eng/doc/mouse/mouse_doc.rst b/docs/source/Eng/doc/mouse/mouse_doc.rst index 6f1654326..6880c342a 100644 --- a/docs/source/Eng/doc/mouse/mouse_doc.rst +++ b/docs/source/Eng/doc/mouse/mouse_doc.rst @@ -49,6 +49,13 @@ Press and immediately release a mouse button: # Left click at specific coordinates click_mouse("mouse_left", x=500, y=300) + # Double-click: two clicks on the same point, 60 ms apart + click_mouse("mouse_left", x=500, y=300, clicks=2, interval=0.06) + +Windows and X11 recognise a double-click from the timing and distance of the +clicks, so keep ``interval`` under the system double-click time (500 ms by +default on Windows). On macOS the clicks arrive as separate single clicks. + Position ======== diff --git a/docs/source/Eng/doc/new_features/v116_features_doc.rst b/docs/source/Eng/doc/new_features/v116_features_doc.rst index 5355aad6d..f0124825b 100644 --- a/docs/source/Eng/doc/new_features/v116_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v116_features_doc.rst @@ -31,15 +31,16 @@ steps (``easing`` is any name from ``path_easings()`` — ``linear`` / ``ease_in_out_quad`` / ``ease_out_cubic`` / ``ease_in_cubic``) and does not duplicate the shared junction points. ``move_along_path`` emits move events through the path; ``drag_path`` presses at the first waypoint, moves through the -whole path, and releases at the last — for multi-stop drags. If a move fails -partway, the button is released where the pointer stopped, so an aborted drag -does not drop at its target. Both take a ``sink`` -override for headless testing. +whole path, and releases at the last — for multi-stop drags. Both take a ``sink`` +override for headless testing. ``drag_path`` also takes ``tween_drag``'s pacing, +``step_delay_s`` (after each move) and ``settle_s`` (before the press, after it and +before the release), and like it releases the button where the pointer stopped if a +step raises. Executor commands ----------------- ``AC_move_along_path`` and ``AC_drag_path`` take ``waypoints`` (a JSON -``[[x, y], ...]`` list) plus ``easing`` / ``per_segment_steps`` (and ``button`` -for the drag). Both are exposed as MCP tools (``ac_move_along_path`` / +``[[x, y], ...]`` list) plus ``easing`` / ``per_segment_steps`` (and ``button``, +``step_delay_s`` and ``settle_s`` for the drag). Both are exposed as MCP tools (``ac_move_along_path`` / ``ac_drag_path``) and as Script Builder commands under **Mouse**. diff --git a/docs/source/Eng/doc/new_features/v29_features_doc.rst b/docs/source/Eng/doc/new_features/v29_features_doc.rst index cc6b319d6..0b467ae42 100644 --- a/docs/source/Eng/doc/new_features/v29_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v29_features_doc.rst @@ -27,3 +27,14 @@ coordinates; ``tween_drag`` presses at the start, moves through the points, and releases at the end. Easings: ``linear`` / ``ease_in_out_quad`` / ``ease_out_cubic`` / ``ease_in_cubic`` (see :func:`easing_names`). Exposed as ``AC_tween_drag`` / ``ac_tween_drag`` (``start`` / ``end`` as ``[x, y]``). + +Pacing, for apps that tell a drag from a click by the pointer's motion +(file managers, drawing programs, games):: + + tween_drag((0, 0), (300, 200), steps=24, step_delay_s=0.012, settle_s=0.08) + +``step_delay_s`` rests after each move; ``settle_s`` rests on the start +before the press, after the press and before the release. Both default to +0 (no pause) and must be finite and non-negative. If a step raises, the +button is released in a ``finally`` at the last point the pointer reached, +not at the end, and the original error propagates. diff --git a/docs/source/Eng/doc/new_features/v2_features_doc.rst b/docs/source/Eng/doc/new_features/v2_features_doc.rst index 8a211b557..ba7f0eb99 100644 --- a/docs/source/Eng/doc/new_features/v2_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v2_features_doc.rst @@ -399,10 +399,10 @@ Generic agent loop (JSON + MCP) language and the MCP tool registry. Parameters: * ``goal`` — natural-language objective. -* ``backend`` — ``"anthropic"`` (uses ``export_anthropic_tools()`` - with tool-use messages; each screenshot is fitted into the model's image - tier and the ``x`` / ``y`` of a tool call mapped back to the screen) or ``"openai"`` (uses ``export_openai_tools()`` - with Chat Completions function calling). +* ``backend`` — ``"anthropic"`` or ``"openai"``. ``AC_run_agent`` uses a focused, low-risk + computer-use allow-list by default instead of exposing the complete ``AC_*`` catalogue. + Applications that need a custom set should construct the backend with + ``export_anthropic_tools(only=[...])`` or ``export_openai_tools(only=[...])``. * ``max_steps`` (default 25) and ``wall_seconds`` (default 300.0). * ``model`` / ``max_tokens`` — backend-specific overrides. diff --git a/docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst b/docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst index 9a0487925..138f5e366 100644 --- a/docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst +++ b/docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst @@ -63,14 +63,30 @@ Tesseract setup notes ===================== Windows users need to install the Tesseract binary separately (UB-Mannheim -build is the most common). If ``tesseract.exe`` is not on ``PATH``:: +build is the most common); its installer does not add itself to ``PATH``. +``find_tesseract_cmd()`` looks for the executable in ``$TESSERACT_CMD`` +(when it names an existing file), then ``PATH``, then the installers' +default folders (``Program Files``, ``Program Files (x86)`` and the per-user +``%LOCALAPPDATA%\\Programs`` on Windows; Homebrew, ``/usr/local`` and +MacPorts on macOS; ``/usr/bin`` and ``/usr/local/bin`` elsewhere). It only +looks; hand the result to ``set_tesseract_cmd`` to make OCR use it:: - from je_auto_control import set_tesseract_cmd - set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") + from je_auto_control import find_tesseract_cmd, set_tesseract_cmd + + command = find_tesseract_cmd() + if command: + set_tesseract_cmd(command) Language pack files (``*.traineddata``) live under ``Tesseract-OCR\\tessdata\\`` — copy the ones you need from the -`tessdata GitHub repo `_. +`tessdata GitHub repo `_. That +folder needs administrator rights on Windows. To keep the files somewhere +writable instead, point the engine there; this sets ``TESSDATA_PREFIX`` for +the process, and ``None`` restores the engine's default:: + + from je_auto_control import set_tessdata_dir + + set_tessdata_dir("ocr_tessdata") # returns the resolved path EasyOCR / PaddleOCR setup notes ================================ @@ -106,12 +122,36 @@ OCR commands are exposed through the executor for JSON action files:: {"command": "AC_locate_text", "text": "Sign in", "lang": "eng"} {"command": "AC_click_text", "text": "OK", "backend": "easyocr"} + {"command": "AC_ocr_status"} + {"command": "AC_ocr_languages"} -Both commands accept the same arguments as their Python counterparts. +The commands accept the same arguments as their Python counterparts. +``AC_ocr_status`` returns ``{"ok": ..., "reason": ...}`` and +``AC_ocr_languages`` returns ``{"languages": [...]}``, with ``null`` when the +engine cannot be asked. Diagnostics =========== +To ask whether Tesseract can work here, and which part is missing if not:: + + from je_auto_control import ocr_languages, ocr_status + + ok, reason = ocr_status() + # reason: "ready", "missing_package" (no pytesseract), "missing_engine" + # (the configured command is not found), "engine_unusable" (it does not + # run), or "no_language_data" (it ran and listed no languages) + + languages = ocr_languages() + # ["chi_tra", "eng", ...] installed language codes, sorted + # [] the engine answered: no language data at all + # None the engine could not be asked + +``ocr_status`` checks the command OCR would use and changes nothing. +``ocr_languages`` asks the engine on every call, so language files added +while the process runs are seen. Keep ``None`` and ``[]`` apart: they call +for opposite handling. + To check which backends are reachable in the current environment:: from je_auto_control.utils.ocr.backends import get_backend diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e3f679e68..e826e9bb4 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -20,6 +20,8 @@ Comprehensive guides for all AutoControl features. doc/scheduler/scheduler_doc doc/socket_driver/socket_driver_doc doc/mcp_server/mcp_server_doc + doc/ai_agents/ai_agents_doc + doc/ai_agents/ai_agents_doc doc/critical_exit/critical_exit_doc doc/cli/cli_doc doc/create_project/create_project_doc diff --git a/docs/source/Zh/doc/ai_agents/ai_agents_doc.rst b/docs/source/Zh/doc/ai_agents/ai_agents_doc.rst new file mode 100644 index 000000000..484023335 --- /dev/null +++ b/docs/source/Zh/doc/ai_agents/ai_agents_doc.rst @@ -0,0 +1,116 @@ +AI Agent 與 Computer Use +=========================== + +AutoControl 是開源、跨平台的 **computer-use 與 GUI 自動化框架**，可以讓 AI agent 操作真實桌面，而不只依賴瀏覽器 DOM 或 API。 + +何時使用 AutoControl +-------------------- + +適合以下情境： + +* 真實滑鼠與鍵盤輸入； +* 螢幕擷取與座標操作； +* OCR 與作業系統無障礙樹； +* 影像／樣板比對； +* Vision-Language Model UI 定位； +* Anchor 與 self-healing locator； +* 原生桌面應用程式與真實瀏覽器； +* 跨平台桌面與行動裝置自動化； +* JSON 動作檔； +* 透過 MCP 讓 AI agent 使用 computer use。 + +MCP for AI agents +----------------- + +安裝並啟動 stdio MCP server： + +.. code-block:: bash + + pip install je_auto_control + je_auto_control_mcp + +MCP 同時提供完整的 ac_* 工具，以及給模型使用的短名稱 alias： + +* click +* move_mouse +* scroll +* type +* press +* hotkey +* screenshot +* screen_size +* find_image +* find_text +* click_text +* drag +* list_windows +* focus_window +* wait_image +* wait_pixel + +若 client 只需要 canonical ac_* 工具： + +.. code-block:: bash + + JE_AUTOCONTROL_MCP_ALIASES=0 je_auto_control_mcp + +只做檢查、不允許修改的 client： + +.. code-block:: bash + + je_auto_control_mcp --read-only + +建議的 Agent Loop +----------------- + +#. 觀察目前畫面。 +#. 優先用無障礙樹、OCR、影像比對或 VLM 找目標。 +#. 執行最小必要操作。 +#. 等待介面穩定。 +#. 驗證預期結果。 +#. UI 改變時切換另一種 locator 策略重新嘗試。 + +OpenAI 工具選擇 +--------------- + +OpenAI Chat Completions 有 provider 的工具數量限制，因此不要把整個 AutoControl 命令目錄一次交給 OpenAI agent。請針對任務建立 allow-list： + +.. code-block:: python + + from je_auto_control.utils.tool_use_schema import export_openai_tools + + tools = export_openai_tools(only=[ + "AC_screenshot", + "AC_click_mouse", + "AC_write", + "AC_hotkey", + "AC_click_text", + ]) + +聚焦工具集也比較安全：除非真的需要，否則不要提供 shell、process execution、package loading 或遞迴 agent 工具。 + +MCP client 範例 +--------------- + +stdio MCP client 可以直接啟動 AutoControl： + +.. code-block:: json + + { + "mcpServers": { + "autocontrol": { + "command": "je_auto_control_mcp" + } + } + } + +安全性 +------ + +AI 控制的 GUI process 具有主機操作權限。 + +* 非必要不要把 server 綁到公開網路。 +* 檢查型 client 使用 --read-only。 +* AI agent 優先使用明確的工具 allow-list。 +* 不要把 shell/process/package-loading 工具暴露給不受信任的模型。 +* 作為服務部署時使用 authentication、audit、rate limit 與 confirmation 控制。 diff --git a/docs/source/Zh/doc/keyboard/keyboard_doc.rst b/docs/source/Zh/doc/keyboard/keyboard_doc.rst index a54d9c1e4..971401dae 100644 --- a/docs/source/Zh/doc/keyboard/keyboard_doc.rst +++ b/docs/source/Zh/doc/keyboard/keyboard_doc.rst @@ -24,6 +24,17 @@ AutoControl 提供模擬鍵盤輸入的功能，包括按鍵按下/釋放、輸 完整的鍵盤按鍵列表請參考 :doc:`/API/special/keyboard_keys`。 +Windows 的按鍵表也收常見的別名（``ctrl``、``alt``、``enter``、``esc``、``win``、``backspace``、 +``del``、``pgup``、``capslock``、``prtsc``、``numpad0``……）。要把錄到的鍵碼換回名字，用 +``keyboard_key_name``；它一律回標準名稱，不會回別名： + +.. code-block:: python + + from je_auto_control import keyboard_key_name + + keyboard_key_name(27) # Windows 上是 "escape" + keyboard_key_name(0xBB) # "oem_plus" + 按下與釋放 ========== diff --git a/docs/source/Zh/doc/mouse/mouse_doc.rst b/docs/source/Zh/doc/mouse/mouse_doc.rst index f4a680b0e..af5530a80 100644 --- a/docs/source/Zh/doc/mouse/mouse_doc.rst +++ b/docs/source/Zh/doc/mouse/mouse_doc.rst @@ -48,6 +48,12 @@ AutoControl 提供模擬滑鼠操作的功能，包括點擊、定位、捲動 # 在指定座標左鍵點擊 click_mouse("mouse_left", x=500, y=300) + # 雙擊：在同一點連點兩下，間隔 60 毫秒 + click_mouse("mouse_left", x=500, y=300, clicks=2, interval=0.06) + +Windows 與 X11 依兩次點擊的時間差與位移判定雙擊，所以 ``interval`` 要小於系統的雙擊判定時間 +（Windows 預設 500 毫秒）。macOS 上這些點擊會被當成各自獨立的單擊。 + 游標位置 ======== diff --git a/docs/source/Zh/doc/new_features/v116_features_doc.rst b/docs/source/Zh/doc/new_features/v116_features_doc.rst index aea6535df..8c779a5df 100644 --- a/docs/source/Zh/doc/new_features/v116_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v116_features_doc.rst @@ -25,11 +25,13 @@ ``plan_path`` 以 ``per_segment_steps`` 個緩動步驟對每個相鄰點對插值(``easing`` 為 ``path_easings()`` 中任一 名稱——``linear`` / ``ease_in_out_quad`` / ``ease_out_cubic`` / ``ease_in_cubic``),且不重複共用的交接點。 ``move_along_path`` 沿路徑發出移動事件;``drag_path`` 在第一個路徑點按下、移動經過整段路徑、在最後一點放開—— -用於多停靠點拖曳。途中某次移動失敗時,會在指標停下的位置放開,所以中斷的拖曳不會在目標上放下。兩者皆可傳入 ``sink`` 以供無頭測試。 +用於多停靠點拖曳。兩者皆可傳入 ``sink`` 以供無頭測試。``drag_path`` 也接受 ``tween_drag`` 的節奏參數 +``step_delay_s``(每次移動後)與 ``settle_s``(按下前、按下後、放開前),而且和它一樣,任何一步丟出例外時會在 +游標停下的位置放開按鍵。 執行器命令 ---------- ``AC_move_along_path`` 與 ``AC_drag_path`` 接受 ``waypoints``(JSON ``[[x, y], ...]`` 列表)以及 ``easing`` / -``per_segment_steps``(拖曳另含 ``button``)。兩者皆以 MCP 工具(``ac_move_along_path`` / ``ac_drag_path``) +``per_segment_steps``(拖曳另含 ``button``、``step_delay_s`` 與 ``settle_s``)。兩者皆以 MCP 工具(``ac_move_along_path`` / ``ac_drag_path``) 以及 Script Builder 中 **Mouse** 分類下的命令提供。 diff --git a/docs/source/Zh/doc/new_features/v29_features_doc.rst b/docs/source/Zh/doc/new_features/v29_features_doc.rst index 728c8c4ad..40b908f9e 100644 --- a/docs/source/Zh/doc/new_features/v29_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v29_features_doc.rst @@ -26,3 +26,11 @@ ``ease_in_out_quad`` / ``ease_out_cubic`` / ``ease_in_cubic``(見 :func:`easing_names`)。對應 ``AC_tween_drag`` / ``ac_tween_drag`` (``start`` / ``end`` 以 ``[x, y]`` 表示)。 + +節奏控制:有些應用程式(檔案總管、繪圖軟體、遊戲)靠游標的移動判定拖曳,而不是點擊:: + + tween_drag((0, 0), (300, 200), steps=24, step_delay_s=0.012, settle_s=0.08) + +``step_delay_s`` 是每次移動後停多久;``settle_s`` 是按下前先停在起點、按下後、放開前各停多久。 +兩者預設 0(不停),必須是有限的非負數。任何一步丟出例外時,按鍵會在 ``finally`` 裡於游標 +最後到達的位置放開,而不是終點,原本的例外照樣往外丟。 diff --git a/docs/source/Zh/doc/new_features/v2_features_doc.rst b/docs/source/Zh/doc/new_features/v2_features_doc.rst index 81e5045ed..d5b457d7f 100644 --- a/docs/source/Zh/doc/new_features/v2_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v2_features_doc.rst @@ -369,9 +369,9 @@ helper（``je_auto_control.gui.flow_editor.layout_steps``）可單元 （規劃 → 執行 → 驗證 → 重試）開放給 JSON action 與 MCP。參數： * ``goal`` — 自然語言目標。 -* ``backend`` — ``"anthropic"``（透過 ``export_anthropic_tools()`` - 以 tool-use messages 驅動；每張截圖先縮到模型的影像層級內，工具呼叫的 ``x`` / ``y`` 再換算回螢幕）或 ``"openai"``（``export_openai_tools()`` - + Chat Completions function calling）。 +* ``backend`` — ``"anthropic"`` 或 ``"openai"``。``AC_run_agent`` 預設只提供聚焦、低風險的 + computer-use allow-list，不再把完整的 ``AC_*`` 命令目錄交給模型。需要自訂工具集時， + 可直接用 ``export_anthropic_tools(only=[...])`` 或 ``export_openai_tools(only=[...])`` 建立 backend。 * ``max_steps``（預設 25）、``wall_seconds``（預設 300.0）。 * ``model`` / ``max_tokens`` — backend 專屬覆寫。 diff --git a/docs/source/Zh/doc/ocr_backends/ocr_backends_doc.rst b/docs/source/Zh/doc/ocr_backends/ocr_backends_doc.rst index df1bae4db..e33752353 100644 --- a/docs/source/Zh/doc/ocr_backends/ocr_backends_doc.rst +++ b/docs/source/Zh/doc/ocr_backends/ocr_backends_doc.rst @@ -86,15 +86,28 @@ AutoControl 提供三個可插拔的 OCR 引擎，共用同一組公開 API。 Tesseract 安裝注意 ================== -Windows 必須另外安裝 Tesseract 執行檔（UB-Mannheim build 最常見）。 -若 ``tesseract.exe`` 不在 ``PATH`` 上:: +Windows 必須另外安裝 Tesseract 執行檔（UB-Mannheim build 最常見），安裝程式不會把自己加進 +``PATH``。``find_tesseract_cmd()`` 依序找 ``$TESSERACT_CMD``（指向存在的檔案時）、``PATH``， +最後是安裝程式的預設資料夾（Windows 的 ``Program Files``、``Program Files (x86)`` 與個人的 +``%LOCALAPPDATA%\\Programs``；macOS 的 Homebrew、``/usr/local`` 與 MacPorts；其他平台的 +``/usr/bin`` 與 ``/usr/local/bin``）。它只負責找，要讓 OCR 用上就把結果交給 +``set_tesseract_cmd``:: - from je_auto_control import set_tesseract_cmd - set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") + from je_auto_control import find_tesseract_cmd, set_tesseract_cmd + + command = find_tesseract_cmd() + if command: + set_tesseract_cmd(command) 語言檔（``*.traineddata``）放在 ``Tesseract-OCR\\tessdata\\`` 底下， 要哪個語言就從 `tessdata GitHub repo -`_ 抓對應檔案複製進去。 +`_ 抓對應檔案複製進去。那個資料夾在 Windows 要 +系統管理員權限才寫得進去；想把語言檔放在可寫的地方，就把引擎指過去。這會設定本行程的 +``TESSDATA_PREFIX``，傳 ``None`` 則恢復引擎預設:: + + from je_auto_control import set_tessdata_dir + + set_tessdata_dir("ocr_tessdata") # 回傳解析後的絕對路徑 EasyOCR / PaddleOCR 安裝注意 ============================ @@ -128,12 +141,32 @@ OCR 也透過 executor 對 JSON 動作檔開放:: {"command": "AC_locate_text", "text": "登入", "lang": "chi_tra"} {"command": "AC_click_text", "text": "確定", "backend": "easyocr"} + {"command": "AC_ocr_status"} + {"command": "AC_ocr_languages"} -兩個指令的參數與 Python 端完全一致。 +這些指令的參數與 Python 端完全一致。``AC_ocr_status`` 回傳 ``{"ok": ..., "reason": ...}``， +``AC_ocr_languages`` 回傳 ``{"languages": [...]}``，問不到引擎時是 ``null``。 診斷 ==== +要問 Tesseract 在這裡能不能用、不能用時缺哪一塊:: + + from je_auto_control import ocr_languages, ocr_status + + ok, reason = ocr_status() + # reason："ready"、"missing_package"（沒有 pytesseract）、"missing_engine" + # （找不到設定的執行檔）、"engine_unusable"（執行檔跑不起來），或 + # "no_language_data"（跑起來了，但一個語言都沒列出來） + + languages = ocr_languages() + # ["chi_tra", "eng", ...] 已安裝的語言代碼，已排序 + # [] 引擎回答了：完全沒有語言資料 + # None 問不到引擎 + +``ocr_status`` 檢查的是 OCR 實際會用的執行檔，不改動任何設定。``ocr_languages`` 每次都重新問 +引擎，所以行程執行中補上的語言檔也看得到。``None`` 與 ``[]`` 要分開處理，兩者該有相反的處置。 + 要確認當前環境有哪些後端可用:: from je_auto_control.utils.ocr.backends import get_backend diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index b857826f4..eb1f7a49f 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -20,6 +20,8 @@ AutoControl 所有功能的完整使用指南。 doc/scheduler/scheduler_doc doc/socket_driver/socket_driver_doc doc/mcp_server/mcp_server_doc + doc/ai_agents/ai_agents_doc + doc/ai_agents/ai_agents_doc doc/critical_exit/critical_exit_doc doc/cli/cli_doc doc/create_project/create_project_doc diff --git a/docs/superpowers/plans/2026-10-02-a-contracts.md b/docs/superpowers/plans/2026-10-02-a-contracts.md new file mode 100644 index 000000000..9e224ec1a --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-a-contracts.md @@ -0,0 +1,300 @@ +# A：既有決策與執行契約 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for native execution, or superpowers:subagent-driven-development if the user chooses delegation. Steps use checkbox syntax for tracking. + +**Goal:** 完成已核准的 Progress 修正，建立之後改版不能破壞的執行邊界。 + +**Architecture:** 依核准設計做獨立可測的增量交付，沿用 headless → executor → GUI/MCP 邊界。先確認 Jeffrey_RPA 的 batch/bot 狀態；其使用中的輸入／擷取修改在隔離 checkout 進行。 + +**Tech Stack:** Python 3.10+、pytest、setuptools、cryptography、既有 executor + +**Spec:** [核准設計](../specs/2026-10-02-platform-gui-modernization-design.md) + +## Global Constraints + +- Python ≥3.10；新檔 ≤750 行；公開 API 有 docstring 與型別；例外屬 AutoControlException。 +- 保留 import、AC_* 名稱與舊 CLI 旗標；headless import 不載入 Qt；相依延遲匯入。 +- 一般 GUI 面板操作經 Actions menu；重寫模組不得擴大 mypy 豁免或全域忽略。 +- `architecture.md`、`architecture_explore.md`、三語 README、對應 Sphinx 文件與 `docs/updates/` 隨交付更新；僅完成的 Progress 條目才刪除。 +- 測試使用受控輸入、不操作使用者桌面；有實機證據才標示實機已驗證。 + +## Review Focus + +- 外掛自動載入不能 import je_auto_control 或 Qt。 +- 前一次執行的變數不能流入不同呼叫者。 +- 逾時回覆不能完成新的 USB 請求。 +- 讀取檔案、參照與下載目的地必須服從同一邊界。 +- 新版錯誤回報與簽章不能破壞既有 CLI/import 名稱。 + +## 驗證與提交規則 + +每個 task 先加重現測試、確認失敗、實作、確認成功、更新相關文件並逐檔提交。 +下列 assertion snippets 的測試資料與 fake fixtures 在同一測試模組定義，不能修改 assertion 迎合實作。 +Step 4 通過後，該階段另外執行 ruff、相關既有回歸與三目標型別 gate；全面 coverage 與平台 smoke 於整合階段執行。 +檔案列出的既有目錄是要逐項檢查的入口；新增模組名稱與以下接口固定，搬移時保留兼容路徑。 + +--- + +### Task A1: 精簡 pytest 進入點 + +**Files:** 修改：`je_auto_control/utils/pytest_plugin/plugin.py`、`pyproject.toml`、`test/unit_test/headless/test_coverage_measurement.py`；新增：`je_auto_control_pytest.py`；測試：`test/unit_test/headless/test_pytest_entrypoint_light.py`。 + +**Interfaces:** 產出 `je_auto_control_pytest.py` 的既有 fixtures 與 hooks；舊 plugin re-export 相同函式。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_entrypoint_import_is_light, test_legacy_plugin_exports_same_hooks`，驗證： + +```python +assert 'je_auto_control' not in imported_modules +assert legacy.pytest_configure is standalone.pytest_configure +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_pytest_entrypoint_light.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 搬移現有輕量 plugin，setuptools 的 py-modules 同時打包頂層模組，pytest11 改指該模組；fixture 內保留延遲 import。更新 coverage 測試前提，保留 coverage run。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A1: 精簡 pytest 進入點'`。 + +### Task A2: 舊 CLI 的失敗結束碼 + +**Files:** 修改：`je_auto_control/__main__.py`、`test/unit_test/headless/test_cli_audit_fixes.py`；測試：`test/unit_test/headless/test_legacy_cli_failure_exit.py`。 + +**Interfaces:** 保留 `-e/--execute_file`、`-d/--execute_dir`、`--execute_str`；命令開始 reset failure count，執行後檢查 recorded_failures。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_execute_str_failure_exit_one, test_legacy_file_and_directory_failure_exit_one`，驗證： + +```python +assert failed.returncode == 1 +assert succeeded.returncode == 0 +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_legacy_cli_failure_exit.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 以不驅動桌面的失敗 command 做 subprocess 重現；頂層只 reset 一次，目錄執行累積所有檔案失敗，維持原有 argparse 錯誤。核對 PyBreeze/TestPioneer 使用端。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A2: 舊 CLI 的失敗結束碼'`。 + +### Task A3: 獨立執行變數範圍 + +**Files:** 修改：`je_auto_control/utils/executor/action_executor.py`、`je_auto_control/utils/executor/flow_control.py`、`je_auto_control/utils/script_vars/scope.py`、`je_auto_control/utils/rest_api/rest_handlers.py`、`je_auto_control/utils/mcp_server/tools/_handlers.py`；測試：`test/unit_test/headless/test_execution_scope_isolation.py`。 + +**Interfaces:** 新增 `execution_scope(variables: Mapping[str, object] | None = None) -> ContextManager[VariableScope]`；各伺服器入口、execute_action_with_vars 與巢狀 adapter 使用一致的 context。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_top_level_vars_do_not_leak, test_parallel_nested_helpers_use_branch_scope`，驗證： + +```python +assert second_run_unknown_variable is True +assert sorted(branch_results) == ['left', 'right'] +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_execution_scope_isolation.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 先記錄現有 scope 生命周期；頂層建立 scope，巢狀使用當前 scope，parallel branch fork。用 contextvars 或等價 thread-safe context，finally 還原；circuit/bulkhead/chaos/DAG adapter 不再硬取全域 scope。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A3: 獨立執行變數範圍'`。 + +### Task A4: USB 回覆的請求身分 + +**Files:** 修改：`je_auto_control/utils/usb/passthrough/protocol.py`、`je_auto_control/utils/usb/passthrough/session.py`、`je_auto_control/utils/usb/passthrough/viewer_client.py`；測試：`test/unit_test/headless/test_usb_request_correlation.py`。 + +**Interfaces:** JSON request/reply 增加可選 `request_id: str`，client pending 依 request_id 配對；新 host 原樣 echo。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_late_open_does_not_complete_next_open, test_late_transfer_does_not_complete_next_transfer, test_legacy_timeout_requires_reconnect`，驗證： + +```python +assert new_request.completed is False +assert next_result == expected_new_reply +assert legacy_claim.reusable is False +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_usb_request_correlation.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 新 peer 的 tombstone pending ID 僅丟棄已過期回覆。舊 peer 在無法判別逾時回覆時將 claim/session 標成不可重用並重連，不使用『丟掉下一則』策略。fragment/error/credit 的配對一併驗證。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A4: USB 回覆的請求身分'`。 + +### Task A5: 公私鑰簽章與執行權限分離 + +**Files:** 修改：`je_auto_control/utils/action_signing/signer.py`、`je_auto_control/utils/action_signing/_key_file.py`、`je_auto_control/utils/executor/action_executor.py`；測試：`test/unit_test/headless/test_signing_execution_roles.py`。 + +**Interfaces:** 新增 version-2 Ed25519 signature envelope 與 `create_signing_keypair(private_path: Path, public_path: Path) -> None`；verify 只需 public key。角色授權另由 A10 交付。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_verifier_has_no_private_key, test_execution_endpoint_cannot_sign, test_legacy_signature_requires_explicit_migration`，驗證： + +```python +assert verifier.private_key_path is None +assert execution_endpoint_sign_denied is True +assert implicit_legacy_signature_accepted is False +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_signing_execution_roles.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 明確配置 signer/private key 與 verifier/public key；執行端不能靠預設個人 key 簽章。舊 HMAC envelope 提供明確遷移模式，使用新簽章時 verifier 不讀 private key。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A5: 公私鑰簽章與執行權限分離'`。 + +### Task A6: MCP 路徑與 viewer 收檔 + +**Files:** 修改：`je_auto_control/utils/mcp_server/tools/_factories.py`、`je_auto_control/utils/mcp_server/server.py`、`je_auto_control/utils/secret_ref/secret_ref.py`、`je_auto_control/utils/remote_desktop/file_transfer.py`、`je_auto_control/utils/remote_desktop/viewer.py`、`je_auto_control/gui/remote_desktop/viewer_panel.py`；新增：`je_auto_control/utils/path_guard/policy.py`；測試：`test/unit_test/headless/test_file_boundary_policy.py`。 + +**Interfaces:** `FileReceiver(base_dir: Path | None = None)` 保留 host 用法；viewer 預設使用下載 root。`PathPolicy.validate(path: str, *, operation: str) -> Path` 採 realpath 與語意 metadata。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_semantic_path_fields_only, test_symlink_escape_is_rejected, test_viewer_file_stays_in_download_root, test_env_ref_allowlist`，驗證： + +```python +assert json_query_path_is_untouched is True +assert escaped_file_rejected is True +assert received_path.is_relative_to(download_root) +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_file_boundary_policy.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 新增 focused policy 模組；按 schema 語意標記真正檔案欄位。配置 roots 與 roots/list 使用同一策略，未配置不預設唯讀。file/env ref 使用明確 policy，reject symlink、absolute viewer dest、drive/UNC escape；更新舊絕對路徑範例。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A6: MCP 路徑與 viewer 收檔'`。 + +### Task A7: 發布名稱與專案網址 + +**Files:** 修改：`je_auto_control/utils/mcp_registry/registry.py`、`pyproject.toml`、`dev.toml`、`README.md`、`README/README_zh-TW.md`、`README/README_zh-CN.md`；測試：`test/unit_test/headless/test_modernization_metadata.py`。 + +**Interfaces:** 沿用套件名與所有公開 import；新 URL 使用 Integration-Automation/AutoControlGUI；安全相依由 A11 處理。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_registry_uses_approved_name, test_project_urls_use_current_repository`，驗證： + +```python +assert server_name == 'io.github.integration-automation/autocontrol' +assert repo_slug == 'Integration-Automation/AutoControlGUI' +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_modernization_metadata.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 改所有現行 URL 與 registry payload；驗證 wheel/sdist metadata 與三語 README 對帳，保留發布套件名稱。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A7: 發布名稱與專案網址'`。 + +### Task A8: 多螢幕與 DPI 座標契約 + +**Files:** 修改：`je_auto_control/windows/screen/win32_screen.py`、`je_auto_control/utils/monitor_layout/logical_frame.py`、`je_auto_control/utils/cv2_utils/screenshot.py`、`je_auto_control/utils/window_capture/window_capture.py`、`je_auto_control/utils/set_of_marks/set_of_marks.py`、`je_auto_control/gui/_screen_geometry.py`；測試：`test/unit_test/headless/test_platform_coordinate_contract.py`。 + +**Interfaces:** 沿用 wrapper 簽章；所有定位回傳 backend 接受的 logical/global point。Windows 啟用 per-monitor v2，macOS 每螢幕縮放並拼接。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_negative_secondary_monitor_capture, test_retina_maps_to_points, test_marks_include_virtual_origin`，驗證： + +```python +assert logical_click == (100, 50) +assert captured_secondary_is_black is False +assert marked_points == expected_global_points +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_platform_coordinate_contract.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 修 per-monitor v2、Windows 區域／視窗擷取與 set-of-marks 原點，macOS 逐螢幕 point 座標擷取、縮放、拼接。使用相同 logical/global 契約；下游兩支 GUI 契約測試必跑。實機 Retina 與混合 DPI 結果附測試條件。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A8: 多螢幕與 DPI 座標契約'`。 + +### Task A9: Agent 歷史與真實 API 驗證 + +**Files:** 修改：`je_auto_control/utils/agent/backends/base.py`、`je_auto_control/utils/agent/backends/anthropic.py`、`je_auto_control/utils/agent/backends/anthropic_computer_use.py`；測試：`test/unit_test/headless/test_agent_append_only_history.py`。 + +**Interfaces:** 新增 `compact_history(messages: Sequence[object], summary: str, latest_screenshot: object) -> list[object]` 的 provider adapter；維持 AC_run_agent 的既有工具集。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_sent_turns_are_never_mutated, test_compaction_opens_new_history, test_default_agent_tools_are_unchanged`，驗證： + +```python +assert sent_turns == original_sent_turns +assert compacted_history[0].contains_goal_and_actions +assert default_tools == original_tools +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_agent_append_only_history.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 超過截圖上限時建立目標／已執行動作摘要與最新截圖的新對話，不改已送訊息。按照當前官方 API 驗證兩條 Anthropic 路徑。paid smoke 僅在配置可用時執行；預設 toolset 切換仍需真實 API 結果。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A9: Agent 歷史與真實 API 驗證'`。 + +### Task A10: REST/MCP 角色授權 + +**Files:** 修改：`je_auto_control/utils/rbac/users.py`、`je_auto_control/utils/rest_api/rest_auth.py`、`je_auto_control/utils/mcp_server/http_transport.py`、`je_auto_control/utils/mcp_server/audit.py`、`je_auto_control/gui/admin_console_tab.py`；測試：`test/unit_test/headless/test_rbac_server_wiring.py`。 + +**Interfaces:** `AuthorizationContext(user_id: str, role: str)`；REST/MCP 在 bearer 驗證後使用既有 `UserStore.authenticate`、`can(role, capability)`。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_viewer_cannot_execute, test_operator_cannot_sign, test_unconfigured_rbac_keeps_shared_token, test_user_id_in_audit`，驗證： + +```python +assert viewer_execute_denied is True +assert legacy_token_accepted is True +assert audit_entry['user_id'] == authenticated_user +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_rbac_server_wiring.py`，確認新測試因原有缺陷 FAIL。 +- [ ] **Step 3:** RBAC opt-in，未配置沿用既有 shared token。路由與 tool 依 capability 授權，搜尋與執行一致，audit 攜帶 user_id；admin GUI 連到同一 UserStore，不能用單一 token 驗證成功就授予全部角色。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0，再執行相關既有回歸與型別 gate。 +- [ ] **Step 5:** 更新對應交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A10: REST/MCP 角色授權'`。 + +### Task A11: 安全相依下限與安裝矩陣 + +**Files:** 修改：`pyproject.toml`、`uv.lock`、`test/unit_test/headless/test_arm64_dependency_markers.py`、`docs/CAPABILITY_MATRIX.md`；測試：`test/unit_test/headless/test_crypto_fifty_install_contract.py`。 + +**Interfaces:** cryptography floor `>=50.0.0`，保留平台 optional markers；公開的 crypto 操作失敗有型別與安裝說明。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_crypto_floor_is_fifty, test_arm64_import_has_no_crypto_requirement, test_missing_crypto_feature_is_typed`，驗證： + +```python +assert crypto_minimum >= (50, 0, 0) +assert facade_import_without_crypto_succeeds is True +assert error.has_install_instruction +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_crypto_fifty_install_contract.py`，確認新測試因原有缺陷 FAIL。 +- [ ] **Step 3:** 重產 lock、查核當前安全與官方 wheel metadata，Windows arm64 與 Intel Mac binary probe 分開留結果。缺依賴時只停受影響能力；簽章使用安全算法，不以舊 wheel 降低下限。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0，再執行相關既有回歸與型別 gate。 +- [ ] **Step 5:** 更新對應交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A11: 安全相依下限與安裝矩陣'`。 + +### Task A12: 視窗管理與擷取位置 + +**Files:** 修改：`je_auto_control/wrapper/window_backends/macos_backend.py`、`je_auto_control/windows/window/windows_window_manage.py`、`je_auto_control/utils/window_capture/window_capture.py`、`je_auto_control/wrapper/auto_control_window.py`、`je_auto_control/utils/window_zorder/window_zorder.py`；測試：`test/unit_test/headless/test_window_lifecycle_contract.py`。 + +**Interfaces:** 沿用 `restore(window_id)`、`capture_window(...)`、`wait_for_window(...)`；_info_for 查詢包括最小化 window；focus 以真實 foreground 狀態驗證。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_restore_minimized_macos_window, test_capture_does_not_move_window, test_foreground_failure_propagates, test_cloaked_window_filtered, test_poll_is_bounded`，驗證： + +```python +assert restored_window.visible is True +assert placement_after == placement_before +assert poll_sleep <= remaining_timeout +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_window_lifecycle_contract.py`，確認新測試因原有缺陷 FAIL。 +- [ ] **Step 3:** CGWindow including-window 查詢；Windows focus/zorder 回傳真實失敗、過濾 DWM cloak、Get/SetWindowPlacement 還原原位；布局使用 work area，poll 用 clamp。post_key 對一般文字與控制鍵各自只送正確事件，避免重複文字。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0，再執行相關既有回歸與型別 gate。 +- [ ] **Step 5:** 更新對應交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A12: 視窗管理與擷取位置'`。 + +### Task A13: 鍵鼠輸入、秘密與 Unicode + +**Files:** 修改：`je_auto_control/wrapper/auto_control_keyboard.py`、`je_auto_control/wrapper/auto_control_mouse.py`、`je_auto_control/utils/text_unicode/text_unicode.py`、`je_auto_control/utils/keyboard_layout/keyboard_layout.py`、`je_auto_control/utils/clipboard_formats/clipboard_formats.py`、`je_auto_control/utils/executor/action_executor.py`；測試：`test/unit_test/headless/test_input_wrapper_contract.py`。 + +**Interfaces:** `write(..., secret: bool = False)`、新增 `write_secret(...)` 與 `AC_write_secret` 相容別名；既有參數名不改，Shift 以 finally 釋放；座標使用 `int(round(value))`。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_case_and_shift_preserved, test_crlf_sends_single_enter, test_nan_does_not_move_mouse, test_scroll_direction_and_rounding, test_write_secret_leaves_no_text, test_dead_key_and_oem_layout`，驗證： + +```python +assert enter_events_for_crlf == 1 +assert mouse_events_for_nan == [] +assert secret not in logs_and_records +assert dead_key_shift_value is None +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_input_wrapper_contract.py`，確認新測試因原有缺陷 FAIL。 +- [ ] **Step 3:** 依 Progress 的逐項重現修 case/shift/CRLF/scroll/NaN/OEM、Unicode控制鍵與clipboard None名。私有 WinDLL 避免 prototype 汙染。secret 在 logger/record/callback/return 邊界都不含原文，WebRunner WR_ac_basic_auth 契約測試一起更新。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0，再執行相關既有回歸與型別 gate。 +- [ ] **Step 5:** 更新對應交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A13: 鍵鼠輸入、秘密與 Unicode'`。 + +### Task A14: 影像讀取、OCR 與區域邊界 + +**Files:** 修改：`je_auto_control/utils/cv2_utils/template_detection.py`、`je_auto_control/utils/cv2_utils/image_file.py`、`je_auto_control/utils/ocr/text_span.py`、`je_auto_control/wrapper/auto_control_image.py`、`je_auto_control/utils/monitor_layout/logical_frame.py`；測試：`test/unit_test/headless/test_image_ocr_coordinate_contract.py`。 + +**Interfaces:** 沿用 image/OCR 公開入口；模板讀取經 `read_image`；center 使用整數 `// 2`；region與虛擬桌面先做交集驗證。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_unicode_template_path, test_grayscale_template, test_partial_ocr_left_span_preserved, test_negative_center_floor, test_empty_region_rejected`，驗證： + +```python +assert found_from_unicode_path is True +assert partial_phrase_found is True +assert negative_center == (-2, -2) +assert invalid_region_rejected_before_capture is True +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_image_ocr_coordinate_contract.py`，確認新測試因原有缺陷 FAIL。 +- [ ] **Step 3:** 支援Unicode路徑與2-D模板，typed miss錯誤。OCR丟左框只在剩餘字數仍足以包含目標時；空／負區域先拒絕。測負座標及跨螢幕region與 A8 契約一致，下游GUI契約驗證後交付。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0，再執行相關既有回歸與型別 gate。 +- [ ] **Step 5:** 更新對應交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'A14: 影像讀取、OCR 與區域邊界'`。 diff --git a/docs/superpowers/plans/2026-10-02-b-journal-healing-codegen.md b/docs/superpowers/plans/2026-10-02-b-journal-healing-codegen.md new file mode 100644 index 000000000..50dd820de --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-b-journal-healing-codegen.md @@ -0,0 +1,93 @@ +# B：動作日誌、自愈量測與候選腳本 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for native execution, or superpowers:subagent-driven-development if the user chooses delegation. Steps use checkbox syntax for tracking. + +**Goal:** 讓動作來源可追溯、自愈版本可比較，並從日誌產生可驗證的候選腳本。 + +**Architecture:** 依核准設計做獨立可測的增量交付，沿用 headless → executor → GUI/MCP 邊界。依賴 A 的執行 scope、秘密處理與座標契約。 + +**Tech Stack:** stdlib JSONL、dataclasses、既有 codegen/locator、pytest + +**Spec:** [核准設計](../specs/2026-10-02-platform-gui-modernization-design.md) + +## Global Constraints + +- Python ≥3.10；新檔 ≤750 行；公開 API 有 docstring 與型別；例外屬 AutoControlException。 +- 保留 import、AC_* 名稱與舊 CLI 旗標；headless import 不載入 Qt；相依延遲匯入。 +- 一般 GUI 面板操作經 Actions menu；重寫模組不得擴大 mypy 豁免或全域忽略。 +- `architecture.md`、`architecture_explore.md`、三語 README、對應 Sphinx 文件與 `docs/updates/` 隨交付更新；僅完成的 Progress 條目才刪除。 +- 測試使用受控輸入、不操作使用者桌面；有實機證據才標示實機已驗證。 + +## Review Focus + +- 秘密不得先寫入日誌才遮罩。 +- 取消或未結束的動作不能看起來已成功。 +- 未知標籤不能被計入自愈正確率。 +- parallel/loop 日誌不能被誤還原成未執行的流程。 +- 產碼不得執行輸入內容。 + +## 驗證與提交規則 + +每個 task 先加重現測試、確認失敗、實作、確認成功、更新相關文件並逐檔提交。 +下列 assertion snippets 的測試資料與 fake fixtures 在同一測試模組定義，不能修改 assertion 迎合實作。 +Step 4 通過後，該階段另外執行 ruff、相關既有回歸與三目標型別 gate；全面 coverage 與平台 smoke 於整合階段執行。 +檔案列出的既有目錄是要逐項檢查的入口；新增模組名稱與以下接口固定，搬移時保留兼容路徑。 + +--- + +### Task B1: 結構化動作日誌 + +**Files:** 修改：`je_auto_control/utils/executor/action_executor.py`、`je_auto_control/utils/executor/action_redaction.py`、`je_auto_control/utils/run_history/history_store.py`；新增：`je_auto_control/utils/action_journal/__init__.py`、`je_auto_control/utils/action_journal/events.py`、`je_auto_control/utils/action_journal/store.py`；測試：`test/unit_test/headless/test_action_journal.py`。 + +**Interfaces:** `ActionEvent` frozen dataclass，schema_version=1；`ActionJournal.append(event: ActionEvent) -> None`；`read_events(path: Path, *, run_id: str | None = None) -> list[ActionEvent]`。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_redaction_precedes_append, test_parallel_parent_and_order, test_incomplete_step_stays_incomplete`，驗證： + +```python +assert password not in journal_text +assert all(event.run_id == run_id for event in events) +assert unfinished.status == 'incomplete' +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_action_journal.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 在 executor 動作邊界記錄 typed input/outcome 與 parent ID，先 redaction 再持久化，保留不可重播欄位的原因。沿用 atomic/json-store 邊界；完整 typed public/export/AC/GUI 入口同步交付。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'B1: 結構化動作日誌'`。 + +### Task B2: 自愈比較與候選修正 + +**Files:** 修改：`je_auto_control/utils/self_healing/heal_log.py`、`je_auto_control/utils/self_healing/locator.py`、`je_auto_control/gui/self_healing_tab.py`；新增：`je_auto_control/utils/self_healing/evaluation.py`、`benchmarks/self_healing`；測試：`test/unit_test/headless/test_self_healing_evaluation.py`。 + +**Interfaces:** `EvaluationSample` 定義 frame/expected_box/origin/scale；`evaluate_locators(samples: Sequence[EvaluationSample], versions: Mapping[str, LocatorStrategy]) -> HealingComparison`；report 帶分子分母與 p50/p95。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_unlabelled_is_unknown, test_false_positive_is_not_recovery, test_same_frame_versions_are_comparable, test_region_passed_to_both_strategies`，驗證： + +```python +assert report.unknown == 1 +assert report.correct == 1 and report.false_positive == 1 +assert image_region == vlm_region +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_self_healing_evaluation.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** HealEvent 增 optional version/context 欄位，讀舊格式；單一 capture 供策略比較，命中與操作後驗證分離。加入固定失敗／縮放／負座標資料集，candidate template revision 可 preview/accept/revert。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'B2: 自愈比較與候選修正'`。 + +### Task B3: 從日誌產生候選腳本 + +**Files:** 修改：`je_auto_control/utils/codegen/codegen.py`、`je_auto_control/cli.py`、`je_auto_control/gui/recording_editor_tab.py`、`je_auto_control/gui/script_builder/builder_tab.py`、`je_auto_control/utils/mcp_server/tools/_handlers_qa.py`；新增：`je_auto_control/utils/codegen/journal_import.py`；測試：`test/unit_test/headless/test_codegen_from_journal.py`。 + +**Interfaces:** `generate_candidate_from_log(path: Path, *, run_id: str, target: str = 'pytest', style: str = 'actions') -> CandidateScript`；包含 code、manifest、warnings。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_run_filter_and_provenance, test_observed_branch_is_labelled, test_secret_reference_survives_codegen, test_generation_has_no_device_effect`，驗證： + +```python +assert candidate.manifest['run_id'] == selected_run +assert candidate.observed_path_only is True +assert device_calls == [] +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_codegen_from_journal.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 驗證 JSONL schema，重建 step/parent 與 retries，控制流程不足時標示 observed path。呼叫既有 generate_code，產物 AST/command/dry-run 驗證，GUI diff 與匯入 builder、CLI --from-log 同步交付；不解析任意 repr 執行。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'B3: 從日誌產生候選腳本'`。 diff --git a/docs/superpowers/plans/2026-10-02-c-sync-sessions.md b/docs/superpowers/plans/2026-10-02-c-sync-sessions.md new file mode 100644 index 000000000..71ee641c9 --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-c-sync-sessions.md @@ -0,0 +1,112 @@ +# C：持久化同步與遠端 session Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for native execution, or superpowers:subagent-driven-development if the user chooses delegation. Steps use checkbox syntax for tracking. + +**Goal:** 讓多機設定在重啟、離線及並行更新後仍一致，並隔離遠端連線所有權。 + +**Architecture:** 依核准設計做獨立可測的增量交付，沿用 headless → executor → GUI/MCP 邊界。依賴 A context；使用 B 的 journal 記同步操作。 + +**Tech Stack:** SQLite、stdlib HTTP、FastAPI 既有 signaling、typed envelopes + +**Spec:** [核准設計](../specs/2026-10-02-platform-gui-modernization-design.md) + +## Global Constraints + +- Python ≥3.10；新檔 ≤750 行；公開 API 有 docstring 與型別；例外屬 AutoControlException。 +- 保留 import、AC_* 名稱與舊 CLI 旗標；headless import 不載入 Qt；相依延遲匯入。 +- 一般 GUI 面板操作經 Actions menu；重寫模組不得擴大 mypy 豁免或全域忽略。 +- `architecture.md`、`architecture_explore.md`、三語 README、對應 Sphinx 文件與 `docs/updates/` 隨交付更新；僅完成的 Progress 條目才刪除。 +- 測試使用受控輸入、不操作使用者桌面；有實機證據才標示實機已驗證。 + +## Review Focus + +- server 重啟不能丟設定。 +- 兩台機器同時 push 不能靜默覆寫。 +- 離線裝置不能讓已刪除資料復活。 +- 舊面板的 callback 不能切斷新連線。 +- 腳本同步不能自動執行或啟用觸發器。 + +## 驗證與提交規則 + +每個 task 先加重現測試、確認失敗、實作、確認成功、更新相關文件並逐檔提交。 +下列 assertion snippets 的測試資料與 fake fixtures 在同一測試模組定義，不能修改 assertion 迎合實作。 +Step 4 通過後，該階段另外執行 ruff、相關既有回歸與三目標型別 gate；全面 coverage 與平台 smoke 於整合階段執行。 +檔案列出的既有目錄是要逐項檢查的入口；新增模組名稱與以下接口固定，搬移時保留兼容路徑。 + +--- + +### Task C1: 持久化與 CAS server + +**Files:** 修改：`je_auto_control/utils/remote_desktop/signaling_server.py`；新增：`je_auto_control/utils/config_sync/store.py`；測試：`test/unit_test/headless/test_config_sync_persistence.py`。 + +**Interfaces:** `ConfigStore.commit(user_id: str, bucket: ConfigBucket, *, base_revision: int, operation_id: str) -> int`；revision 檢查／寫入在一個 SQLite transaction，重複 operation 回原 revision。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_reopen_preserves_bucket, test_stale_revision_conflicts, test_operation_retry_is_idempotent`，驗證： + +```python +assert reopened.get(user).revision == committed_revision +assert stale_response.status_code == 409 +assert repeated.revision == original.revision +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_config_sync_persistence.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** version-2 PUT 以 base_revision/operation_id 加 envelope，GET 回 committed revision。設定 store 路徑 lazy resolve，account isolation、body cap 與共享 secret 規則沿用；舊 blind write 需明確相容設定。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'C1: 持久化與 CAS server'`。 + +### Task C2: 因果合併、刪除與離線 outbox + +**Files:** 修改：`je_auto_control/utils/config_sync/client.py`、`je_auto_control/utils/config_sync/__init__.py`；新增：`je_auto_control/utils/config_sync/versions.py`、`je_auto_control/utils/config_sync/outbox.py`；測試：`test/unit_test/headless/test_config_sync_convergence.py`。 + +**Interfaces:** `SyncEntry` 帶 version vector/operation/origin；`merge_entries(left: SyncEntry, right: SyncEntry) -> MergeDecision`；`SyncOutbox.enqueue(operation: SyncOperation) -> None`。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_parallel_edits_preserve_conflict, test_restart_retries_outbox, test_retired_peer_requires_full_sync, test_clock_skew_does_not_choose_winner`，驗證： + +```python +assert conflict.local == left and conflict.remote == right +assert tombstone.is_deleted is True +assert wall_clock_does_not_change_merge is True +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_config_sync_convergence.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 同 key 並行變更保留 conflict；outbox SQLite 以 operation ID 重送，有限退避與取消。tombstone 依 peer acknowledged revision 回收；退休 peer 明確 full resync。修掉 __init__ 的過期 endpoint 說明。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'C2: 因果合併、刪除與離線 outbox'`。 + +### Task C3: 設定／資產 adapter 與同步 UI + +**Files:** 修改：`je_auto_control/utils/remote_desktop/file_sync.py`、`je_auto_control/utils/remote_desktop/clipboard_sync.py`；新增：`je_auto_control/utils/config_sync/adapters.py`、`je_auto_control/utils/config_sync/assets.py`、`je_auto_control/gui/config_sync_tab.py`；測試：`test/unit_test/headless/test_sync_adapters.py`。 + +**Interfaces:** `SyncAdapter.snapshot() -> Mapping[str, SyncEntry]`、`apply(entries: Mapping[str, SyncEntry]) -> ApplyReport`；`sync_assets(manifest: AssetManifest, transport: AssetTransport) -> AssetSyncResult`。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_script_asset_hash_round_trip, test_sync_never_enables_trigger, test_secret_is_local, test_clipboard_does_not_echo`，驗證： + +```python +assert received_hash == source_hash +assert enabled_triggers == [] +assert sync_payload_contains_secret is False +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_sync_adapters.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 先寫 script/locator/hotkey/trigger/address-book adapters，秘密及機器路徑留下參照。hash 驗證與 atomic replace；同步面板顯示 revision/pending/conflict/offline，取消 worker 釋放；folder/clipboard loop 與斷線場景也回歸。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'C3: 設定／資產 adapter 與同步 UI'`。 + +### Task C4: 遠端 session 擁有權 + +**Files:** 修改：`je_auto_control/utils/remote_desktop/registry.py`、`je_auto_control/gui/remote_desktop/connection_screen.py`、`je_auto_control/gui/remote_desktop/viewer_panel.py`、`je_auto_control/gui/remote_desktop/webrtc_panel.py`、`je_auto_control/utils/executor/action_executor.py`；新增：`je_auto_control/utils/remote_desktop/sessions.py`；測試：`test/unit_test/headless/test_remote_session_ownership.py`。 + +**Interfaces:** `RemoteSession` 擁有 id/owner/transport/state；`disconnect_session(session_id: str, *, owner: str | None = None) -> SessionStatus`；既有 AC_remote_* 加 optional session_id。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_panel_disconnect_only_own_session, test_stale_callback_is_ignored, test_default_script_session_is_compatible`，驗證： + +```python +assert other_session.connected is True +assert stale_frame_delivered is False +assert default_script_target == script_session.id +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_remote_session_ownership.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** registry 改 map 與 script default alias；每個 GUI 面板持有自己的 session，生命周期事件通知 owner，舊 callback 查 session ID。拆 webrtc_panel 過長責任，與工具 schema、builder 一起更新。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'C4: 遠端 session 擁有權'`。 diff --git a/docs/superpowers/plans/2026-10-02-d-wayland-dependencies.md b/docs/superpowers/plans/2026-10-02-d-wayland-dependencies.md new file mode 100644 index 000000000..213205d8b --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-d-wayland-dependencies.md @@ -0,0 +1,93 @@ +# D：Wayland 與原生依賴可靠性 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for native execution, or superpowers:subagent-driven-development if the user chooses delegation. Steps use checkbox syntax for tracking. + +**Goal:** 修正 Wayland 生命周期，提供可測替代後端及正確的能力診斷。 + +**Architecture:** 依核准設計做獨立可測的增量交付，沿用 headless → executor → GUI/MCP 邊界。依賴 A 座標、B 日誌、C session；不修改使用者桌面環境或自動取得裝置權限。 + +**Tech Stack:** D-Bus portal、libei/liboeffis、helper process、stdlib IPC、既有 Docker jobs + +**Spec:** [核准設計](../specs/2026-10-02-platform-gui-modernization-design.md) + +## Global Constraints + +- Python ≥3.10；新檔 ≤750 行；公開 API 有 docstring 與型別；例外屬 AutoControlException。 +- 保留 import、AC_* 名稱與舊 CLI 旗標；headless import 不載入 Qt；相依延遲匯入。 +- 一般 GUI 面板操作經 Actions menu；重寫模組不得擴大 mypy 豁免或全域忽略。 +- `architecture.md`、`architecture_explore.md`、三語 README、對應 Sphinx 文件與 `docs/updates/` 隨交付更新；僅完成的 Progress 條目才刪除。 +- 測試使用受控輸入、不操作使用者桌面；有實機證據才標示實機已驗證。 + +## Review Focus + +- 拒絕 portal 授權必須立即停止請求。 +- 半開交握不能 crash GUI 或無限漏 fd。 +- 裝置暫停／移除後不能繼續送輸入。 +- 錄製不能把本身注入的輸入再錄一次。 +- native CI segfault 必須產生可定位的 crash artifact。 + +## 驗證與提交規則 + +每個 task 先加重現測試、確認失敗、實作、確認成功、更新相關文件並逐檔提交。 +下列 assertion snippets 的測試資料與 fake fixtures 在同一測試模組定義，不能修改 assertion 迎合實作。 +Step 4 通過後，該階段另外執行 ruff、相關既有回歸與三目標型別 gate；全面 coverage 與平台 smoke 於整合階段執行。 +檔案列出的既有目錄是要逐項檢查的入口；新增模組名稱與以下接口固定，搬移時保留兼容路徑。 + +--- + +### Task D1: 能力與授權狀態機 + +**Files:** 修改：`je_auto_control/linux_wayland/portal.py`、`je_auto_control/linux_wayland/_select_input.py`、`je_auto_control/linux_wayland/capture.py`、`je_auto_control/gui/diagnostics_tab.py`；新增：`je_auto_control/wrapper/capabilities.py`；測試：`test/unit_test/headless/test_wayland_capability_states.py`。 + +**Interfaces:** `CapabilityStatus` 與 `probe_capabilities(context: BackendContext) -> CapabilitySnapshot`；探測沒有控制或授權副作用。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_cancel_does_not_fallback_silently, test_revoked_session_cannot_send, test_xwayland_scope_is_explicit`，驗證： + +```python +assert canceled.state == 'needs_permission' +assert writes_after_revoke == 0 +assert xwayland.desktop_wide is False +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_wayland_capability_states.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 輸入／擷取獨立選擇與診斷，portal request/session close、restore token、compositor 重啟走明確狀態；GUI 顯示可操作修正與停止控制，headless/AC/MCP 使用同一 capability。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'D1: 能力與授權狀態機'`。 + +### Task D2: libei crash 隔離與 backend 相依 + +**Files:** 修改：`je_auto_control/linux_wayland/libei.py`、`docker/libei_verify.py`、`docker/eis_verify.py`、`test/unit_test/headless/test_arm64_dependency_markers.py`；新增：`je_auto_control/linux_wayland/ei_worker.py`、`je_auto_control/linux_wayland/ei_transport.py`；測試：`test/unit_test/headless/test_wayland_worker_cleanup.py`。 + +**Interfaces:** `EiWorkerClient.send(batch: Sequence[InputEvent], *, timeout_s: float) -> InputAck`、`close() -> None`；有界 batch、request IDs 與 cancellation。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_half_open_worker_exit_reclaims_resources, test_worker_death_releases_pressed_keys, test_missing_dependency_is_typed`，驗證： + +```python +assert active_workers == 0 +assert fd_count_after == fd_count_before +assert error.capability == unavailable_capability +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_wayland_worker_cleanup.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 先跑現有 real-EI verify 精確分類 unsafe teardown；合法清理仍 crash 才啟用 helper。IPC 上限、超時與退場，測量延遲。arm64 影像能力用 backend seam 提供可行替代並逐項矩陣驗證；crypto 維持安全版本與可選診斷。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'D2: libei crash 隔離與 backend 相依'`。 + +### Task D3: 錄製、全域停止與原生驗證 + +**Files:** 修改：`je_auto_control/linux_wayland/listener.py`、`je_auto_control/linux_wayland/record.py`、`docker`、`test/unit_test/headless/test_usb_acl_prompt.py`、`.github/workflows`；新增：`je_auto_control/linux_wayland/input_events.py`、`je_auto_control/linux_wayland/global_shortcuts.py`；測試：`test/unit_test/headless/test_wayland_recording_paths.py`。 + +**Interfaces:** `PhysicalRecorder.start(devices: Sequence[InputDevice]) -> None`、`stop() -> list[InputEvent]`；`StopShortcutSession.close() -> None`。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_action_recording_needs_no_global_hook, test_physical_reader_excludes_virtual_device, test_permission_denial_is_actionable, test_shortcut_session_closes`，驗證： + +```python +assert injected_events == [] +assert executor_journal_steps == expected_steps +assert permission_error.has_recovery_instruction +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_wayland_recording_paths.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 明確 opt-in 的物理 event reader，排除 virtual/uinput source；不將 portal InputCapture 誤用為普通全域 hook。整合現有 WL compositor CI，新增 GNOME/KDE 授權允許／拒絕人工步驟。USB ACL 3.10 用 faulthandler/native backtrace 重現後修 Qt/native lifecycle，保留 crash log artifact。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'D3: 錄製、全域停止與原生驗證'`。 diff --git a/docs/superpowers/plans/2026-10-02-e-mobile.md b/docs/superpowers/plans/2026-10-02-e-mobile.md new file mode 100644 index 000000000..a20395f1c --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-e-mobile.md @@ -0,0 +1,112 @@ +# E：Android 與 iOS 功能完整化 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for native execution, or superpowers:subagent-driven-development if the user chooses delegation. Steps use checkbox syntax for tracking. + +**Goal:** 補齊行動自動化功能並提供隔離的多裝置工作流程。 + +**Architecture:** 依核准設計做獨立可測的增量交付，沿用 headless → executor → GUI/MCP 邊界。依賴 A context/座標、B frame/journal、自愈、D capability；F 可先接舊 shell 的 mobile panel。 + +**Tech Stack:** ADB、uiautomator2、WDA、可選 Appium adapter、typed device contexts + +**Spec:** [核准設計](../specs/2026-10-02-platform-gui-modernization-design.md) + +## Global Constraints + +- Python ≥3.10；新檔 ≤750 行；公開 API 有 docstring 與型別；例外屬 AutoControlException。 +- 保留 import、AC_* 名稱與舊 CLI 旗標；headless import 不載入 Qt；相依延遲匯入。 +- 一般 GUI 面板操作經 Actions menu；重寫模組不得擴大 mypy 豁免或全域忽略。 +- `architecture.md`、`architecture_explore.md`、三語 README、對應 Sphinx 文件與 `docs/updates/` 隨交付更新；僅完成的 Progress 條目才刪除。 +- 測試使用受控輸入、不操作使用者桌面；有實機證據才標示實機已驗證。 + +## Review Focus + +- 同時連兩台裝置不能串台。 +- 裝置輸入不能靜默丟 Unicode。 +- 旋轉／Retina point/pixel 不能點錯位置。 +- 連線中斷不能留下 App/session 在未知狀態。 +- 不支援的 device 功能要給原因與可用替代。 + +## 驗證與提交規則 + +每個 task 先加重現測試、確認失敗、實作、確認成功、更新相關文件並逐檔提交。 +下列 assertion snippets 的測試資料與 fake fixtures 在同一測試模組定義，不能修改 assertion 迎合實作。 +Step 4 通過後，該階段另外執行 ruff、相關既有回歸與三目標型別 gate；全面 coverage 與平台 smoke 於整合階段執行。 +檔案列出的既有目錄是要逐項檢查的入口；新增模組名稱與以下接口固定，搬移時保留兼容路徑。 + +--- + +### Task E1: 裝置 context 與 matrix + +**Files:** 修改：`je_auto_control/android/client.py`、`je_auto_control/ios/client.py`、`je_auto_control/utils/device_matrix/matrix.py`；新增：`je_auto_control/wrapper/device_context.py`；測試：`test/unit_test/headless/test_mobile_context_isolation.py`。 + +**Interfaces:** `DeviceContext` frozen identity/config，`open_device(context: DeviceContext) -> DeviceSession`；session 有 capabilities、close、cancel。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_parallel_devices_do_not_change_default, test_session_cancel_only_target, test_device_capability_probe_has_no_input`，驗證： + +```python +assert left_calls.device_id == 'left' +assert right_session.connected is True +assert probe_input_calls == [] +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_mobile_context_isolation.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 保留 default helper 相容入口，但 matrix 使用 explicit context；WDA/ADB session timeout、ownership、error translation 統一。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'E1: 裝置 context 與 matrix'`。 + +### Task E2: 文字、手勢、畫面與定位 + +**Files:** 修改：`je_auto_control/android/adb_client.py`、`je_auto_control/android/find.py`、`je_auto_control/ios/input.py`、`je_auto_control/ios/screen.py`、`je_auto_control/ios/find.py`、`je_auto_control/utils/self_healing/locator.py`；新增：`je_auto_control/android/input.py`；測試：`test/unit_test/headless/test_mobile_input_capture.py`。 + +**Interfaces:** `DeviceSession.capture() -> DeviceFrame`、`perform(gesture: Gesture) -> None`、`type_text(text: str) -> None`；frame 帶 pixel_to_point 與 orientation。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_unicode_round_trip, test_long_press_drag_pinch, test_rotated_frame_maps_to_device_points, test_mobile_pipeline_uses_device_frame`，驗證： + +```python +assert received_text == '測試 café 🙂' +assert click_point == expected_device_point +assert desktop_capture_calls == 0 +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_mobile_input_capture.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** Android Unicode 經可用 IME/SDK 路徑，不把不支援文字送 adb input text 假成功。WDA point/pixel 在 adapter 換算，OCR/template/VLM/self-heal 共用 device frame。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'E2: 文字、手勢、畫面與定位'`。 + +### Task E3: App lifecycle 與擴充能力 + +**Files:** 修改：`je_auto_control/utils/device_matrix/matrix.py`；新增：`je_auto_control/android/apps.py`、`je_auto_control/ios/apps.py`、`je_auto_control/wrapper/mobile_extensions.py`；測試：`test/unit_test/headless/test_mobile_app_lifecycle.py`。 + +**Interfaces:** `launch_app(session: DeviceSession, app_id: str) -> AppState`、`wait_for_app(session: DeviceSession, app_id: str, *, timeout_s: float) -> AppState`；`MobileExtension` Protocol 明確定義 install/files/clipboard/recording。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_launch_wait_stop, test_alert_accept_and_dismiss, test_clipboard_file_recording_capabilities, test_adapter_absent_reports_dependency`，驗證： + +```python +assert app_state_after_stop == 'not_running' +assert unsupported_result.reason +assert absent_adapter.state == 'needs_dependency' +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_mobile_app_lifecycle.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 先填已存在 backend 可支援項目，WDA 額外能力用明確可選 adapter；ADB shell 功能不假借到 iOS。採固定 timeout 與取消，metadata 矩陣覆蓋每個功能。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'E3: App lifecycle 與擴充能力'`。 + +### Task E4: 所有交付入口與實機案例 + +**Files:** 修改：`je_auto_control/gui/device_matrix_tab.py`、`je_auto_control/gui/script_builder/command_schema.py`、`je_auto_control/utils/executor/action_executor.py`、`je_auto_control/utils/mcp_server/tools/_factories.py`、`je_auto_control/__init__.py`；新增：`je_auto_control/gui/mobile_tab.py`；測試：`test/unit_test/headless/test_mobile_surface_parity.py`。 + +**Interfaces:** 每項 mobile API re-export，`AC_android_*`、`AC_ios_*` 與 MCP/builder schema 同源；mobile 面板實作 menu_actions。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_every_mobile_capability_has_api_command_gui_schema, test_wda_remote_endpoint_setup, test_android_authorization_error`，驗證： + +```python +assert missing_surface_commands == set() +assert unauthorized_input_calls == [] +assert device_setup_report.backend_version +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_mobile_surface_parity.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 以 code-backed matrix 逐項對照全部 executor 功能；桌面專屬項目記限制與替代。配置 Android emulator/實機及 remote WDA smoke；安裝／簽章／權限文件與範例同步交付。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'E4: 所有交付入口與實機案例'`。 diff --git a/docs/superpowers/plans/2026-10-02-f-gui.md b/docs/superpowers/plans/2026-10-02-f-gui.md new file mode 100644 index 000000000..c6747b1a2 --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-f-gui.md @@ -0,0 +1,112 @@ +# F：GUI 重寫與效能 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for native execution, or superpowers:subagent-driven-development if the user chooses delegation. Steps use checkbox syntax for tracking. + +**Goal:** 提供一致、可搜尋、延遲建立與可取消的桌面自動化工作區。 + +**Architecture:** 依核准設計做獨立可測的增量交付，沿用 headless → executor → GUI/MCP 邊界。使用 A–E 共用能力與 context，保持舊分頁 API；theme 不要求新 bitmap 相依。 + +**Tech Stack:** PySide6、typed factories/worker Protocol、既有語系目錄、GUI tests + +**Spec:** [核准設計](../specs/2026-10-02-platform-gui-modernization-design.md) + +## Global Constraints + +- Python ≥3.10；新檔 ≤750 行；公開 API 有 docstring 與型別；例外屬 AutoControlException。 +- 保留 import、AC_* 名稱與舊 CLI 旗標；headless import 不載入 Qt；相依延遲匯入。 +- 一般 GUI 面板操作經 Actions menu；重寫模組不得擴大 mypy 豁免或全域忽略。 +- `architecture.md`、`architecture_explore.md`、三語 README、對應 Sphinx 文件與 `docs/updates/` 隨交付更新；僅完成的 Progress 條目才刪除。 +- 測試使用受控輸入、不操作使用者桌面；有實機證據才標示實機已驗證。 + +## Review Focus + +- 未開過的功能不能在啟動時匯入或建立。 +- 關閉 widget 後 worker 不能呼叫銷毀物件。 +- 網路與裝置操作不能阻塞 UI event loop。 +- 語系、字級、混合 DPI 改變後導航不能失效。 +- 所有既有功能必須可達且操作仍服從 Actions menu 規範。 + +## 驗證與提交規則 + +每個 task 先加重現測試、確認失敗、實作、確認成功、更新相關文件並逐檔提交。 +下列 assertion snippets 的測試資料與 fake fixtures 在同一測試模組定義，不能修改 assertion 迎合實作。 +Step 4 通過後，該階段另外執行 ruff、相關既有回歸與三目標型別 gate；全面 coverage 與平台 smoke 於整合階段執行。 +檔案列出的既有目錄是要逐項檢查的入口；新增模組名稱與以下接口固定，搬移時保留兼容路徑。 + +--- + +### Task F1: 延遲 tab registry + +**Files:** 修改：`je_auto_control/gui/main_widget.py`、`je_auto_control/gui/main_window.py`；新增：`je_auto_control/gui/tab_registry.py`；測試：`test/unit_test/headless/test_gui_lazy_registry.py`。 + +**Interfaces:** `TabSpec` frozen key/title_key/category/factory/actions metadata；`TabRegistry.open(key: str) -> QWidget`、`close(key: str) -> None`。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_unopened_feature_is_not_imported, test_show_hide_list_api_compatible, test_close_releases_subscriptions`，驗證： + +```python +assert unopened_module not in sys.modules +assert reopened.tab_key == previous.tab_key +assert listeners_after_close == listeners_before_open +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_gui_lazy_registry.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 先記錄舊 tab show/hide/list 契約，所有面板改成 factory，不在 registry load 建立 QWidget。Actions menu metadata 不能依賴未建立的 widget，首次打開再綁 handler。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'F1: 延遲 tab registry'`。 + +### Task F2: 導航、布局與主題 + +**Files:** 修改：`je_auto_control/gui/main_window.py`、`je_auto_control/gui/language_wrapper/english.py`、`je_auto_control/gui/language_wrapper/traditional_chinese.py`、`je_auto_control/gui/language_wrapper/simplified_chinese.py`、`je_auto_control/gui/language_wrapper/japanese.py`；新增：`je_auto_control/gui/navigation.py`、`je_auto_control/gui/workspace.py`、`je_auto_control/gui/theme.py`；測試：`test/unit_test/headless/test_gui_workspace_navigation.py`。 + +**Interfaces:** `WorkspaceShell(registry: TabRegistry, parent: QWidget | None = None)`；`ThemeTokens` 統一 color/spacing/font，`apply_theme(window: QWidget, theme: ThemeTokens) -> None`。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_search_reaches_all_features, test_keyboard_navigation_and_actions, test_language_and_font_refresh, test_small_window_is_usable`，驗證： + +```python +assert reachable_keys == registered_keys +assert focused_action.triggered is True +assert horizontal_content_clipped is False +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_gui_workspace_navigation.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 左導航／中工作區／右可收合詳情，預設錄製／腳本／遠端。使用 native Qt vector/icon 或既有資產，完整空白／錯誤／permission states；small-window 行為與真實截圖審閱。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'F2: 導航、布局與主題'`。 + +### Task F3: worker、取消與關閉 + +**Files:** 修改：`je_auto_control/gui/_worker_thread.py`、`je_auto_control/gui/device_matrix_tab.py`、`je_auto_control/gui/remote_desktop/webrtc_panel.py`；新增：`je_auto_control/gui/task_controller.py`、`je_auto_control/gui/config_sync_tab.py`；測試：`test/unit_test/headless/test_gui_task_lifecycle.py`。 + +**Interfaces:** `TaskController.submit(work: Callable[[CancellationToken], object], *, owner: QObject) -> TaskHandle`；signals 傳 typed result/error/progress。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_ui_tick_continues_during_network_wait, test_cancel_releases_session, test_owner_death_drops_result, test_no_worker_touches_widget`，驗證： + +```python +assert ui_ticks_during_work > 0 +assert result_after_owner_death is None +assert active_sessions_after_close == 0 +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_gui_task_lifecycle.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 共用 QObject/worker 管理，不跨執行緒操作 widget。timeout 與 cancel 傳入 backend；完成先核對 owner/run/session。拆超限 panel 的 dialog/worker/session 責任，遍查 GUI I/O 呼叫。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'F3: worker、取消與關閉'`。 + +### Task F4: GUI 基準與回歸 + +**Files:** 修改：`test/unit_test/headless/test_actions_menu_gui.py`、`test/unit_test/headless/test_gui_tab_lifecycle_audit.py`；新增：`benchmarks/gui_startup.py`、`benchmarks/gui_workloads.py`；測試：`test/unit_test/headless/test_gui_feature_parity.py`。 + +**Interfaces:** benchmark JSON 記環境、startup_ms、memory、first_open_ms、event_loop_p95_ms；compare 相同 workload/env 的 before/after。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_default_workflow_and_full_catalog, test_mixed_dpi_geometry, test_import_facade_is_qt_free`，驗證： + +```python +assert default_keys == ['record', 'script_builder', 'remote_desktop'] +assert unsupported_feature_has_reason is True +assert facade_imports_qt is False +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_gui_feature_parity.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 基準依量測設定門檻；保存可審閱截图與報告，驗證各平台 layout、語系、鍵盤、cancel/close；所有舊 feature 可達，所有一般面板 menu_actions 仍過既有 gate。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'F4: GUI 基準與回歸'`。 diff --git a/docs/superpowers/plans/2026-10-02-g-mcp.md b/docs/superpowers/plans/2026-10-02-g-mcp.md new file mode 100644 index 000000000..03976f511 --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-g-mcp.md @@ -0,0 +1,93 @@ +# G：MCP 逐步揭露 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for native execution, or superpowers:subagent-driven-development if the user chooses delegation. Steps use checkbox syntax for tracking. + +**Goal:** 依 session 搜尋與揭露必要工具，降低 schema 體積且維持權限與協定相容。 + +**Architecture:** 依核准設計做獨立可測的增量交付，沿用 headless → executor → GUI/MCP 邊界。依賴 A 路徑/RBAC 與 D/E capabilities，完整 registry 與 AC_run_agent 預設保留。 + +**Tech Stack:** 既有 MCP JSON-RPC/HTTP、typed descriptor index、stdlib locks + +**Spec:** [核准設計](../specs/2026-10-02-platform-gui-modernization-design.md) + +## Global Constraints + +- Python ≥3.10；新檔 ≤750 行；公開 API 有 docstring 與型別；例外屬 AutoControlException。 +- 保留 import、AC_* 名稱與舊 CLI 旗標；headless import 不載入 Qt；相依延遲匯入。 +- 一般 GUI 面板操作經 Actions menu；重寫模組不得擴大 mypy 豁免或全域忽略。 +- `architecture.md`、`architecture_explore.md`、三語 README、對應 Sphinx 文件與 `docs/updates/` 隨交付更新；僅完成的 Progress 條目才刪除。 +- 測試使用受控輸入、不操作使用者桌面；有實機證據才標示實機已驗證。 + +## Review Focus + +- 兩個 HTTP session 的工具清單不能互相影響。 +- readonly 或 root 限制不能經搜尋／啟用繞過。 +- 插件熱更新不能產生混雜 cursor 頁面。 +- 不支援 listChanged 的 client 仍有可用 profile。 +- 搜尋結果不能每次帶回所有完整 schema。 + +## 驗證與提交規則 + +每個 task 先加重現測試、確認失敗、實作、確認成功、更新相關文件並逐檔提交。 +下列 assertion snippets 的測試資料與 fake fixtures 在同一測試模組定義，不能修改 assertion 迎合實作。 +Step 4 通過後，該階段另外執行 ruff、相關既有回歸與三目標型別 gate；全面 coverage 與平台 smoke 於整合階段執行。 +檔案列出的既有目錄是要逐項檢查的入口；新增模組名稱與以下接口固定，搬移時保留兼容路徑。 + +--- + +### Task G1: 工具搜尋與 schema 查詢 + +**Files:** 修改：`je_auto_control/utils/mcp_server/tools/_base.py`、`je_auto_control/utils/mcp_server/tools/_factories.py`；新增：`je_auto_control/utils/mcp_server/discovery.py`；測試：`test/unit_test/headless/test_mcp_discovery_index.py`。 + +**Interfaces:** `ToolIndex.search(query: str, *, limit: int = 10) -> list[ToolSummary]`、`get_schema(name: str) -> MCPToolDescriptor`；輸入limit限制且 category/capability typed。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_search_returns_summary_not_full_schema, test_only_authorized_tools_are_indexed, test_schema_lookup_is_single_tool`，驗證： + +```python +assert 'inputSchema' not in search_results[0] +assert hidden_tool not in search_names +assert schema_reply.name == selected_tool +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_mcp_discovery_index.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** registry 唯一來源，index 快照與 plugin version；新增 discover/schema 工具及 GUI MCP 設定入口，所有輸入通過既有驗證。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'G1: 工具搜尋與 schema 查詢'`。 + +### Task G2: session 揭露與分頁 + +**Files:** 修改：`je_auto_control/utils/mcp_server/server.py`、`je_auto_control/utils/mcp_server/http_sessions.py`、`je_auto_control/utils/mcp_server/_protocol.py`；新增：`je_auto_control/utils/mcp_server/disclosure.py`；測試：`test/unit_test/headless/test_mcp_progressive_sessions.py`。 + +**Interfaces:** `ToolView.enable(names: Sequence[str]) -> DisclosureResult`、`list_page(cursor: str | None) -> ToolPage`；ToolPage 有 nextCursor 與 snapshot ID。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_enable_is_session_local, test_cursor_snapshot_survives_plugin_change, test_enable_emits_list_changed, test_full_mode_is_compatible`，驗證： + +```python +assert tool not in other_session.visible_names +assert list_changed_count == 1 +assert full_names == registry_names +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_mcp_progressive_sessions.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** progressive 首次 core discover/enable/state，小清單；啟用後工具加入 session view。stale cursor 明確錯誤或完成原 snapshot，session終了回收。完整模式與靜態 profile 保持既有部署。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'G2: session 揭露與分頁'`。 + +### Task G3: 權限一致與成本量測 + +**Files:** 修改：`je_auto_control/utils/mcp_server/server.py`、`je_auto_control/utils/mcp_server/audit.py`、`je_auto_control/utils/mcp_server/__main__.py`；新增：`benchmarks/mcp_discovery.py`；測試：`test/unit_test/headless/test_mcp_disclosure_policy.py`。 + +**Interfaces:** 揭露只改变 availability，仍經 `_prepare_tool_call` 與現有 dispatcher；report 包含 count/json_bytes/handshake_ms/search_ms。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_discovered_call_uses_existing_policy, test_readonly_forbids_mutating_enable, test_removed_tool_cannot_run, test_plain_client_static_profile`，驗證： + +```python +assert policy_checks == expected_checks +assert mutating_tool_visible is False +assert removed_tool_executed is False +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_mcp_disclosure_policy.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 核心工具也依 readonly/root/user capabilities；插件移除即使舊 session 存名稱也禁止調用。測試 stdio、HTTP、版本協商、cancel 與無listChanged client；對照 full/progressive schema bytes。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'G3: 權限一致與成本量測'`。 diff --git a/docs/superpowers/plans/2026-10-02-h-typing-docs-validation.md b/docs/superpowers/plans/2026-10-02-h-typing-docs-validation.md new file mode 100644 index 000000000..d4bbae32a --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-h-typing-docs-validation.md @@ -0,0 +1,93 @@ +# H：型別深化、完整文件與總驗收 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for native execution, or superpowers:subagent-driven-development if the user chooses delegation. Steps use checkbox syntax for tracking. + +**Goal:** 驗證全套型別契約、所有新流程文件與平台功能矩陣一致。 + +**Architecture:** 依核准設計做獨立可測的增量交付，沿用 headless → executor → GUI/MCP 邊界。各階段都更新文件與型別；本計畫做跨階段的最終驗收。 + +**Tech Stack:** mypy、ruff、bandit、radon、coverage、Sphinx、既有 CI + +**Spec:** [核准設計](../specs/2026-10-02-platform-gui-modernization-design.md) + +## Global Constraints + +- Python ≥3.10；新檔 ≤750 行；公開 API 有 docstring 與型別；例外屬 AutoControlException。 +- 保留 import、AC_* 名稱與舊 CLI 旗標；headless import 不載入 Qt；相依延遲匯入。 +- 一般 GUI 面板操作經 Actions menu；重寫模組不得擴大 mypy 豁免或全域忽略。 +- `architecture.md`、`architecture_explore.md`、三語 README、對應 Sphinx 文件與 `docs/updates/` 隨交付更新；僅完成的 Progress 條目才刪除。 +- 測試使用受控輸入、不操作使用者桌面；有實機證據才標示實機已驗證。 + +## Review Focus + +- 安裝 GUI/行動 extras 後仍要維持 type gate 意義。 +- 文件指令不能只在某位開發者機器可用。 +- 離線例子不能在 validate 時操作桌面。 +- fake/device smoke 不等於實機驗證。 +- 下游 editable install 不能被測試环境導到另一份套件。 + +## 驗證與提交規則 + +每個 task 先加重現測試、確認失敗、實作、確認成功、更新相關文件並逐檔提交。 +下列 assertion snippets 的測試資料與 fake fixtures 在同一測試模組定義，不能修改 assertion 迎合實作。 +Step 4 通過後，該階段另外執行 ruff、相關既有回歸與三目標型別 gate；全面 coverage 與平台 smoke 於整合階段執行。 +檔案列出的既有目錄是要逐項檢查的入口；新增模組名稱與以下接口固定，搬移時保留兼容路徑。 + +--- + +### Task H1: 型別檢查深度與相依矩陣 + +**Files:** 修改：`pyproject.toml`、`test/verify/typing_contract_verify.py`、`test/verify/typing_contract_exempt.txt`、`.github/workflows/quality.yml`；測試：`test/unit_test/headless/test_typing_modernization_contract.py`。 + +**Interfaces:** 新／重寫模組限定 disallow_untyped_defs、disallow_any_generics；win32/linux/darwin 目標維持，extras 型別檢查另列 job。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_exemptions_remain_empty, test_new_modules_have_complete_annotations, test_adapter_is_only_sdk_any_boundary`，驗證： + +```python +assert exemptions == set() +assert untyped_public_signatures == [] +assert sdk_any_leaks == [] +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_typing_modernization_contract.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 為 SDK 邊界建立 Protocol/typed adapters，新增 py-modules 也加入範圍。不要 blanket skip GUI；保留確實無法被3.10解析的依賴特定理由，空豁免不變。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'H1: 型別檢查深度與相依矩陣'`。 + +### Task H2: 完整範例與三語文件 + +**Files:** 修改：`examples/README.md`、`README.md`、`README/README_zh-TW.md`、`README/README_zh-CN.md`、`docs/CAPABILITY_MATRIX.md`、`docs/API_LIFECYCLE.md`、`docs/source/Eng`、`docs/source/Zh`；新增：`examples/28_wayland_diagnostics.py`、`examples/29_config_sync.py`、`examples/30_mobile_devices.py`、`examples/31_healing_comparison.py`、`examples/32_codegen_from_log.py`、`examples/33_mcp_progressive.py`；測試：`test/unit_test/headless/test_modernization_examples.py`。 + +**Interfaces:** 每個 example 提供 --validate 或可無副作用的測試入口；功能矩陣由能力 metadata 校對；範例數由既有測量 gate 更新。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_examples_compile_and_validate_without_device, test_readme_configuration_parity, test_matrix_matches_capabilities`，驗證： + +```python +assert device_effects_in_validate == [] +assert translated_configuration_keys == english_configuration_keys +assert undocumented_capabilities == set() +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_modernization_examples.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** 交付安裝、啟動、權限、GUI操作、headless與CLI/MCP、故障與遷移完整流程。Sphinx 引用 shared examples，三README一致，API lifecycle標示兼容／新format，所有計數重新量測。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'H2: 完整範例與三語文件'`。 + +### Task H3: 跨平台、下游與發布前回歸 + +**Files:** 修改：`.github/workflows/platform-smoke.yml`、`.github/workflows/quality.yml`、`test/verify/macos_verify.py`、`test/integrated_test`、`architecture.md`、`architecture_explore.md`、`Progress.md`、`docs/updates/README.md`；測試：`test/unit_test/headless/test_modernization_integration.py`。 + +**Interfaces:** integration reports 帶 platform/backend/version、actual/skipped reason；只在測試證據存在時移除 Progress 對應條目。 + +- [ ] **Step 1:** 定義 fake fixtures 並新增 `test_journal_to_script_to_device_result, test_sync_restart_and_gui_session, test_platform_capability_report_has_evidence`，驗證： + +```python +assert replay_report.source_steps == manifest.source_steps +assert reopened_sync.revision == committed_revision +assert all_verified_capabilities_have_evidence is True +``` + +- [ ] **Step 2:** 執行 `.venv/Scripts/python.exe -m pytest -q --tb=short --basetemp=.test-tmp/modernization-task -o cache_dir=.test-tmp/modernization-pytest-cache test/unit_test/headless/test_modernization_integration.py`，確認新測試因原有缺陷或尚未提供接口而 FAIL；不要把環境錯誤當成功的重現。 +- [ ] **Step 3:** Python3.10–3.14與各OS CI、現有coverage floor不下降；手動／CI native EI、GNOME/KDE、Retina、Android、iOS結果附文件。執行下游 test_gui_facade/test_gui_control 與跨專案契約；有阻礙寫進 Progress，不能假完成。 +- [ ] **Step 4:** 重跑 Step 2，預期 exit 0；再跑這些修改檔所對應的既有回歸，確認公開契約。 +- [ ] **Step 5:** 更新本 task 的型別、所有交付入口、文件、測量計數與進度，僅 stage 本 task 檔案，提交 `git commit -m 'H3: 跨平台、下游與發布前回歸'`。 diff --git a/docs/superpowers/plans/2026-10-02-modernization-index.md b/docs/superpowers/plans/2026-10-02-modernization-index.md new file mode 100644 index 000000000..a74da6f7a --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-modernization-index.md @@ -0,0 +1,61 @@ +# 跨平台與 GUI 改版實作計畫 + +狀態：設計已核准；以下實作計畫待審閱，產品實作尚未開始。 + +設計：[2026-10-02-platform-gui-modernization-design.md](../specs/2026-10-02-platform-gui-modernization-design.md)。 + +## 執行方式 + +建議直接在本次工作中逐 task 實作，依 `superpowers:executing-plans` 執行。 +每個 task 做失敗重現、修正、驗證與獨立提交；依賴未完成不能跳過。 +可獨立審閱的完成階段交付後，繼續下一階段，直到全範圍完成或遇到必須由外部提供的條件。 + +隔離開發依實際執行中模組判斷，尤其 Jeffrey_RPA 使用 editable 工作樹。 +不能先把尚未驗證的改版放進正在執行的正式流程；也不能停掉使用者批次來迴避問題。 + +## 子計畫 + +| 順序 | 可審閱實作計畫 | 交付 | +| --- | --- | --- | +| A | [A：既有決策與執行契約](2026-10-02-a-contracts.md) | 完成已核准的 Progress 修正，建立之後改版不能破壞的執行邊界。 | +| B | [B：動作日誌、自愈量測與候選腳本](2026-10-02-b-journal-healing-codegen.md) | 讓動作來源可追溯、自愈版本可比較，並從日誌產生可驗證的候選腳本。 | +| C | [C：持久化同步與遠端 session](2026-10-02-c-sync-sessions.md) | 讓多機設定在重啟、離線及並行更新後仍一致，並隔離遠端連線所有權。 | +| D | [D：Wayland 與原生依賴可靠性](2026-10-02-d-wayland-dependencies.md) | 修正 Wayland 生命周期，提供可測替代後端及正確的能力診斷。 | +| E | [E：Android 與 iOS 功能完整化](2026-10-02-e-mobile.md) | 補齊行動自動化功能並提供隔離的多裝置工作流程。 | +| F | [F：GUI 重寫與效能](2026-10-02-f-gui.md) | 提供一致、可搜尋、延遲建立與可取消的桌面自動化工作區。 | +| G | [G：MCP 逐步揭露](2026-10-02-g-mcp.md) | 依 session 搜尋與揭露必要工具，降低 schema 體積且維持權限與協定相容。 | +| H | [H：型別深化、完整文件與總驗收](2026-10-02-h-typing-docs-validation.md) | 驗證全套型別契約、所有新流程文件與平台功能矩陣一致。 | + +## 需求覆蓋 + +- 原有 Progress 已核准決策：A1–A14，遠端 session 在 C4；RBAC 在 A10 使用核准設計預設。 +- Wayland 與函式庫：D1–D3，座標與原生平台契約 A8/A12–A14，安全相依 A11。 +- 跨機器不同步：C1–C4，file/clipboard 循環與資產完整性 C3。 +- UI 與 GUI 重寫／最佳化：F1–F4；各新能力同步交付面板，最後接入新 shell。 +- mypy：所有 task 與 H1；既有三目標檢查基準為 0 錯誤模組。 +- iOS／Android：E1–E4；能力矩陣 H2，實機與平台證據 H3。 +- MCP 逐步揭露：G1–G3；root/RBAC 沿用 A6/A10。 +- 自愈可量測：B2；錄影與對應 frame 來源 B1/E2。 +- 動作日誌 codegen：B1、B3。 +- 完整範例與文件：每個 task 同步交付，H2–H3 做全套校對與建置。 +- 原有未完成項目：A8/A9、D2/D3 與 H3；未有真實 API／實機證據時保持待驗證。 + +## 整體驗證命令 + +```powershell +& '.venv/Scripts/python.exe' -m ruff check je_auto_control/ je_auto_control_pytest.py +& '.venv/Scripts/python.exe' test/verify/typing_contract_verify.py +& '.venv/Scripts/python.exe' test/unit_test/headless/test_doc_line_counts.py --fix +& '.venv/Scripts/python.exe' -m coverage run -m pytest --timeout=120 +& '.venv/Scripts/python.exe' -m coverage report +& '.venv/Scripts/python.exe' -m build +``` + +依專案規範再執行 bandit、pylint、radon；Sphinx 英文與中文各自建置。 +平台測試要分開列出實測／未測項，CI artifact 保留對應版本與執行條件。 +新增頂層 pytest plugin 後須重新安裝本專案讓 entry point 生效。 + +## 狀態追蹤 + +進度只記未完成的工作；完成項目移至 docs/updates，設計／計畫不是已完成功能的證據。 +現有 Progress 的使用者決策修改仍保留，提交只納入本階段內容。 diff --git a/docs/superpowers/specs/2026-10-02-platform-gui-modernization-design.md b/docs/superpowers/specs/2026-10-02-platform-gui-modernization-design.md new file mode 100644 index 000000000..2964c9432 --- /dev/null +++ b/docs/superpowers/specs/2026-10-02-platform-gui-modernization-design.md @@ -0,0 +1,284 @@ +# 跨平台自動化與 GUI 改版設計 + +狀態：設計已核准；本文件描述預定行為，不代表功能已實作。 + +## 1. 目標與範圍 + +依根目錄 `Progress.md` 的未完成項目與維護者已填寫的 `[Answer]`，完成下列工作： + +1. 重新設計 UI，重寫並優化 GUI。 +2. 修正 Wayland 後端、資源管理與可用性；有問題的函式庫以修正、替代後端或隔離方式處理。 +3. 解決跨機器設定、檔案、工作狀態與遠端連線不同步的問題。 +4. 補強全套 mypy 檢查。 +5. 補齊 iOS、Android 的自動化功能與所有交付介面。 +6. MCP 採取逐步揭露，降低首次載入與每次互動的 schema 成本。 +7. 讓自愈定位器的改動可透過固定資料集與真實執行結果量測。 +8. 從動作日誌接上 codegen，產生可審閱的候選腳本。 +9. 補齊可執行範例、完整設定說明、API 文件與疑難排解。 + +成功條件是實際功能、相容性測試、平台測試與文件一致。能力受作業系統或授權限制時， +必須提供可用的替代流程與具體原因，不能回傳假成功、假座標或以跳過測試宣稱支援。 + +這是多個子專案；各階段各自維持可用版本與獨立提交。既有公開 import、`AC_*` 名稱、 +舊式 CLI 與跨專案契約沿用，必要的行為變更按照 `Progress.md` 已核准的決策遷移。 + +## 2. 已確認的現況 + +| 領域 | 原始碼證據 | 改版要處理的差距 | +| --- | --- | --- | +| GUI | `gui/main_widget.py` 在建構時匯入及建立全部面板；`main_window.py` 使用分頁與 qt-material | 首次啟動成本、導航、工作執行緒與關閉生命週期 | +| Wayland | `linux_wayland/libei.py` 依交握狀態釋放資源；`listener.py`、`record.py` 為受限實作 | 半開交握 crash 迴避、取消授權、裝置暫停與重連、錄製與停止機制 | +| 設定同步 | `utils/config_sync/client.py` 執行 fetch → merge → 無條件 push | 並行更新、離線重送、時鐘偏差、刪除項目過期與衝突可見性 | +| 同步 server | `utils/remote_desktop/signaling_server.py::_ConfigStore` 使用記憶體 dict | 重啟資料遺失、持久化與原子版本檢查 | +| 遠端連線 | `utils/remote_desktop/registry.py` 每 transport 共用 host/viewer 槽位 | 面板互相中斷、過期回呼、連線所有權 | +| 型別 | `test/verify/typing_contract_verify.py` 檢查三個目標；豁免清單為空 | 未標註函式、泛用容器與第三方 `Any` 邊界的檢查深度 | +| 行動裝置 | `android/` 提供 ADB、UIAutomator；`ios/` 提供 WDA、基本輸入與定位 | 能力一致性、裝置隔離、生命週期、GUI 與文件完整度 | +| MCP | `utils/mcp_server/server.py::_handle_tools_list` 回傳整份 registry descriptor | 首次 schema 體積、搜尋、依 session 揭露與權限一致性 | +| 自愈 | `utils/self_healing/{locator,heal_log}.py` 記錄 image/VLM/miss 與耗時 | 定位版本、結果驗證、資料集評估、品質與成本比較 | +| codegen | `utils/codegen/codegen.py` 接收動作清單；支援 pytest/python/robot | 結構化日誌轉換、步驟來源、秘密遮罩、候選預覽與驗證 | + +2026-10-02 本機執行 `.venv/Scripts/python.exe test/verify/typing_contract_verify.py`： +win32、linux、darwin 均為 0 個錯誤模組。這只能證明現有設定通過，不能推論所有函式已嚴格標註。 + +`Progress.md` 已有使用者修改；實作與提交只處理本階段內容，保留原有決策。 +`config_sync/__init__.py` 所稱尚無 `/config` endpoint 與實際 server 不一致，隨同步階段修正。 + +## 3. 改版方式 + +採用分階段替換：保留 headless 核心與公開契約，重寫 GUI shell、平台縫與確定有缺陷的模組。 +測試與量測先建立基準，再替換實作。所有新功能同時交付 Python API、executor、GUI、 +必要的 CLI/MCP 介面與使用文件。 + +整套一次重寫會讓既有後端與契約難以驗證；只換配色則無法處理同步、生命週期與平台問題。 +分階段替換能逐步驗證每個子系統，並讓 GUI 在核心功能成熟後接上統一能力資訊。 + +### 3.1 共用能力與執行邊界 + +以型別化的能力描述表示 available、needs_permission、needs_dependency、unsupported 與原因。 +探測不自動要求權限、不開啟裝置控制；真正的操作才建立 session。 +GUI、executor 與 MCP 使用同一份資訊，並共享參數驗證及錯誤型別。 + +每個頂層執行、裝置、遠端連線與背景工作有自己的 context，包含 run/session/device ID、 +取消訊號與變數範圍。context 以明確參數或受控邊界傳遞，不以另一個全域可變 singleton 代替。 +例外繼承框架的 `AutoControlException`；結果不把「未取得能力」編碼成成功。 + +### 3.2 UI 與 GUI + +桌面工具保留 PySide6，主介面改為可搜尋的左側功能導航、中間工作區、右側可收合的執行詳情。 +導航分成錄製／腳本、定位／檢查、裝置／遠端、執行／報告、設定／診斷。 +預設入口呈現錄製、腳本與遠端連線。功能以 factory 延遲匯入、第一次開啟才建立， +關閉時釋放 worker、session 與訂閱。保留舊的分頁識別碼及 show/hide/list 相容入口。 + +使用一致的間距、字級、焦點狀態、空白狀態、錯誤提示、淺色與深色主題。 +支援視窗縮放、混合 DPI、多螢幕、鍵盤導航與動態語系；依現有語系目錄同步所有新增字串。 +一般面板的操作仍由 Actions 選單提供，符合專案既有規範；Script Builder 與 Remote Desktop +保留互動操作。背景工作顯示進度與取消結果。 + +檔案、網路、ADB/WDA、OCR/VLM、腳本執行與同步移到 worker。worker 僅回傳資料， +UI 更新透過 Qt signal。關閉畫面不能留下 callback 存取已銷毀的 Qt 物件。 + +先量測冷啟動、開啟面板、記憶體、事件迴圈延遲與關閉耗時；同一台機器、相同依賴與 +相同 workload 比較。效能門檻在取得基準後記入 benchmark 設定，不能用未量測的數字宣稱改善。 + +### 3.3 Wayland 與函式庫問題 + +輸入控制優先使用 RemoteDesktop portal 的授權 session 與 libei；ydotool 是明確配置的替代。 +擷取共用 ScreenCast/PipeWire 或已有可用的擷取後端，所有路徑遵守同一座標與螢幕原點契約。 +backend 選擇、授權取消、權限撤銷、compositor 重啟、裝置移除、鍵盤布局與按鍵釋放都要可觀察。 +XWayland 的能力必須按照實際覆蓋範圍呈現。 + +先以 `docker/libei_verify.py`、`docker/eis_verify.py` 重現目前 crash 狀態，並修正與程式不符的 +進度敘述。修 ctypes 原型、fd 擁有權與狀態機；若原生函式庫在合法清理仍會崩潰， +將 session 放入專用 helper process，使用有界、可取消的 IPC，讓作業系統回收該 helper 的資源。 +先量測 IPC 對輸入延遲的影響，並驗證 helper crash、啟動失敗與主程式退出。 +禁止在 GUI 主行程呼叫已知會 SIGSEGV 的清理來取代現有迴避。 + +動作記錄分為 executor 已執行動作與實體輸入。前者在所有後端提供一致的結構化日誌。 +後者在 Wayland 採取實際可用、需要明確配置的裝置事件讀取替代，並排除自己注入的事件。 +InputCapture 的觸發與啟用由 compositor 決定，不能當成任意時間啟動的一般全域錄製器。 +提供 GUI 停止控制與受支援的 GlobalShortcuts 路徑；權限不足時明確提示可用停止方法。 + +依序驗證現有 wlroots 測試環境、GNOME、KDE；授權允許與拒絕、不同 portal 版本與 +libei 支援版本均需留下測試報告。只通過 fake backend 不能把該功能標示為實機驗證。 + +### 3.4 跨機器同步 + +同步範圍包括腳本與引用資產、定位器、快捷鍵、觸發器、通訊錄，以及需要同步的工作設定。 +機器專屬路徑、認證秘密、裝置連線與暫存授權資訊保留在本機；腳本的秘密以參照表示。 +「同步」不自動執行同步過來的腳本或啟用觸發器。 + +server 使用 SQLite 持久化 user bucket 與修訂版；在同一交易比較 base revision 並寫入。 +client push 帶 base revision，落後回覆 conflict，重新 fetch、merge 並有限次重試。 +成功回傳 committed revision；網路中斷與不確定成功使用 operation ID 去重。 +舊 client 的無條件寫入只在明確相容模式開啟；新的 GUI 顯示 server 是否具備競爭保護。 + +每筆變更有 origin device、operation ID 與因果版本資訊。不同 entry 的更新自動合併； +同一 entry 的並行變更保留兩份資料與衝突供選擇，不依本機 wall clock 靜默覆寫。 +刪除使用版本化 tombstone；回收依已知裝置確認版本，離線裝置過期後必須完整重同步， +不能單靠經過幾天就丟棄 tombstone 而使刪除內容復活。 + +本機 outbox 持久化、退避重送；每個帳號與 endpoint 隔離。GUI 顯示 pending、syncing、 +synced、conflict、offline 與 last successful revision。資產以內容 hash 驗證，暫存後原子替換， +避免只同步到腳本而缺少樣板。既有 folder mirror 與 clipboard sync 分開驗證， +避免循環傳送、部分檔案、不同原點與斷線重連造成假同步。 + +遠端 registry 增加 session ID 與 owner，狀態與斷線指定 session。 +省略 session 的既有 `AC_remote_*` 指令仍操作腳本的預設 session；GUI 面板各自持有自己的 session。 +替換或結束 session 發出通知，過期 frame/error callback 依 session ID 丟棄。 + +### 3.5 Android 與 iOS + +保留已有 ADB、UIAutomator、WDA 的延遲匯入，建立共用 device context 與能力契約。 +每台裝置的連線、session、取消與重試互相隔離，平行執行不可修改預設 singleton 指向。 + +| 功能組 | Android 實作路徑 | iOS 實作路徑 | +| --- | --- | --- | +| 裝置與 session | ADB 探測、授權狀態、UIAutomator session | WDA endpoint 探測、session 建立與關閉 | +| 輸入與手勢 | tap、長按、swipe、drag、pinch、Unicode 輸入、按鍵 | WDA 對應輸入、手勢、文字與受支援按鍵 | +| 畫面與定位 | 截圖、原點／尺寸／旋轉、UI tree、OCR、樣板、VLM、自愈 | 相同 headless pipeline，適配 WDA point/pixel 座標 | +| App 與等待 | 啟動、停止、狀態、等待元素、alert | WDA App 狀態、啟動／終止、等待與 alert | +| 擴充裝置功能 | 安裝、檔案、剪貼簿、錄影等逐項 capability | WDA 不提供的項目以可配置的受支援 adapter 補足 | +| 工作與報告 | device matrix、結構化日誌、codegen、run history | 相同 context 與資料格式 | + +「所有功能」逐項對照 executor、Script Builder、MCP 與文件矩陣，不能只新增 SDK wrapper。 +桌面視窗管理、COM、USB 主機等不具行動平台對應語意的功能，需在矩陣列出替代或限制。 +iOS 擴充以現有 WDA 能力為基礎；確需安裝／檔案等額外服務時使用可選 adapter， +不以 ADB shell 的形式假設 iOS 可執行相同操作。 +實機建置與簽章需求按選定 backend 寫入文件；本機 Windows 可連遠端 WDA， +但不以此宣稱已在 Windows 本機完成 Xcode 建置與實機驗證。 + +### 3.6 MCP 逐步揭露 + +保留完整 registry 作為單一工具定義來源，另加 typed 搜尋索引與 session 可見工具集。 +漸進模式先提供狀態、工具搜尋與啟用工具，搜尋預設僅回傳名稱、簡短說明與能力需求； +需要時取得單一工具的完整 schema，啟用後才加入該 session 的 `tools/list`。 +完整模式保留既有使用方式；文件與啟動設定明確提供兩種模式。 + +啟用、停用與插件變更發出 `notifications/tools/list_changed`。tools/list 使用不透明 cursor +與穩定 snapshot；舊 cursor、已刪除 plugin、重複啟用與 session 關閉有一致處理。 +HTTP session 之間不能共享揭露狀態，stdio server 有自己的 session。 +對無法即時更新清單的 client，提供設定好的靜態能力 profile。 + +工具搜尋、schema 取得、啟用與呼叫都套用同一 read-only、root、env allowlist 與權限規則。 +漸進揭露不取代授權，也不新增可繞過驗證的通用 executor 工具。 +`AC_run_agent` 維持 `Progress.md` 的「不改」決策；本階段的 MCP 改版不暗中縮減其工具集合。 +量測 descriptor 數、JSON bytes、首次交握與搜尋／啟用耗時，並以完整模式作比較。 + +### 3.7 自愈定位器量測 + +保留舊 HealEvent 的讀取相容性，加入 schema version、run/step/locator ID、locator version、 +選用策略、frame hash、候選座標、各策略耗時、backend/model 與成本資料。 +完整修正 `screen_region` 在 image 與 VLM 兩條路徑的語意與座標換算。 + +定位成功與操作成功分別記錄。沒有 ground truth 或操作後的驗證不能算正確命中。 +固定資料集提供畫面、目標 box、原點／縮放與期望 miss；受控 replay 比較相同輸入的版本。 +報告包含 image hit、fallback、miss、正確率、false-positive、恢復成功率、p50/p95 延遲及 +有資料時的模型成本。每個比率附分子、分母；未標註項目顯示 unknown，不算成正確樣本。 + +回歸資料集與 thresholds 進版控，結果輸出 JSON 與可分享報告。 +GUI 顯示版本比較、失敗案例與原始步驟；headless 與 MCP 產生同一份報告。 +樣板修改產生候選版本，可預覽、驗證與回復；不直接用未驗證 VLM 座標覆寫舊定位器。 + +### 3.8 動作日誌與 codegen + +定義 append-only JSONL 日誌：schema version、run/step/parent ID、device/session、順序、 +command、可序列化參數、起訖、status、錯誤與可引用 artifact。 +在 executor 的動作邊界記錄，GUI、CLI、REST、MCP 走同一條路。 +秘密在寫入前轉成 secret reference 或遮罩；未知 payload、不可重播參數與缺失來源明確標記。 + +日誌 importer 驗證版本、順序與 command schema，按 run 選擇步驟並保留來源索引。 +保留 loop/branch/parallel 的階層；無法還原控制流程時產生明確標示的已觀察路徑候選， +不可憑空恢復沒有執行的分支。 +失敗、重試與補救步驟提供明確選取規則，不能把執行結果誤當輸入參數。 + +轉換後呼叫現有 `generate_code`，交付 Python、pytest、Robot 候選與 provenance manifest。 +靜態解析、command 驗證與無副作用 dry-run 後，GUI 提供 diff/預覽與匯入 Script Builder。 +產碼不自動執行；真正 replay 是獨立操作。 +不使用 eval 或執行文字 log 的 repr，任意文字 log 僅在能可靠解析的格式下支援。 + +### 3.9 型別、文件與範例 + +保留三目標 mypy 與空豁免清單。新寫／重寫模組啟用完整函式標註、型別化容器與 Protocol； +逐步把 SDK `Any` 限縮於 adapter，優先 GUI worker、同步 wire payload、device、MCP 與日誌。 +另外驗證已安裝 GUI 等 extras 的型別環境；不以全域 skip/ignore_errors 隱藏新版 SDK 的不相容。 +對 Python 3.10 無法解析的新型別語法維持必要的特定相依邊界，並記錄理由。 + +每個新流程都有最小可執行範例與整合範例，包含 GUI 對應操作、無 GUI 使用、權限配置、 +錯誤處理與結果。範例支援 validate/dry-run；需要實機的執行條件與指令明列。 +包含 Wayland 診斷／授權、兩台機器同步、Android/iOS、多裝置平行、自愈版本比較、 +日誌轉腳本、MCP 漸進模式與舊 client 相容模式。 + +同步更新三語 README、Sphinx 英文／中文、API lifecycle、capability matrix、CHANGELOG、 +`architecture.md`、`architecture_explore.md`、examples 索引與 measured counts。 +未完成工作留在 `Progress.md`；完成後才移入 `docs/updates/`。 + +## 4. 沿用 Progress.md 決策 + +| 原有項目 | 實作契約 | +| --- | --- | +| 簽章 | 簽章金鑰與執行權限分離；執行端不能取得預設簽章能力 | +| USB request/reply | 加 request ID；遲到回覆不能完成新的請求；舊 peer 無 ID 時需安全處理逾時歧義 | +| 執行變數 | 每次頂層執行使用新的 VariableScope；巢狀與平行任務繼承正確 context | +| 舊式 CLI | 動作失敗回 exit 1，維持既有旗標與跨專案契約 | +| MCP registry 名稱 | `io.github.integration-automation/autocontrol`，同步修正專案 URL | +| pytest11 | 頂層精簡 plugin；舊套件 import 路徑 re-export；重裝與 coverage 前提一起修正 | +| cryptography | 下限提升至已核准的 50；可選能力明確診斷，不能以不安全版本換 wheel | +| Viewer 收檔 | 預設下載根目錄與相對路徑，拒絕越界；文件與範例遷移 | +| AC_run_agent | 保持預設工具集決策，不加入未核准的工具縮減 | +| MCP 路徑 | 同時支援配置根目錄與 roots/list；不預設啟用唯讀限制 | +| Windows DPI | 換成 per-monitor v2，檢查 Jeffrey_RPA 使用端與混合 DPI 座標 | +| 遠端 registry | GUI 與 AC/MCP session 語意一起修正 | + +其他尚未完成項目也納入實作：macOS 最小化還原、Windows 副螢幕擷取、Retina 多螢幕、 +輸入 wrapper 修正、agent 截圖歷史壓縮、arm64 可選相依、Python 3.10 USB ACL 間歇 crash。 +agent API 與原生依賴版本在實作時查核官方文件，不能只依進度檔中的版本或 API 名稱猜測。 +Computer-use 預設切換須有真實 API 驗證才落地。 + +RBAC 與強制簽章的舊部署過渡仍需細化；建議以明確 opt-in 的 UserStore 配置啟用角色， +沒有配置時保留既有共用 token。角色到 route/tool 的對應由能力分類產生，稽核帶 user_id。 +本設計審閱時需確認這個尚未有 `[Answer]` 的預設方案。 + +## 5. 實作順序與驗收 + +| 階段 | 工作 | 必須通過的驗收 | +| --- | --- | --- | +| A | 基準、契約、已核准 Progress 修正 | 失敗重現、契約測試、pytest/plugin/CLI/簽章/USB 回覆隔離 | +| B | 日誌、自愈量測、codegen | 固定 frame 的前後比較；遮罩後日誌可轉碼；mock replay 與來源一致 | +| C | 持久同步、衝突與遠端 session | server 重啟、雙 client 並寫、斷線重送、刪除不復活、面板互不誤斷 | +| D | Wayland 與有問題的依賴 | 真實 EI peer、半開交握退出、授權允許／取消、裝置暫停／移除與資源界線 | +| E | 行動平台 | 多裝置隔離、Unicode、旋轉與 point/pixel、所有交付介面與實機案例 | +| F | GUI 重寫 | 全功能可達、延遲建立、無 UI 執行緒 I/O、取消／關閉、DPI／鍵盤／語系 | +| G | MCP 漸進模式 | 首次 schema 減少、搜尋／啟用、session 隔離、唯讀／root／plugin／cursor 相容 | +| H | 型別深化與總驗收 | 各階段同步深化型別；完整回歸、文件建置、範例 validate、矩陣一致 | + +每階段都同步交付型別、GUI/CLI/MCP 入口與文件；H 是整合驗收，不把文件延至最後才補。 +新的檔案遵守 750 行上限；重寫已超限模組先拆清楚的責任邊界。 + +單元測試使用可控 clock、frame、network、device 與 session，不驅動使用者桌面。 +整合驗證覆蓋 Python 3.10–3.14 與既有 Windows/Linux/macOS CI。 +coverage 沿用 `coverage run -m pytest` 與現有 floor；pytest11 改動後重新量測而不放寬 floor。 +GUI 測試另驗證真實 Qt 生命週期；headless import 維持 Qt-free 與 optional dependency lazy。 + +實機結果附 backend、版本、輸入資料與限制。無法在本機驗證的平台可交付測試腳本與 CI job， +但該項保留待驗證狀態。任何修正都需證明先前錯誤已不再出現，不能只宣稱新增了測試。 + +## 6. 跨專案與環境 + +Jeffrey_RPA 使用這個工作樹的 editable install。動到它正在執行的輸入、截圖與座標模組前， +先確認 batch/bot 狀態；必要時以隔離 checkout 開發，不停止或改寫其正式批次。 +下游驗證使用 `D:/Codes/AI_CONTEXT.md` 指定的 `test_gui_facade.py`、`test_gui_control.py`， +WebRunner 相關驗證另按其契約處理。不得把 `test_je_facade.py` 當成 AutoControl 契約唯一證據。 +修改其他 repo 或安裝相依時按實際檔案／網路權限處理。 + +提交按階段只 stage 本階段檔案。發佈、推送或合併遵照專案分支流程， +不能讓尚未完成的平台改版以完成狀態進入發佈流程。 + +## 7. 官方參考 + +- [XDG RemoteDesktop](https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.RemoteDesktop.html)：session、ScreenCast 共用與 ConnectToEIS。 +- [XDG InputCapture](https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.InputCapture.html)：compositor 啟動、pointer barrier、裝置能力與版本探測。 +- [MCP tools](https://modelcontextprotocol.io/specification/2025-06-18/server/tools)：tools/list、分頁與 listChanged；實作保持 server 目前協商版本的語意。 +- [Android ADB](https://developer.android.com/tools/adb)：裝置指定、連線與 shell 工具的邊界。 +- [Appium XCUITest capabilities](https://appium.github.io/appium-xcuitest-driver/latest/reference/capabilities/)：WDA 與可選擴充的設定及建置需求。 + +以上限制據官方文件；分階段替換、同步版本、GUI 布局與日誌格式是本專案提出的設計選擇。 diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 344fd9335..fa95e543c 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -37,6 +37,59 @@ WebRunner's native commands (its U-20261001-28 and -29), `WR_ac_fill_native_file - `Progress.md`, in the keyboard fixes waiting for Jeffrey_RPA's batch to stop, records the missing piece for WebRunner's `WR_ac_basic_auth`. `write()` and `type_keyboard()` log the typed text at INFO, record it and return it, so a password typed for WebRunner would reach this package's log and both run records. A typing call that does none of that is needed. - **Files**: `architecture.md`, `Progress.md`, `test/unit_test/headless/test_cross_project_contracts.py`. +## U-20261001-03 · 2026-10-01 · Tesseract setup helpers: find the engine, set tessdata, report status and languages · #feature #ocr #contract + +Jeffrey_RPA's `_gui_control.py` imported `pytesseract` itself to list languages and check the engine, and wrote `pytesseract.pytesseract.tesseract_cmd` directly, so that module global had two writers. The library now answers those questions itself and `TesseractBackend.set_cmd` stays the only writer. + +- **New module** `utils/ocr/tesseract_setup.py`, exported from the facade and `utils/ocr`: + - `find_tesseract_cmd() -> Optional[str]`: `$TESSERACT_CMD` when it names a file, then `PATH`, then the installers' default folders (Windows `Program Files`, `Program Files (x86)`, `%LOCALAPPDATA%\Programs`; Homebrew, `/usr/local`, MacPorts; `/usr/bin`, `/usr/local/bin`). Looks only. + - `set_tessdata_dir(path) -> Optional[str]`: realpath, must be a directory, sets `TESSDATA_PREFIX`; `None` removes it. + - `ocr_languages() -> Optional[List[str]]`: sorted codes, `[]` when the engine answered with none, `None` when it could not be asked. Never cached. + - `ocr_status() -> OCRStatus(ok, reason)`: `missing_package`, `missing_engine` (the configured command is neither on `PATH` nor a file), `engine_unusable`, `no_language_data` (only for an answered empty list), else `ready`. Checks the command OCR would use and configures nothing. +- **Backend**: `TesseractBackend.cmd`, `version()` and `languages()`, each raising `OCRBackendNotAvailableError` when the engine cannot run. `version()` also catches the `SystemExit` pytesseract raises for a version string it cannot parse, which would otherwise end the calling process. +- **Surfaces**: `AC_ocr_status` → `{ok, reason}`, `AC_ocr_languages` → `{languages}` (`null` when unknown), the matching `ac_ocr_status` / `ac_ocr_languages` MCP tools (read-only) and two Script Builder entries. 776 commands, 679 MCP tools. +- **Real engine** (this host, read-only probe): `ocr_status()` gave `missing_engine` before configuration; `find_tesseract_cmd()` found `C:\Program Files\Tesseract-OCR\tesseract.exe` (not on `PATH`); after `set_tesseract_cmd` it gave `ready`, `ocr_languages()` listed `eng`, `osd`, and after `set_tessdata_dir` on a folder with three language files it listed those three. +- **Tests**: new `test_ocr_tesseract_setup.py` (27 tests, fake pytesseract): `None` versus `[]`, no caching, each status reason including `SystemExit`, one writer for the command, tessdata validation, every lookup step, the executor, MCP and Script Builder wiring. `test_cross_project_contracts.py` pins the four names for Jeffrey_RPA. +- **Docs**: `docs/source/{Eng,Zh}/doc/ocr_backends/ocr_backends_doc.rst` (setup notes, JSON surface, diagnostics), `README.md` and both translations (install note, OCR row, counts), `architecture.md` §6, `architecture_explore.md` (OCR rows, counts, re-measured lines), `je_auto_control/actions.pyi` regenerated (which also picks up the drifted signatures of `AC_queue_complete`, `AC_queue_fail` and `AC_shell_command`, and re-escapes one docstring quote), `CHANGELOG.md`. +- **Open items**: none. + +## U-20261001-04 · 2026-10-01 · click_mouse(clicks=, interval=): a double-click in one call · #feature #input + +Jeffrey_RPA double-clicked with two `click_mouse` calls and its own sleep. The wrapper now does it in one call. + +- **Signature**: `click_mouse(mouse_keycode, x=None, y=None, clicks=1, interval=0.0)` in `wrapper/auto_control_mouse.py`. The point is resolved once and every click lands on it; `time.sleep(interval)` runs only between clicks and only when `interval` is non-zero. Defaults are the old single click, and a single click records the same `click_mouse` action as before (the `clicks` / `interval` keys appear only when used). +- **Validation** (`_click_plan`, raising `AutoControlMouseException` before any click): `clicks` is an int or integer string, at least 1, never a `bool` or a float (a float would be truncated silently); `interval` is finite and at least 0 (NaN passes a plain `< 0` test). +- **Surfaces**: `AC_click_mouse` is the wrapper itself, so action files get both parameters; the `ac_click_mouse` MCP tool and its handler, the MCP fake backend (one recorded click per click) and the Script Builder fields follow. `je_auto_control/actions.pyi` regenerated. +- **macOS**: apps there read a click count the osx backend never sets, so `clicks=2` arrives as two single clicks. Documented in the docstring and docs, and recorded in `Progress.md`. +- **Tests**: new `test_click_mouse_multi.py` (22 tests, stand-in backend and clock): the unchanged default and its record, the double-click and its single pause, no pause at `interval=0`, macOS argument order on every click, each refused `clicks` / `interval` value clicking nothing, and the executor, MCP handler, fake backend, MCP schema and Script Builder wiring. +- **Docs**: `docs/source/API/wrapper/mouse.rst`, `docs/source/{Eng,Zh}/doc/mouse/mouse_doc.rst`, the Mouse row of `README.md` and both translations, `CHANGELOG.md`; `architecture.md` §6 lists the two keywords for Jeffrey_RPA and `test_cross_project_contracts.py` pins them. +- **Open items**: `Progress.md` "macOS 上 `click_mouse(clicks=2)` 不是雙擊". + +## U-20261001-05 · 2026-10-01 · Drag pacing for tween_drag / drag_path; an aborted drag lets go where it stopped · #feature #bugfix #input + +Jeffrey_RPA kept its own drag loop because the library's drags moved as fast as the events could be sent, which apps that judge a drag by the pointer's motion read as a click, and because an aborted drag released the button at the drop target. + +- **Shared sequence**: `utils/tween_drag/tween_drag.py` `_drag_through(points, button, dispatch, step_delay_s, settle_s)` now holds the press / move / release for both `tween_drag` and `mouse_path.drag_path`, which previously had two copies of the same loop. +- **Pacing**: `step_delay_s` rests after each move; `settle_s` (when non-zero) moves to the start and rests before the press, then rests after the press and before the release. Both default to 0, where the events are exactly press, moves, release with no sleep, as before. NaN, infinities, negatives and non-numbers raise `ValueError` before anything is dispatched (a NaN reaching `time.sleep` would raise with the button held). +- **Release on failure**: after the press, a `held` flag and a `finally` (the pattern of `hotkey` / `type_keyboard` and `_release_still_held`) release the button at the last point the pointer reached when any step raises, `KeyboardInterrupt` included. It used to release at the last point of the path, completing a drop the drag never reached. The cleanup release never raises over the original error (logged instead); a failing final release propagates after one cleanup retry, as `hotkey` does. A failed press releases nothing. +- **Surfaces**: the `AC_tween_drag` / `AC_drag_path` adapters, the `ac_tween_drag` / `ac_drag_path` MCP tools and handlers (`_handlers_executor_bridge.py` kept its line count, as its `Progress.md` ceiling requires) and the Script Builder fields take both. `je_auto_control/actions.pyi` regenerated. +- **Signatures**: `tween_drag(start, end, *, steps=30, easing="ease_in_out_quad", button="mouse_left", sink=None, step_delay_s=0.0, settle_s=0.0)` and `drag_path(waypoints, *, button="mouse_left", easing="linear", per_segment_steps=20, sink=None, step_delay_s=0.0, settle_s=0.0)`. `tween_drag` now has eight parameters, one over the `CLAUDE.md` review limit; all are keyword-only after `start` / `end`, and `wait_for_text` already has eight. +- **Tests**: new `test_drag_pacing.py` (25 tests, recording sink and clock): unchanged defaults for both drags, the pause order for each setting, every refused value sending nothing, release at the last reached point (mid-path, first move, interrupted pause), no release after a failed press, a failing cleanup not hiding the original error, the final-release retry, and the executor, MCP and Script Builder wiring. The existing drag tests pass unchanged. +- **Docs**: `docs/source/{Eng,Zh}/doc/new_features/v29_features_doc.rst` and `v116_features_doc.rst`, the Mouse row of `README.md` and both translations, `architecture.md` §6 (with `test_cross_project_contracts.py` pinning the keywords), `architecture_explore.md` (tween_drag row, the `finally` section, re-measured lines), `CHANGELOG.md` (Added and Fixed). +- **Open items**: none. + +## U-20261001-06 · 2026-10-01 · Windows OEM keys, browser_home, launch_app2 and key aliases; canonical reverse lookup · #feature #input #contract + +Jeffrey_RPA kept its own `KEY_ALIASES` (40 friendly names) and `_EXTRA_KEY_CODES` (16 keys the library could not name: the OEM keys, `browser_home`, and `launch_app2`, which existed only upper-case) and converted names to codes before every call. The Windows table now has all of them, with the same names and codes. + +- **Table** (`wrapper/_platform_windows.py`, constants in `windows/core/utils/win32_vk.py`): `oem_1` .. `oem_8`, `oem_102`, `oem_plus`, `oem_comma`, `oem_minus`, `oem_period`, `oem_clear`, `browser_home` (0xAC) and `launch_app2` (0xB7) beside the old `LAUNCH_APP2`. Codes checked against Microsoft's "Virtual-Key Codes". Layout-dependent OEM keys keep Microsoft's names; a character name such as `semicolon` would be wrong on other layouts. +- **Aliases**: one map, `keyboard_key_aliases` (alias → canonical name; the `"ctrl"` entry moved there from the table literal), merged into the table after the canonical names. Every platform module and `platform_wrapper` publish the map (empty off Windows). 230 names on Windows, 40 of them aliases (was 175 names). +- **Reverse lookup**: new `keyboard_key_name(keycode) -> Optional[str]` in `wrapper/auto_control_keyboard.py`, exported from the facade. Aliases never answer. Among the rest a typable name (`[a-z0-9_]+`) wins, then the shortest, then alphabetical order. For every code the old table already named, it returns exactly what a shortest-lower-case lookup over the pre-alias table returned (`control`, `return`, `escape`, `lwin`, `snapshot`, `num0`, `down`, `a`, ...). The new keys get their new names (`launch_app2`, not `LAUNCH_APP2`). +- **Jeffrey_RPA's current `_vk_to_name`** takes the shortest name from the raw table and then maps it through its own aliases. Simulated over VK 1-254 against both tables, it changes for one key: VK 0x2C writes `prtsc` instead of `printscreen`. Both names are the same key and validate and replay identically. Its modifier keys never reach the table. Calling `keyboard_key_name`, or skipping `keyboard_key_aliases`, removes the difference. Its `test_every_code_the_library_can_name_can_also_be_written` positive control ("the real table still has a key named only in upper case") stops holding once `launch_app2` exists in lower case. That test's own message says to revisit the premise in that case. +- **Tests**: new `test_key_names_and_aliases.py` (68 tests; the table checks run on Windows): Microsoft codes for the 16 added keys, the upper-case spelling kept, every alias resolving through `_resolve_keycode` to its target's code, no alias pointing at another alias or at a layout-dependent key, the pre-alias canonical name for every aliased code, an alias-free answer for every code in the table, the preference order on a synthetic table (runs everywhere), and the facade export. `test_cross_project_contracts.py` pins `keyboard_key_name`, `keyboard_key_aliases` and the names Jeffrey_RPA uses. +- **Docs**: `docs/source/API/special/keyboard_keys.rst` (the Windows list rewritten to names that exist, with aliases, OEM and launch keys; it listed `print_screen`, `lalt` and `numpad0`, which did not exist), `docs/source/API/wrapper/keyboard.rst`, `docs/source/{Eng,Zh}/doc/keyboard/keyboard_doc.rst`, the Keyboard row of `README.md` and both translations, `architecture.md` §6, `architecture_explore.md` (two rows, public-API and test counts re-measured, lines), `CHANGELOG.md` (Added and Changed). +- **Open items**: none. + ## U-20261001-07 · 2026-10-01 · write_secret / AC_write_secret: type a password without logging, recording or returning it · #done #keyboard #security #webrunner `Progress.md`'s "typing always leaves the text behind" item, needed by WebRunner's `WR_ac_basic_auth` (its `progress.md` #20). `write()` logs the text at INFO, records it and returns it, and the executor keyed its record by the action, so a password typed through it reached AutoControl's log and both run records. Done on its own branch, `feat/secret-typing` (from `feat/coverage-to-80`), while Jeffrey_RPA's batch, which imports this working tree, was stopped; the working tree itself was not touched. @@ -103,3 +156,51 @@ WebRunner's native commands (its U-20261001-28 and -29), `WR_ac_fill_native_file - **Docs**: `architecture.md` (§2 `scripts/` row, §3 PyPI packages row), `architecture_explore.md` (§5.6 `scripts/dev_release.py` row, §7 `dev.yml` row), `CLAUDE.md` › Key Conventions, `CHANGELOG.md` (Changed). No README or installation page mentions the dev package, so none changed. - **Files**: `.github/workflows/dev.yml`, `scripts/dev_release.py`, `dev.toml`, `test/unit_test/headless/{test_dev_release,test_dev_toml_parity}.py`, `architecture.md`, `architecture_explore.md`, `CLAUDE.md`, `CHANGELOG.md`. - **Open items**: the points under **Not running yet**. `Progress.md` is not part of this change, so it does not list them. + +## U-20261001-10 · 2026-10-01 · The publish jobs install hash-locked build tooling and build with the locked setuptools · #release #ci #security #X-13 + +- **What**: workspace X-13, decided by the owner. The two jobs that receive `secrets.PYPI_API_TOKEN`, `publish` in `stable.yml` and `publish-dev` in `dev.yml`, installed `build==1.5.0` and `twine==6.2.0` by version alone: no hashes, and their dependencies at whatever was newest that day. `python -m build` then created an isolated environment and downloaded the newest `setuptools` (`build-system.requires` is `setuptools>=82.0.1`). All of it runs in the job that is about to upload with the token. + - New `.github/requirements/publish.in` names `build==1.5.0` and `twine==6.2.0`, the versions the workflows named, and `setuptools`, the build backend. `publish.txt` is generated from it with `uv pip compile`: wheels only, with hashes, for Python 3.12 on `x86_64-manylinux_2_28`, `--exclude-newer 2026-09-24`. The command is at the top of `publish.in` and in the header of `publish.txt`. + - Both jobs now run one install, `python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt`, and build with `python -m build --no-isolation`, so the backend is the locked `setuptools`. Triggers, the other steps and the secret are unchanged. + - `release.yml` is unchanged. Its `build` job has no PyPI token (it checks the tag, smoke-tests the wheel and attests provenance) and its `publish` job is switched off, so it keeps `build==1.5.0 twine==6.2.0` and an isolated build. + - `.github/dependabot.yml`: the `pip` entry names `/` and `/.github/requirements` (`directories`). From `/` Dependabot does not look that deep, so it would never have proposed an update to the lock. Nothing else in that file changed. +- **Guards**: new `test_publish_tooling_lock.py`, 11 cases read from the files as text: + - the jobs that read the token are exactly `dev.yml:publish-dev` and `stable.yml:publish`; + - each runs exactly one `pip install`, the locked one, so an unpinned install, a second install or a pip upgrade fails (2 cases); + - every `python -m build` in them carries `--no-isolation` (2 cases); + - `build-system.requires` in `pyproject.toml` and in `dev.toml` names only packages `publish.txt` pins, at a version inside the written specifier (2 cases). `--no-isolation` checks the requirement instead of installing it, so a floor raised without regenerating the lock fails in the suite, not in the publish job; + - `publish.in` names exactly what the jobs run with `python -m` or import in an inline script, less pip and the standard library, plus the build backend; + - every name in `publish.in` is pinned in `publish.txt`; + - the lock's header records `--generate-hashes`, `--only-binary :all:` and the `--python-version` both jobs set up; + - Dependabot's `pip` entry covers `/` and `/.github/requirements`. + `test_dev_release.py` compared the `build==` line of `dev.yml` with `stable.yml`'s; it now compares the locked install line and the `--no-isolation` build line of the two jobs. +- **Result / numbers**: + - 30 pins: `build` 1.5.0, `twine` 6.2.0, `setuptools` 84.0.0 and 27 dependencies (among them `packaging` 26.3, `pyproject-hooks` 1.3.3, `requests` 2.34.2, `urllib3` 2.8.0, `cryptography` 50.0.1, `keyring` 25.7.0). + - `pip download --require-hashes --only-binary :all: --no-deps` for CPython 3.12 on manylinux x86_64 fetches all 30 wheels with matching hashes. + - Built in a throwaway worktree, in a scratch Python 3.12 environment holding the 30 locked versions: the stable metadata (0.0.225) and the dev metadata (`scripts/dev_release.py prepare`, 0.0.137), each with `python -m build --no-isolation` and, for comparison, with an isolated build. Both sdist and wheel build, `twine check` passes on all four files, each wheel has 1,062 members with the same names and the same content as the isolated build's, and each sdist the same 1,067 files. + - Against the published `je_auto_control_dev` 0.0.136 the dev wheel differs, line endings aside, only in its `Version:` line and `RECORD`, so `publish-dev` has nothing to upload for this change. + - Twelve ways of breaking the rule (an unpinned or a second install, an isolated build in either job, a floor above the lock in either metadata file, a backend the lock does not pin, Dependabot on `/` alone, a tool not in `publish.in`, a name in `publish.in` not in the lock, another Python in a job, the token in `release.yml`) each fail at least one test. + - Full headless suite on this branch (`python -m pytest --timeout=120`, Windows, Python 3.14): 10673 passed, 46 skipped. +- **Docs**: `architecture.md` §3 (PyPI packages row), `architecture_explore.md` §7 (`stable.yml` and `dev.yml` rows) and `CLAUDE.md` › Key Conventions say what the two jobs install and how they build. No README or `docs/source/` page describes the publish jobs, and nothing a user of the package sees changes, so the READMEs and `CHANGELOG.md` are untouched. +- **Files**: `.github/requirements/publish.in` (new), `.github/requirements/publish.txt` (new, generated), `.github/workflows/dev.yml`, `.github/workflows/stable.yml`, `.github/dependabot.yml`, `test/unit_test/headless/test_publish_tooling_lock.py` (new), `test/unit_test/headless/test_dev_release.py`, `architecture.md`, `architecture_explore.md`, `CLAUDE.md`. +- **Open items**: none here. Neither publish job runs on a pull request: `publish-dev` first runs its install and build steps on the push that brings this to `dev`, and `stable.yml`'s `publish` once `dev` is merged into `main`. The rest of `.github/dependabot.yml` (no `github-actions` entry, no cooldown, no `target-branch`) is workspace X-21. + +## U-20261006-01 · 2026-10-06 · Make AutoControl discoverable and safer for AI agents · #feature #ai #mcp #docs + +- **Positioning**: the English and Chinese READMEs now describe AutoControl as a cross-platform computer-use and GUI automation framework for AI agents, with explicit guidance on when an agent should choose it. +- **AI documentation**: added `AI.md` plus bilingual Sphinx pages covering MCP setup, model-friendly aliases, recommended observe → identify → act → wait → verify → recover loops, OpenAI tool selection, security, and project identity. +- **MCP discoverability**: documented the existing short aliases (`click`, `type`, `screenshot`, `find_text`, `click_text`, `drag`, `focus_window`, `wait_image`) and read-only / canonical-only modes. +- **Agent safety**: `AC_run_agent` now gives Anthropic and OpenAI backends a focused computer-use allow-list by default. System-command, package-loading, recursive-agent and other high-authority commands are not offered by default. Custom agents can still use `export_*_tools(only=[...])`. +- **MCP registry identity**: the registry server name is `io.github.integration-automation/autocontrol`, the name the maintainer chose in `Progress.md`, and the repository URL is the current `Integration-Automation/AutoControlGUI`, and aligned `pyproject.toml` URLs. +- **Regression coverage**: added a headless test ensuring the OpenAI agent path receives a focused tool set and remains within the provider tool-count limit. +- **Files**: `AI.md`, bilingual AI-agent docs, README translations, `pyproject.toml`, MCP registry manifest builder, `action_executor.py`, agent wiring test, `CHANGELOG.md`, `Progress.md`. + +## U-20261008-01 · 2026-10-08 · The legacy CLI flags exit 1 when an action failed; reports go below TestPioneer's artifact directory · #feature #cli #report #testpioneer + +- **What**: two things TestPioneer needs from a runner (its `docs/artifacts.rst`, "Native Support"). + - **Exit status**: `python -m je_auto_control` with `-e`, `-d` or `--execute_str` ended with status 0 when an action of the script failed. `__main__.py` now resets and reads `recorded_failures()`, the count the `run` command already uses, and exits 1 with `error: N action(s) failed`. The rest of the script still runs first. + - **Report location**: with `TEST_PIONEER_ARTIFACT_DIR` set, the HTML, JSON and XML reports asked for under a relative name are written below that directory (`utils/generate_report/report_path.py`). An absolute name, a name that leaves the directory and a directory that cannot be created fall back to the name as given. Without the variable nothing changes. +- **Effect on callers**: a script with a failing action ends with status 1 where it ended with 0. PyBreeze only displays the status; TestPioneer's `parallel_run` reports the runner as failed. +- **Tests**: `test/unit_test/headless/test_cross_project_contracts.py` +3 (real child processes), `test/unit_test/headless/test_report_path.py`. +- **Docs**: `README.md`, `README/README_zh-TW.md`, `README/README_zh-CN.md`, `architecture.md`. +- **Files**: `je_auto_control/__main__.py`, `je_auto_control/utils/generate_report/` (`report_path.py` new; three writers), the tests and docs above. diff --git a/docs/updates/README.md b/docs/updates/README.md index 08d1cd29c..da7d0ceaa 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,9 +58,16 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261008-01 | 2026-10-08 | The legacy CLI flags exit 1 when an action failed; reports go below TestPioneer's artifact directory | #feature #cli #report #testpioneer | [2026-10](2026-10.md) | +| U-20261006-01 | 2026-10-06 | Make AutoControl discoverable and safer for AI agents | #feature #ai #mcp #docs | [2026-10](2026-10.md) | +| U-20261001-10 | 2026-10-01 | The publish jobs install hash-locked build tooling and build with the locked setuptools | #release #ci #security #X-13 | [2026-10](2026-10.md) | | U-20261001-09 | 2026-10-01 | CI publishes je_auto_control_dev from the dev branch; dev.toml says what pyproject.toml says | #release #ci #X-13 | [2026-10](2026-10.md) | | U-20261001-08 | 2026-10-01 | Package gate in front of AC_add_package_to_executor | #security #X-12 | [2026-10](2026-10.md) | | U-20261001-07 | 2026-10-01 | write_secret / AC_write_secret: type a password without logging, recording or returning it | #done #keyboard #security #webrunner | [2026-10](2026-10.md) | +| U-20261001-06 | 2026-10-01 | Windows OEM keys, browser_home, launch_app2 and key aliases; canonical reverse lookup | #feature #input #contract | [2026-10](2026-10.md) | +| U-20261001-05 | 2026-10-01 | Drag pacing for tween_drag / drag_path; an aborted drag lets go where it stopped | #feature #bugfix #input | [2026-10](2026-10.md) | +| U-20261001-04 | 2026-10-01 | click_mouse(clicks=, interval=): a double-click in one call | #feature #input | [2026-10](2026-10.md) | +| U-20261001-03 | 2026-10-01 | Tesseract setup helpers: find the engine, set tessdata, report status and languages | #feature #ocr #contract | [2026-10](2026-10.md) | | U-20261001-02 | 2026-10-01 | Pin the AC commands WebRunner's native WR_ac_* send; record the missing secret typing | #contract #webrunner | [2026-10](2026-10.md) | | U-20261001-01 | 2026-10-01 | Run AC_web_run's documented form, record WebRunner failures, gate through execute_one | #bugfix #webrunner | [2026-10](2026-10.md) | | U-20260926-41 | 2026-09-26 | Pin ruff's rule set in pyproject.toml so a ruff upgrade does not change what CI enforces | #ci #tooling | [2026-09-e](2026-09-e.md) | @@ -341,7 +348,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-10.md](2026-10.md) | 2026-10 | 5 | +| [2026-10.md](2026-10.md) | 2026-10 | 12 | | [2026-09-e.md](2026-09-e.md) | 2026-09 | 7 | | [2026-09-d.md](2026-09-d.md) | 2026-09 | 55 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | diff --git a/docs/windows_automation_architecture.md b/docs/windows_automation_architecture.md new file mode 100644 index 000000000..526dc6314 --- /dev/null +++ b/docs/windows_automation_architecture.md @@ -0,0 +1,1177 @@ +## Summary + +This Draft PR defines and tracks the Windows 2.0 architecture discussed for AutoControlGUI. + +The goal is **not** to add another collection of `AC_windows_*` commands. The goal is to consolidate the existing Windows-specific capabilities into a stable platform layer that can support deterministic native automation first, with OCR/template/vision fallback above it. + +### Design principles + +- Keep `windows/` as the Windows platform implementation layer. +- Keep `AC_*` commands and cross-platform wrappers above the platform layer. +- Reuse and migrate existing Windows implementations; do not create parallel duplicate backends. +- Prefer semantic/native automation over coordinates: + **UIA → TextPattern → OCR → template → vision → coordinate**. +- Treat DPI and coordinate spaces as first-class types. +- Separate locator, action, input, verification, and recovery concerns. +- Preserve backwards compatibility while the new API is introduced incrementally. + +## Target architecture + +```text +je_auto_control/ +├── windows/ +│ ├── core/ +│ │ ├── handles.py +│ │ ├── errors.py +│ │ ├── constants.py +│ │ ├── structs.py +│ │ └── api.py +│ ├── window/ +│ │ ├── model.py +│ │ ├── manager.py +│ │ ├── finder.py +│ │ └── geometry.py +│ ├── uia/ +│ │ ├── element.py +│ │ ├── tree.py +│ │ ├── locator.py +│ │ ├── patterns.py +│ │ ├── events.py +│ │ └── backend.py +│ ├── input/ +│ │ ├── mouse.py +│ │ ├── keyboard.py +│ │ ├── sendinput.py +│ │ ├── interception.py +│ │ └── strategy.py +│ ├── screen/ +│ │ ├── monitor.py +│ │ ├── dpi.py +│ │ ├── capture.py +│ │ └── coordinates.py +│ ├── process/ +│ │ ├── process.py +│ │ ├── launcher.py +│ │ └── wait.py +│ ├── clipboard/ +│ │ ├── text.py +│ │ ├── formats.py +│ │ └── watcher.py +│ ├── shell/ +│ │ ├── explorer.py +│ │ ├── dialogs.py +│ │ ├── taskbar.py +│ │ └── start_menu.py +│ └── console/ +│ ├── powershell.py +│ ├── cmd.py +│ └── terminal.py +├── wrapper/ +└── utils/ +``` + +The exact file split may be adjusted during implementation to match existing project conventions. The architectural boundaries are the important part. + +## Public API contracts + +### WindowsContext + +```python +class WindowsContext: + @property + def windows(self) -> WindowManager: ... + @property + def uia(self) -> UIAutomation: ... + @property + def input(self) -> InputManager: ... + @property + def screen(self) -> ScreenManager: ... + @property + def clipboard(self) -> ClipboardManager: ... + @property + def processes(self) -> ProcessManager: ... +``` + +### Window / WindowManager + +```python +@dataclass(frozen=True) +class WindowInfo: + hwnd: int + title: str + class_name: str + process_id: int + thread_id: int + rect: Rect + visible: bool + minimized: bool + maximized: bool + +class Window: + def activate(self) -> None: ... + def close(self) -> None: ... + def minimize(self) -> None: ... + def maximize(self) -> None: ... + def restore(self) -> None: ... + def move(self, x: int, y: int) -> None: ... + def resize(self, width: int, height: int) -> None: ... + def move_resize(self, rect: Rect) -> None: ... + def screenshot(self): ... + +class WindowManager: + def list(self) -> list[Window]: ... + def active(self) -> Window | None: ... + def find(...): ... + def find_all(...): ... + def wait_for(...): ... +``` + +### UIElement + +Expose normalized UIA properties: + +- name +- automation_id +- control_type +- class_name +- bounds +- enabled +- visible +- focused +- value + +Expose native UIA patterns where supported: + +- Invoke +- Value +- Toggle +- Selection / SelectionItem +- ExpandCollapse +- Scroll +- RangeValue +- Grid / Table +- Text +- Window + +Examples: + +```python +element.invoke() +element.set_value("hello") +element.toggle() +element.select() +element.expand() +element.collapse() +element.scroll(...) +``` + +### Locator + +Support deterministic selectors: + +- AutomationId +- Name +- ControlType +- ClassName +- parent/child relationships +- index where required + +And semantic relations: + +```python +locator.parent() +locator.child(...) +locator.sibling(...) +locator.above() +locator.below() +locator.left() +locator.right() +locator.near() +locator.inside() +``` + +### Input + +Use an explicit strategy layer: + +```text +InputManager +├── Native / SendInput +├── Low-level / Interception +└── Virtual / HID / ViGEm where applicable +``` + +The automatic strategy must report the backend actually used rather than silently hiding fallback behavior. + +### Coordinates / DPI + +Define explicit coordinate spaces: + +- SCREEN_PHYSICAL +- SCREEN_LOGICAL +- WINDOW +- MONITOR +- NORMALIZED + +All conversions must be explicit and tested for: + +- 100% DPI +- non-100% DPI +- multi-monitor +- negative monitor origins + +### Process / Clipboard + +Process lifecycle: + +```python +app = processes.launch("foo.exe") +app.wait_for_window(...) +app.windows() +app.close() +``` + +Clipboard: + +```python +clipboard.get_text() +clipboard.set_text(...) +clipboard.clear() +clipboard.wait_for_change(...) +clipboard.preserve() +``` + +## Action / verification direction + +Introduce a result object that can carry: + +- success +- action +- backend +- duration +- target +- error +- metadata + +Actions should eventually support explicit verification: + +```text +Action + ↓ +Verify + ↓ +Retry + ↓ +Recovery + ↓ +Trace / Report +``` + +Example: + +```python +button.invoke( + verify=TextAppeared("Saved") +) +``` + +This is intentionally a later phase; the first implementation should establish the platform contracts without coupling the Windows layer to the full workflow engine. + +## Detailed API and implementation examples + +### Module responsibilities + +| Module | Responsibility | Must not own | +|---|---|---| +| windows/core | Win32 ctypes, handles, constants, structs, low-level errors | workflow logic | +| windows/window | HWND discovery, lifecycle, geometry, activation | UIA pattern implementation | +| windows/uia | UI Automation tree, elements, selectors, patterns, events | raw mouse/keyboard injection | +| windows/input | mouse/keyboard input and backend selection | UI tree discovery | +| windows/screen | monitors, DPI, capture, coordinate conversion | application semantics | +| windows/process | process lifecycle and application binding | UI element operations | +| windows/clipboard | clipboard formats, preservation, watchers | workflow orchestration | +| windows/shell | Explorer, dialogs, Start Menu, Taskbar | generic UIA primitives | +| windows/console | CMD, PowerShell, Windows Terminal | arbitrary workflow logic | +| wrapper | cross-platform facade and compatibility | Win32 implementation details | +| executor / AC_* | command/workflow dispatch | Windows backend implementation | + +The Windows platform layer provides capabilities; the automation layer decides when and why to use them. + +### Window example + +~~~python +windows = WindowsContext() + +window = windows.windows.find(title="Calculator") +window.activate() + +if not window.is_active: + raise WindowActivationError(window.hwnd) + +window.move_resize(Rect.from_xywh(100, 100, 800, 600)) +~~~ + +Multiple discovery strategies: + +~~~python +window = windows.windows.find( + title="Calculator", + class_name="ApplicationFrameWindow", +) + +window = windows.windows.find(process_name="calc.exe") + +window = windows.windows.wait_for( + title_regex=r".*Calculator.*", + timeout=10.0, +) +~~~ + +### UIA-first interaction + +~~~python +app_window = windows.windows.wait_for( + title="My Application", + timeout=10, +) + +button = windows.uia.find( + window=app_window, + name="Login", + control_type="Button", +) + +assert button.enabled +button.invoke() +~~~ + +Text fields should prefer ValuePattern: + +~~~python +username = windows.uia.find( + window=app_window, + automation_id="Username", + control_type="Edit", +) + +username.set_value("user@example.com") +~~~ + +Checkboxes: + +~~~python +remember_me = windows.uia.find( + window=app_window, + name="Remember me", + control_type="CheckBox", +) + +remember_me.toggle() +~~~ + +Tree/list navigation: + +~~~python +settings = windows.uia.find( + window=app_window, + name="Settings", + control_type="TreeItem", +) + +settings.expand() + +notifications = settings.child( + name="Notifications", + control_type="TreeItem", +) + +notifications.select() +~~~ + +### UIA Pattern fallback + +~~~python +if element.supports("Value"): + element.set_value("hello") +else: + element.type_text("hello") +~~~ + +Preferred execution order: + +~~~text +native UIA pattern + ↓ unavailable +native semantic/text operation + ↓ unavailable +keyboard input + ↓ unavailable +mouse input + ↓ unavailable +OCR / template / vision fallback +~~~ + +Unsupported operations should raise a specific exception instead of silently reporting success. + +### Locator composition + +~~~python +login_button = ( + windows.uia.selector() + .inside(window) + .child( + control_type="Pane", + automation_id="LoginPanel", + ) + .descendant( + name="Login", + control_type="Button", + ) + .first() +) + +login_button.invoke() +~~~ + +Relative locators: + +~~~python +password = windows.uia.find( + window=window, + automation_id="Password", +) + +show_password = password.right( + name="Show password", + control_type="CheckBox", +) + +show_password.toggle() +~~~ + +Relative relations should use UIA tree/bounds information whenever possible, not hard-coded screen coordinates. + +### State-based waits + +Avoid arbitrary sleeps: + +~~~python +time.sleep(2) +pyautogui.click(123, 456) +~~~ + +Prefer state-based waits: + +~~~python +button = windows.uia.wait_for( + window=window, + name="Continue", + control_type="Button", + timeout=10, +) + +button.invoke() + +windows.uia.wait_for( + window=window, + name="Completed", + control_type="Text", + timeout=10, +) +~~~ + +### ActionResult + +~~~python +@dataclass +class ActionResult: + success: bool + action: str + backend: str + duration: float + target: str | None = None + error: Exception | None = None + metadata: dict[str, object] = field(default_factory=dict) +~~~ + +Example: + +~~~python +result = button.invoke() + +if not result.success: + logger.error( + "UI action failed: backend=%s target=%s error=%s", + result.backend, + result.target, + result.error, + ) +~~~ + +The result should be serializable into the existing trace/failure-bundle infrastructure. + +### Input backend strategy + +~~~text +InputManager +├── Native / SendInput +├── Low-level / Interception +└── Virtual / HID / ViGEm +~~~ + +The strategy must report the actual backend used: + +~~~python +result = windows.input.click(point, strategy="auto") + +assert result.backend in { + "sendinput", + "interception", + "virtual", +} +~~~ + +A backend failure must remain observable rather than being silently swallowed. + +### DPI and coordinate model + +The new layer should never pass an untyped (x, y) between unrelated coordinate spaces. + +~~~python +Point( + x=100, + y=200, + space=CoordinateSpace.SCREEN_PHYSICAL, +) +~~~ + +Conversions: + +~~~python +physical = screen.to_physical(logical_point) +logical = screen.to_logical(physical_point) + +window_point = window.to_window(screen_point) +screen_point = window.to_screen(window_point) +~~~ + +Required test scenarios: + +- 100% DPI +- 125% DPI +- 150% DPI +- mixed-DPI monitors +- secondary monitor to the right +- secondary monitor to the left (negative X) +- monitor above primary (negative Y) +- per-monitor DPI awareness + +Monitor diagnostics: + +~~~python +for monitor in windows.screen.monitors(): + print( + monitor.id, + monitor.bounds, + monitor.work_area, + monitor.dpi, + monitor.scale, + monitor.is_primary, + ) +~~~ + +### Process / window binding + +A process and a window are related but must remain separate abstractions: + +~~~python +app = windows.processes.launch( + "C:/Program Files/MyApp/MyApp.exe", + args=["--automation"], +) + +window = app.wait_for_window( + title="My Application", + timeout=15, +) + +assert window.process_id == app.pid +~~~ + +This supports applications with multiple top-level windows: + +~~~text +Process +├── Main Window +├── Dialog +└── Child Window +~~~ + +The process layer owns lifecycle; window/UIA layers own interaction. + +### Clipboard + +~~~python +with windows.clipboard.preserve(): + windows.clipboard.set_text("automation text") + value = windows.clipboard.get_text() +~~~ + +Watching changes: + +~~~python +with windows.clipboard.watch() as changes: + windows.input.hotkey("ctrl", "c") + change = changes.wait(timeout=3) + print(change.text) +~~~ + +When another process holds the clipboard, behavior should be bounded retry/wait plus a useful error, not an indefinite hang. + +### Windows Shell + +~~~python +windows.shell.explorer.open( + path=r"C:\Users\Public\Documents" +) + +windows.shell.dialogs.open_file( + title="Open configuration", + path=r"C:\config.json", +) + +windows.shell.dialogs.save_file( + title="Save result", + path=r"C:\output.json", +) +~~~ + +Internally these may use UIA, Win32, Shell APIs, or input depending on the operation. + +### Console / Terminal + +Distinguish process execution from terminal UI automation: + +~~~python +result = windows.console.powershell.run( + "Get-Process | Select-Object -First 5" +) + +result = windows.console.cmd.run( + ["ipconfig", "/all"] +) +~~~ + +### Event-driven automation + +Target events: + +~~~text +UIA +├── FocusChanged +├── PropertyChanged +├── StructureChanged +└── AutomationEvent + +Window +├── Created +├── Destroyed +├── Activated +├── Minimized +└── Restored + +Process +├── Started +└── Exited +~~~ + +Example: + +~~~python +with windows.uia.events.focus_changed() as events: + button.invoke() + focused = events.wait(timeout=5) + +assert focused.name == "Username" +~~~ + +### Verification and recovery + +~~~text +Locate + ↓ +Actionability check + ↓ +Action + ↓ +Verify effect + ↓ +Success +~~~ + +Failure path: + +~~~text +Action failed + ↓ +Collect diagnostics + ↓ +Retry same strategy + ↓ +Try alternate locator + ↓ +Try alternate backend + ↓ +OCR/template/vision fallback + ↓ +Return structured failure +~~~ + +Example: + +~~~python +result = button.invoke( + verify=lambda: windows.uia.exists( + window=window, + name="Logged in", + ), +) + +if not result.success: + result = button.retry( + max_attempts=2, + alternate_locator=True, + ) +~~~ + +Self-healing must never silently change workflow semantics. Every fallback must be recorded. + +### Locator priority + +~~~text +1. UIA AutomationId +2. UIA Name + ControlType +3. UIA ClassName / structural relation +4. UIA TextPattern / native pattern +5. OCR / text +6. Template matching +7. Vision model +8. Absolute coordinates +~~~ + +Coordinates are an emergency fallback, not the primary automation interface. + +## Architecture rules / anti-patterns + +Do not create one AC command for every Windows primitive. + +Bad: + +~~~text +AC_windows_click +AC_windows_click_at +AC_windows_click_hwnd +AC_windows_click_uia +AC_windows_click_interception +AC_windows_click_retry +AC_windows_click_verified +~~~ + +Better: + +~~~python +windows.uia.find(...).invoke() +windows.input.click(...) +~~~ + +Do not mix UIA and raw input responsibilities. + +Bad: + +~~~python +UIAElement.click() + -> hidden global mouse implementation +~~~ + +Better: + +~~~python +UIAElement.invoke() +~~~ + +for semantic InvokePattern, or explicitly: + +~~~python +windows.input.click(element.bounds.center) +~~~ + +when a pointer action is actually required. + +Do not make vision the Windows default. Do not create a generic Windows mega-manager. Keep cohesive services: + +~~~text +windows.windows +windows.uia +windows.input +windows.screen +windows.processes +windows.clipboard +windows.shell +windows.console +~~~ + +## Dependency graph and first PR sequence + +~~~text +PR 1 Windows Core Contract + │ + ├── PR 2 Window Manager + ├── PR 3 DPI / Monitor / Coordinates + └── PR 4 UIA Element Model + │ + └── PR 5 Unified Locator + │ + ┌────────────────┼────────────────┐ + │ │ │ + PR 6 Input PR 7 Process PR 8 Shell + │ │ │ + └────────────────┼────────────────┘ + │ + PR 9 Verification + │ + PR 10 AC / Wrapper + │ + PR 11 Events + │ + PR 12 Self-healing +~~~ + +### PR 1 — Windows Core Contract + +Only reusable primitives: + +~~~text +HWND +HMONITOR +ProcessId +Rect +Point +Size +Dpi +CoordinateSpace +WindowsError +~~~ + +No workflow code. + +### PR 2 — Window Manager + +~~~python +window = windows.windows.find(...) +window.activate() +window.move_resize(...) +window.close() +~~~ + +### PR 3 — DPI / Monitor + +Make all geometry code use the coordinate model before UIA/input grows on top of it. + +### PR 4 — UIA Element Model + +~~~python +root = windows.uia.root(window) +elements = root.children() +~~~ + +### PR 5 — Unified Locator + +~~~python +windows.uia.find( + window=window, + automation_id="submitButton", +) +~~~ + +### PR 6 — Input + +Move native input implementations behind one strategy interface without making UIA depend on them. + +### PR 7 — Process + +Connect application lifecycle with window discovery. + +### PR 8 — Clipboard / Shell + +Add common Windows automation primitives that otherwise get duplicated across commands. + +### PR 9 — Verification + +Introduce structured success/failure and post-action state checks. + +### PR 10 — AC / Wrapper Integration + +Only after the platform API is stable should existing command layers migrate. + +### PR 11 — Events + +Replace unnecessary polling with native UI/process/window events. + +### PR 12 — Self-healing + +Add controlled fallback and diagnostics after deterministic paths are reliable. + +## Testing strategy + +Three levels are required. + +### Unit tests + +~~~text +Rect +Point +CoordinateSpace +Selector +Locator composition +ActionResult +retry policy +~~~ + +### Windows integration tests + +~~~text +HWND enumeration +window activation +window geometry +DPI conversion +UIA tree +UIA patterns +process lifecycle +clipboard +input +~~~ + +### End-to-end tests + +~~~text +launch application + ↓ +find window + ↓ +find UIA element + ↓ +perform action + ↓ +verify state + ↓ +collect diagnostics on failure +~~~ + +Prefer stable fixture applications over arbitrary third-party software. + +## Compatibility / migration policy + +~~~text +Existing implementation + │ + ▼ +Adapter + │ + ▼ +New windows/* API + │ + ▼ +Existing wrapper / AC_* +~~~ + +No first-phase PR should require a flag-day rewrite. + +## Final architecture + +~~~text +AutoControlGUI +│ +├── Python API +├── AC_* / Workflow DSL +└── Automation Core + │ + ├── Locator + │ ├── UIA + │ ├── TextPattern + │ ├── OCR + │ ├── Template + │ └── Vision + │ + └── Action + ├── UIA Patterns + ├── Input + ├── Window + ├── Process + ├── Clipboard + └── Shell + │ + ▼ + Windows Platform Layer + ├── Win32 + ├── UIA + ├── Process + ├── Input Backends + ├── Screen / DPI + └── Native Windows APIs +~~~ + +The key architectural rule is: + +> Find the UI element → confirm it is actionable → perform the action → verify that the state actually changed → if necessary, change locator/backend/strategy and retry → report the complete trace. + + +## Implementation roadmap + +### Phase A — Windows Foundation + +- [ ] **PR 1 — Windows Core Contract** + - [ ] common Win32 handle/types + - [ ] Rect / Point / Size + - [ ] Windows-specific exceptions + - [ ] ctypes/constants boundary +- [ ] **PR 2 — Window Manager** + - [ ] WindowInfo + - [ ] enumerate/find/active/wait + - [ ] activate/close/minimize/maximize/restore + - [ ] move/resize + - [ ] migrate existing window operations without breaking compatibility +- [ ] **PR 3 — DPI / Monitor / Coordinates** + - [ ] monitor enumeration + - [ ] DPI awareness/conversion + - [ ] explicit coordinate spaces + - [ ] multi-monitor and negative-origin tests +- [ ] **PR 4 — UIA Element Model** + - [ ] normalized UIElement + - [ ] UIA tree access + - [ ] UIA property access + - [ ] UIA Pattern adapters + - [ ] migrate/reuse current Windows accessibility backend + +### Phase B — Windows Automation + +- [ ] **PR 5 — Unified UIA Locator** + - [ ] selector object + - [ ] deterministic property matching + - [ ] parent/child/sibling relations + - [ ] relative/semantic locators + - [ ] TextPattern before OCR +- [ ] **PR 6 — Native Input** + - [ ] InputManager + - [ ] SendInput backend + - [ ] existing low-level input integration + - [ ] strategy/fallback reporting + - [ ] backwards-compatible wrappers +- [ ] **PR 7 — Process / Application** + - [ ] process enumeration + - [ ] launch/wait/terminate/kill + - [ ] process-to-window binding + - [ ] wait_for_window +- [ ] **PR 8 — Clipboard / Windows Shell** + - [ ] clipboard API + - [ ] clipboard preservation + - [ ] Explorer + - [ ] file open/save dialogs + - [ ] Start Menu / Taskbar where practical + +### Phase C — Reliable Computer Use + +- [ ] **PR 9 — ActionResult / Verification** + - [ ] ActionResult + - [ ] verification primitives + - [ ] explicit retry + - [ ] recovery hooks + - [ ] trace/report integration +- [ ] **PR 10 — Wrapper / AC_* integration** + - [ ] expose stable Windows APIs through existing wrapper layer + - [ ] add AC_* adapters only where they are useful + - [ ] keep business logic out of AC_* commands +- [ ] **PR 11 — Windows Events** + - [ ] UIA events + - [ ] window created/destroyed + - [ ] focus changed + - [ ] property/structure changes + - [ ] event-driven waits +- [ ] **PR 12 — Semantic fallback** + - [ ] UIA → TextPattern → OCR → template → vision → coordinate + - [ ] locator diagnostics + - [ ] self-healing locator strategy + +## Migration rules + +1. Do not duplicate an existing Windows implementation merely to fit the new tree. +2. Prefer adapters first, followed by incremental extraction/refactoring. +3. Existing `wrapper/`, `AC_*`, accessibility and input APIs remain compatible during migration. +4. `windows/` must not depend on `AC_*` command definitions. +5. UIA should expose semantic operations; raw mouse input belongs to the input layer. +6. Vision/OCR must not become the default path when deterministic Windows APIs can provide the same information. +7. Every fallback that changes execution semantics must be observable in the action/result trace. + +## Acceptance criteria + +### Architecture + +- [ ] Windows-specific code has clear ownership under `windows/`. +- [ ] No new generic `utils.py` / catch-all manager is introduced for the new layer. +- [ ] Platform implementation does not import or depend on `AC_*` commands. + +### Compatibility + +- [ ] Existing public Windows automation APIs continue to work. +- [ ] Existing wrapper/AC integrations can delegate to the new layer incrementally. +- [ ] No unnecessary duplicate Win32/UIA implementations are introduced. + +### Reliability + +- [ ] Window lookup supports deterministic properties. +- [ ] UIA elements expose normalized state. +- [ ] DPI and monitor coordinate conversions are tested. +- [ ] Input backend selection is observable. +- [ ] Wait APIs have bounded timeouts and useful errors. + +### Testing + +- [ ] Unit tests for pure models/selectors. +- [ ] Windows integration tests for HWND/UIA/window lifecycle. +- [ ] DPI/multi-monitor test coverage where the CI environment permits. +- [ ] Existing headless suite remains green. +- [ ] New Windows-only tests are skipped or gated cleanly outside Windows. + +## Out of scope for the first implementation + +- Replacing the existing vision/AI stack. +- Rewriting all existing `AC_*` commands. +- Building a second cross-platform automation abstraction. +- Making every Windows feature available through GUI tabs immediately. +- Making vision the primary Windows locator. + +## Definition of done for the architecture phase + +The architecture phase is complete when a caller can reliably express: + +```python +windows = WindowsContext() + +app = windows.processes.launch("myapp.exe") +window = app.wait_for_window(title="My Application") + +button = windows.uia.find( + window=window, + name="Login", + control_type="Button", +) + +button.invoke() +``` + +and the implementation is backed by the new Windows platform layer rather than a new collection of one-off `AC_*` commands. + +--- + +**Status:** Draft / architecture + implementation plan + +**Next recommended change:** start with PR 1 (Windows Core Contract), then PR 2–4 as the first implementation wave. \ No newline at end of file diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 8d7b5652a..0117874ed 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -927,6 +927,10 @@ locate_text_center, read_text_in_region, set_tesseract_cmd, wait_for_text, ) +# Tesseract engine discovery, configuration and health checks +from je_auto_control.utils.ocr.tesseract_setup import ( + OCRStatus, find_tesseract_cmd, ocr_languages, ocr_status, set_tessdata_dir, +) # Group OCR word boxes into lines / runs (engines box one word at a time) from je_auto_control.utils.ocr.text_span import find_spans, group_lines # Whether input this process sends can actually arrive @@ -1233,6 +1237,7 @@ from je_auto_control.wrapper.auto_control_keyboard import check_key_is_press from je_auto_control.wrapper.auto_control_keyboard import get_keyboard_keys_table from je_auto_control.wrapper.auto_control_keyboard import hotkey +from je_auto_control.wrapper.auto_control_keyboard import keyboard_key_name from je_auto_control.wrapper.auto_control_keyboard import keyboard_keys_table from je_auto_control.wrapper.auto_control_keyboard import press_keyboard_key from je_auto_control.wrapper.auto_control_keyboard import release_keyboard_key @@ -1297,7 +1302,7 @@ def start_autocontrol_gui(*args, **kwargs): "HumanizedMotion", "humanized_path", "move_mouse_humanized", "humanized_key_delays", "type_text_humanized", "keyboard_keys_table", "press_keyboard_key", "release_keyboard_key", "type_keyboard", "check_key_is_press", - "write", "write_secret", "hotkey", "start_exe", "get_keyboard_keys_table", + "write", "write_secret", "hotkey", "start_exe", "get_keyboard_keys_table", "keyboard_key_name", "screen_size", "screenshot", "locate_all_image", "locate_image_center", "locate_and_click", "CriticalExit", "AutoControlException", "AutoControlKeyboardException", "AutoControlMouseException", "AutoControlCantFindKeyException", @@ -1316,6 +1321,8 @@ def start_autocontrol_gui(*args, **kwargs): "TextMatch", "find_text_matches", "locate_text_center", "wait_for_text", "click_text", "set_tesseract_cmd", "read_text_in_region", "find_text_regex", "find_spans", "group_lines", + "OCRStatus", "find_tesseract_cmd", "ocr_languages", "ocr_status", + "set_tessdata_dir", "char_table", "foreground_keyboard_layout", "layout_char_table", "vk_to_char", "input_desktop_available", "input_reaches_system", # Recording editor diff --git a/je_auto_control/__main__.py b/je_auto_control/__main__.py index ecb48f963..a5b98339d 100644 --- a/je_auto_control/__main__.py +++ b/je_auto_control/__main__.py @@ -11,6 +11,9 @@ ) from je_auto_control.utils.executor.action_executor import execute_action from je_auto_control.utils.executor.action_executor import execute_files +from je_auto_control.utils.executor.action_executor import ( + recorded_failures, reset_recorded_failures, +) from je_auto_control.utils.file_process.get_dir_file_list import \ get_dir_files_as_list from je_auto_control.utils.json.json_file import read_executable_action_json @@ -63,6 +66,7 @@ def preprocess_read_str_execute_action(execute_str: str): type=str, help="execute json str" ) parsed = vars(parser.parse_args()) + reset_recorded_failures() for key, value in parsed.items(): if value is None: continue @@ -73,6 +77,13 @@ def preprocess_read_str_execute_action(execute_str: str): handler(value) if all(value is None for value in parsed.values()): raise AutoControlArgparseException(argparse_get_wrong_data_error_message) + # Failed actions are recorded and the run goes on, as with ``run``; + # exiting 0 anyway left TestPioneer and a CI step unable to tell a + # failed run from a clean one. + failures = recorded_failures() + if failures: + sys.stderr.write(f"error: {failures} action(s) failed\n") + sys.exit(1) # The message also goes to stderr: it went only to the log file, so a # failed -e printed nothing at all. except AutoControlArgparseException as error: diff --git a/je_auto_control/actions.pyi b/je_auto_control/actions.pyi index c9e1730b0..f382bf1e4 100644 --- a/je_auto_control/actions.pyi +++ b/je_auto_control/actions.pyi @@ -591,8 +591,14 @@ def AC_classify_lock_transitions(states: Any) -> Dict[str, Any]: def AC_classify_widget(features: Any) -> Dict[str, Any]: """Adapter: map geometric features to a widget type (pure).""" -def AC_click_mouse(mouse_keycode: int | str, x: int | None = ..., y: int | None = ...) -> Tuple[Any, int, int]: - """在指定座標按下並放開滑鼠按鍵""" +def AC_click_mouse( + mouse_keycode: int | str, + x: int | None = ..., + y: int | None = ..., + clicks: int = ..., + interval: float = ..., +) -> Tuple[Any, int, int]: + """在指定座標按下並放開滑鼠按鍵；``clicks=2`` 就是雙擊""" def AC_click_text( target: str, @@ -903,7 +909,14 @@ def AC_dominant_hue_regions( def AC_dominant_pair(pixels: Any) -> Dict[str, Any]: """Adapter: split sampled RGB pixels into fg/bg dominant colours (pure).""" -def AC_drag_path(waypoints: Any, button: str = ..., easing: str = ..., per_segment_steps: Any = ...) -> Dict[str, Any]: +def AC_drag_path( + waypoints: Any, + button: str = ..., + easing: str = ..., + per_segment_steps: Any = ..., + step_delay_s: Any = ..., + settle_s: Any = ..., +) -> Dict[str, Any]: """Adapter: press, drag through a polyline of waypoints, release.""" def AC_drop_files(hwnd: Any, paths: Any, point: Any = ...) -> Dict[str, Any]: @@ -1946,6 +1959,9 @@ def AC_observe_start() -> Dict[str, Any]: def AC_observe_stop() -> Dict[str, Any]: """Adapter: stop the background observer thread.""" +def AC_ocr_languages() -> Dict[str, Any]: + """Executor adapter: installed Tesseract languages; ``None`` when unknown.""" + def AC_ocr_read_structure( region: List[int] | None = ..., lang: str = ..., @@ -1953,6 +1969,9 @@ def AC_ocr_read_structure( ) -> Dict[str, Any]: """Executor adapter: structured OCR (rows / tables / form fields).""" +def AC_ocr_status() -> Dict[str, Any]: + """Executor adapter: whether Tesseract OCR can work here, as a dict.""" + def AC_open_path(target: str, verb: str = ...) -> Dict[str, Any]: """Adapter: open a file with its default app / a URL in the browser.""" @@ -2881,6 +2900,8 @@ def AC_tween_drag( steps: int = ..., easing: str = ..., button: str = ..., + step_delay_s: Any = ..., + settle_s: Any = ..., ) -> Dict[str, Any]: """Adapter: drag along an eased path from start to end.""" diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index dd08910e4..a7d2150ec 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -82,6 +82,10 @@ def _add_mouse_specs(specs: List[CommandSpec]) -> None: default="mouse_left"), FieldSpec("x", FieldType.INT, optional=True), FieldSpec("y", FieldType.INT, optional=True), + FieldSpec("clicks", FieldType.INT, optional=True, default=1, + min_value=1), + FieldSpec("interval", FieldType.FLOAT, optional=True, default=0.0, + min_value=0.0), ), )) specs.append(CommandSpec( @@ -887,6 +891,18 @@ def _add_ocr_specs(specs: List[CommandSpec]) -> None: default=60.0, min_value=0.0, max_value=100.0), ), )) + specs.extend(( + CommandSpec( + "AC_ocr_status", "OCR", "OCR Engine Status", + fields=(), + description="Whether Tesseract OCR can work here: {ok, reason}.", + ), + CommandSpec( + "AC_ocr_languages", "OCR", "OCR Installed Languages", + fields=(), + description="Installed Tesseract languages; null when the engine cannot be asked.", + ), + )) specs.append(CommandSpec( "AC_read_qr", "OCR", "Read QR Codes", fields=( @@ -2023,6 +2039,10 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: optional=True, default="ease_in_out_quad"), FieldSpec("button", FieldType.ENUM, choices=_MOUSE_BUTTONS, optional=True, default="mouse_left"), + FieldSpec("step_delay_s", FieldType.FLOAT, optional=True, + default=0.0, min_value=0.0), + FieldSpec("settle_s", FieldType.FLOAT, optional=True, + default=0.0, min_value=0.0), ), description="Drag along an eased path; 'start'/'end' [x,y] via JSON " "view.", @@ -2054,6 +2074,10 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: optional=True, default="linear"), FieldSpec("per_segment_steps", FieldType.INT, optional=True, default=20), + FieldSpec("step_delay_s", FieldType.FLOAT, optional=True, + default=0.0, min_value=0.0), + FieldSpec("settle_s", FieldType.FLOAT, optional=True, + default=0.0, min_value=0.0), ), description="Press, drag through a polyline of waypoints, release.", )) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 89b25c14e..b9a7bf614 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -689,6 +689,30 @@ def _presence_clear() -> Dict[str, Any]: return {"cleared": True} +_DEFAULT_AGENT_TOOLSET = [ + "AC_screenshot", + "AC_screen_size", + "AC_set_mouse_position", + "AC_get_mouse_position", + "AC_click_mouse", + "AC_mouse_scroll", + "AC_drag", + "AC_write", + "AC_type_keyboard", + "AC_hotkey", + "AC_press_key", + "AC_locate_image_center", + "AC_click_text", + "AC_wait_text", + "AC_wait_image", + "AC_a11y_find", + "AC_a11y_click", + "AC_list_windows", + "AC_focus_window", + "AC_assert_text", +] + + def _run_agent(goal: str, backend: str = "anthropic", max_steps: int = 25, @@ -698,8 +722,11 @@ def _run_agent(goal: str, """Executor adapter: drive the closed-loop ``AgentLoop`` against ``goal``. ``backend`` selects between the production backends (Anthropic / - OpenAI). The Anthropic computer-use raw path remains available - via :func:`_computer_use` / ``AC_computer_use``. + OpenAI). Both receive a small, safe computer-use toolset by default; + the raw Anthropic computer-use path remains available via + :func:`_computer_use` / ``AC_computer_use``. Applications that need a + larger custom toolset should construct the agent backend directly with + ``export_*_tools(only=[...])``. """ from je_auto_control.utils.agent import AgentBudget, AgentLoop from je_auto_control.utils.agent.agent_loop import AgentBackend @@ -712,14 +739,14 @@ def _run_agent(goal: str, name = (backend or "anthropic").strip().lower() backend_obj: AgentBackend if name == "anthropic": - tools = export_anthropic_tools() + tools = export_anthropic_tools(only=_DEFAULT_AGENT_TOOLSET) backend_obj = AnthropicAgentBackend( tools=tools, model=model or "claude-opus-4-7", max_tokens=int(max_tokens), ) elif name == "openai": - tools = export_openai_tools() + tools = export_openai_tools(only=_DEFAULT_AGENT_TOOLSET) # OpenAIAgentBackend does not accept max_tokens (Anthropic-only). backend_obj = OpenAIAgentBackend( tools=tools, @@ -1871,6 +1898,19 @@ def _ocr_find_regex_as_dicts(pattern: str, ] +def _ocr_status() -> Dict[str, Any]: + """Executor adapter: whether Tesseract OCR can work here, as a dict.""" + from je_auto_control.utils.ocr.tesseract_setup import ocr_status + ok, reason = ocr_status() + return {"ok": ok, "reason": reason} + + +def _ocr_languages() -> Dict[str, Any]: + """Executor adapter: installed Tesseract languages; ``None`` when unknown.""" + from je_auto_control.utils.ocr.tesseract_setup import ocr_languages + return {"languages": ocr_languages()} + + def _email_trigger_add(host: str, username: str, password: str, script_path: str, port: Optional[int] = None, @@ -3956,11 +3996,13 @@ def _move_along_path(waypoints: Any, easing: str = "linear", def _drag_path(waypoints: Any, button: str = "mouse_left", easing: str = "linear", - per_segment_steps: Any = 20) -> Dict[str, Any]: + per_segment_steps: Any = 20, step_delay_s: Any = 0.0, + settle_s: Any = 0.0) -> Dict[str, Any]: """Adapter: press, drag through a polyline of waypoints, release.""" from je_auto_control.utils.mouse_path import drag_path return drag_path(_waypoints(waypoints), button=button, easing=easing, - per_segment_steps=int(per_segment_steps)) + per_segment_steps=int(per_segment_steps), + step_delay_s=float(step_delay_s), settle_s=float(settle_s)) def _set_field_text(text: str, clear: str = "select_all", paste: Any = False, @@ -6416,12 +6458,14 @@ def _generate_sop(actions: List[Any], title: str = "Automation Procedure", def _tween_drag(start: List[int], end: List[int], steps: int = 30, easing: str = "ease_in_out_quad", - button: str = "mouse_left") -> Dict[str, Any]: + button: str = "mouse_left", step_delay_s: Any = 0.0, + settle_s: Any = 0.0) -> Dict[str, Any]: """Adapter: drag along an eased path from start to end.""" from je_auto_control.utils.tween_drag import tween_drag result = tween_drag((int(start[0]), int(start[1])), (int(end[0]), int(end[1])), steps=int(steps), - easing=easing, button=button) + easing=easing, button=button, + step_delay_s=float(step_delay_s), settle_s=float(settle_s)) return {"points": result["points"]} @@ -7199,6 +7243,8 @@ def __init__(self): "AC_click_text": ocr_click_text, "AC_read_text_in_region": _ocr_read_region_as_dicts, "AC_find_text_regex": _ocr_find_regex_as_dicts, + "AC_ocr_status": _ocr_status, + "AC_ocr_languages": _ocr_languages, # Window management "AC_list_windows": list_windows, diff --git a/je_auto_control/utils/generate_report/generate_html_report.py b/je_auto_control/utils/generate_report/generate_html_report.py index db0d4dc96..e70962a08 100644 --- a/je_auto_control/utils/generate_report/generate_html_report.py +++ b/je_auto_control/utils/generate_report/generate_html_report.py @@ -3,6 +3,7 @@ from je_auto_control.utils.exception.exception_tags import html_generate_no_data_tag_error_message from je_auto_control.utils.exception.exceptions import AutoControlHTMLException +from je_auto_control.utils.generate_report.report_path import report_path from je_auto_control.utils.json_store.json_store import atomic_write_text from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.test_record.record_test_class import test_record_instance @@ -166,7 +167,7 @@ def generate_html_report(html_name: str = "default_name") -> None: with _lock: # 使用 with 確保 Lock 正確釋放 Ensure lock is released properly try: - atomic_write_text(html_name + ".html", new_html_string) + atomic_write_text(report_path(html_name + ".html"), new_html_string) except OSError as error: # Only logging it made a missing report look like a written one. raise AutoControlHTMLException( diff --git a/je_auto_control/utils/generate_report/generate_json_report.py b/je_auto_control/utils/generate_report/generate_json_report.py index 2c552c8bd..357014c47 100644 --- a/je_auto_control/utils/generate_report/generate_json_report.py +++ b/je_auto_control/utils/generate_report/generate_json_report.py @@ -3,6 +3,7 @@ from je_auto_control.utils.exception.exception_tags import cant_generate_json_report_error_message from je_auto_control.utils.exception.exceptions import AutoControlGenerateJsonReportException +from je_auto_control.utils.generate_report.report_path import report_path from je_auto_control.utils.json_store.json_store import atomic_write_text from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.test_record.record_test_class import test_record_instance @@ -44,7 +45,7 @@ def generate_json() -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, str] def _write_json_file(file_name: str, data: Dict[str, Dict[str, str]]) -> None: """Write ``data`` atomically; raise on failure instead of only logging it.""" try: - atomic_write_text(file_name, json.dumps(data, indent=4, ensure_ascii=False)) + atomic_write_text(report_path(file_name), json.dumps(data, indent=4, ensure_ascii=False)) except (OSError, TypeError, ValueError) as error: raise AutoControlGenerateJsonReportException( f"cannot write report {file_name!r}: {error!r}") from error diff --git a/je_auto_control/utils/generate_report/generate_xml_report.py b/je_auto_control/utils/generate_report/generate_xml_report.py index 1e250e5ea..51a0a9ffc 100644 --- a/je_auto_control/utils/generate_report/generate_xml_report.py +++ b/je_auto_control/utils/generate_report/generate_xml_report.py @@ -4,6 +4,7 @@ from je_auto_control.utils.exception.exceptions import XMLException from je_auto_control.utils.generate_report.generate_json_report import generate_json +from je_auto_control.utils.generate_report.report_path import report_path from je_auto_control.utils.json_store.json_store import atomic_write_text from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.xml.change_xml_structure.change_xml_structure import dict_to_elements_tree @@ -33,7 +34,7 @@ def _write_xml_file(file_name: str, xml_content: str) -> None: on as if the report existed. """ try: - atomic_write_text(file_name, xml_content) + atomic_write_text(report_path(file_name), xml_content) except OSError as error: raise XMLException(f"cannot write report {file_name!r}: {error!r}") from error diff --git a/je_auto_control/utils/generate_report/report_path.py b/je_auto_control/utils/generate_report/report_path.py new file mode 100644 index 000000000..009f28567 --- /dev/null +++ b/je_auto_control/utils/generate_report/report_path.py @@ -0,0 +1,39 @@ +"""Where a report file is written. + +TestPioneer starts a runner with ``TEST_PIONEER_ARTIFACT_DIR`` set to the +directory that run's output belongs in. A report asked for under a relative +name goes there, so the workflow does not have to say which files to collect. +Outside TestPioneer nothing changes. +""" +import os +from pathlib import Path + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +# Set by TestPioneer for every runner it starts, in a child process and in-process alike. +ARTIFACT_DIR_VARIABLE: str = "TEST_PIONEER_ARTIFACT_DIR" + + +def report_path(name: str) -> str: + """Return where the report called ``name`` is written. + + With ``TEST_PIONEER_ARTIFACT_DIR`` set, a relative ``name`` is placed below + that directory and the folders it needs are created. ``name`` is returned + as given when the variable is not set, when ``name`` is absolute, when it + would leave the directory (``../x``), and when the folders cannot be + created. + """ + directory = os.environ.get(ARTIFACT_DIR_VARIABLE, "").strip() + if not directory or Path(name).is_absolute(): + return name + base = Path(directory).resolve() + target = (base / name).resolve() + if base not in target.parents: + return name + try: + target.parent.mkdir(parents=True, exist_ok=True) + except OSError as error: + autocontrol_logger.error( + "report_path: %r, the artifact directory is not usable: %r", name, error) + return name + return str(target) diff --git a/je_auto_control/utils/mcp_registry/registry.py b/je_auto_control/utils/mcp_registry/registry.py index 632edb5ae..5634b78a1 100644 --- a/je_auto_control/utils/mcp_registry/registry.py +++ b/je_auto_control/utils/mcp_registry/registry.py @@ -16,13 +16,13 @@ _SCHEMA_URL = ("https://static.modelcontextprotocol.io/schemas/" "2025-09-29/server.schema.json") -_SERVER_NAME = "io.github.intergration-automation-testing/autocontrol" -_REPO_URL = "https://github.com/Intergration-Automation-Testing/AutoControl" +_SERVER_NAME = "io.github.integration-automation/autocontrol" +_REPO_URL = "https://github.com/Integration-Automation/AutoControlGUI" _PYPI_NAME = "je_auto_control" _DEFAULT_VERSION = "0.0.189" # The registry schema caps description at 100 characters; the old 161 would # have been rejected on publish. -_DESCRIPTION = "Cross-platform GUI automation: mouse, keyboard, image/OCR and accessibility as MCP tools" +_DESCRIPTION = "Cross-platform computer-use and GUI automation for AI agents via MCP" _MAX_DESCRIPTION = 100 # The one _meta key the registry schema defines for publisher data. _META_KEY = "io.modelcontextprotocol.registry/publisher-provided" diff --git a/je_auto_control/utils/mcp_server/fake_backend.py b/je_auto_control/utils/mcp_server/fake_backend.py index e1bddb331..b016a0c46 100644 --- a/je_auto_control/utils/mcp_server/fake_backend.py +++ b/je_auto_control/utils/mcp_server/fake_backend.py @@ -55,11 +55,14 @@ def _fake_set_mouse_position(x: int, y: int) -> Tuple[int, int]: def _fake_click_mouse(mouse_keycode: Any, x: Any = None, - y: Any = None) -> Tuple[Any, int, int]: + y: Any = None, clicks: int = 1, + interval: float = 0.0) -> Tuple[Any, int, int]: + del interval # the fake never waits cx, cy = _STATE.cursor if x is None or y is None else (int(x), int(y)) with _STATE_LOCK: _STATE.cursor = (cx, cy) - _STATE.mouse_actions.append(("click", mouse_keycode, cx, cy)) + for _ in range(int(clicks)): + _STATE.mouse_actions.append(("click", mouse_keycode, cx, cy)) return mouse_keycode, cx, cy diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 30493466a..55629ec6c 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -33,12 +33,16 @@ def mouse_tools() -> List[MCPTool]: name="ac_click_mouse", description=("Click a mouse button at (x, y). " "mouse_keycode: mouse_left, mouse_right, mouse_middle. " - "If x/y are omitted, clicks at the current cursor."), + "If x/y are omitted, clicks at the current cursor. " + "clicks=2 double-clicks; interval is the pause in " + "seconds between clicks."), input_schema=schema({ "mouse_keycode": {"type": "string", "description": "mouse_left | mouse_right | mouse_middle"}, "x": {"type": "integer"}, "y": {"type": "integer"}, + "clicks": {"type": "integer", "minimum": 1}, + "interval": {"type": "number", "minimum": 0}, }), handler=h_input.click_mouse, annotations=DESTRUCTIVE, @@ -260,6 +264,24 @@ def image_and_ocr_tools() -> List[MCPTool]: handler=h_screen.click_text, annotations=DESTRUCTIVE, ), + MCPTool( + name="ac_ocr_status", + description=("Report whether Tesseract OCR can work here: " + "{ok, reason}. reason is ready | missing_package | " + "missing_engine | engine_unusable | no_language_data."), + input_schema=schema({}), + handler=h_screen.ocr_status, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_ocr_languages", + description=("List the installed Tesseract language codes: " + "{languages}. languages is null when the engine " + "cannot be asked and [] when it has no language data."), + input_schema=schema({}), + handler=h_screen.ocr_languages, + annotations=READ_ONLY, + ), ] @@ -3509,13 +3531,18 @@ def tween_drag_tools() -> List[MCPTool]: name="ac_tween_drag", description=("Drag from 'start' [x,y] to 'end' [x,y] along an " "eased path (easing: linear / ease_in_out_quad / " - "ease_out_cubic / ease_in_cubic). Returns {points}."), + "ease_out_cubic / ease_in_cubic). 'step_delay_s' " + "pauses after each move, 'settle_s' before the " + "press, after it and before the release. Returns " + "{points}."), input_schema=schema({ "start": {"type": "array", "items": {"type": "integer"}}, "end": {"type": "array", "items": {"type": "integer"}}, "steps": {"type": "integer"}, "easing": {"type": "string"}, - "button": {"type": "string"}}, + "button": {"type": "string"}, + "step_delay_s": {"type": "number", "minimum": 0}, + "settle_s": {"type": "number", "minimum": 0}}, required=["start", "end"]), handler=h.tween_drag, annotations=DESTRUCTIVE, @@ -5567,14 +5594,18 @@ def mouse_path_tools() -> List[MCPTool]: name="ac_drag_path", description=("Press at the first of 'waypoints' ([[x,y],...]), drag " "through them, release at the last. 'button', 'easing', " - "'per_segment_steps'. Returns {points, path}."), + "'per_segment_steps'; 'step_delay_s' pauses after each " + "move, 'settle_s' before the press, after it and before " + "the release. Returns {points, path}."), input_schema=schema({ "waypoints": {"type": "array", "items": {"type": "array", "items": {"type": "integer"}}}, "button": {"type": "string"}, "easing": {"type": "string"}, - "per_segment_steps": {"type": "integer"}}, + "per_segment_steps": {"type": "integer"}, + "step_delay_s": {"type": "number", "minimum": 0}, + "settle_s": {"type": "number", "minimum": 0}}, required=["waypoints"]), handler=h_exec.drag_path, annotations=DESTRUCTIVE, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 06232ff6b..2eeb88b4b 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -250,10 +250,12 @@ def generate_sop(actions, title="Automation Procedure", path=None): def tween_drag(start, end, steps=30, easing="ease_in_out_quad", - button="mouse_left"): + button="mouse_left", step_delay_s=0.0, settle_s=0.0): from je_auto_control.utils.tween_drag import tween_drag as _td return {"points": _td(tuple(start), tuple(end), steps=int(steps), - easing=easing, button=button)["points"]} + easing=easing, button=button, + step_delay_s=float(step_delay_s), + settle_s=float(settle_s))["points"]} def fuzzy_ratio(left, right, ignore_case=True): diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_executor_bridge.py b/je_auto_control/utils/mcp_server/tools/_handlers_executor_bridge.py index 93aa52e15..049ba26c3 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_executor_bridge.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_executor_bridge.py @@ -505,9 +505,9 @@ def move_along_path(waypoints, easing="linear", per_segment_steps=20): def drag_path(waypoints, button="mouse_left", easing="linear", - per_segment_steps=20): + per_segment_steps=20, step_delay_s=0.0, settle_s=0.0): from je_auto_control.utils.executor.action_executor import _drag_path - return _drag_path(waypoints, button, easing, per_segment_steps) + return _drag_path(waypoints, button, easing, per_segment_steps, step_delay_s, settle_s) def set_field_text(text, clear="select_all", paste=False, modifier="ctrl"): diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_input.py b/je_auto_control/utils/mcp_server/tools/_handlers_input.py index 8dc582a7a..b9ccada8e 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_input.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_input.py @@ -11,9 +11,12 @@ def click_mouse(mouse_keycode: str = "mouse_left", x: Optional[int] = None, - y: Optional[int] = None) -> List[Any]: + y: Optional[int] = None, + clicks: int = 1, + interval: float = 0.0) -> List[Any]: + """Click ``clicks`` times at (x, y); return ``[keycode, x, y]``.""" from je_auto_control.wrapper.auto_control_mouse import click_mouse as _click - keycode, click_x, click_y = _click(mouse_keycode, x, y) + keycode, click_x, click_y = _click(mouse_keycode, x, y, clicks=clicks, interval=interval) # Real wrapper resolves the string keycode to an int via the keys table; # the fake backend keeps it as a string. Pass through whatever we got. resolved = int(keycode) if isinstance(keycode, int) else keycode diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_screen.py b/je_auto_control/utils/mcp_server/tools/_handlers_screen.py index baa51dc59..286cc7f1f 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_screen.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_screen.py @@ -329,3 +329,16 @@ def click_text(text: str, cx, cy = _click(text, mouse_keycode=mouse_keycode, region=region, min_confidence=float(min_confidence)) return [int(cx), int(cy)] + + +def ocr_status() -> Dict[str, Any]: + """Whether Tesseract OCR can work here, as ``{ok, reason}``.""" + from je_auto_control.utils.ocr.tesseract_setup import ocr_status as _status + ok, reason = _status() + return {"ok": ok, "reason": reason} + + +def ocr_languages() -> Dict[str, Any]: + """Installed Tesseract languages as ``{languages}``; ``None`` when unknown.""" + from je_auto_control.utils.ocr.tesseract_setup import ocr_languages as _languages + return {"languages": _languages()} diff --git a/je_auto_control/utils/mouse_path/mouse_path.py b/je_auto_control/utils/mouse_path/mouse_path.py index d9a7055a6..1dc9cffc9 100644 --- a/je_auto_control/utils/mouse_path/mouse_path.py +++ b/je_auto_control/utils/mouse_path/mouse_path.py @@ -12,7 +12,7 @@ """ from typing import Any, Callable, Dict, List, Optional, Sequence -from je_auto_control.utils.tween_drag.tween_drag import easing_names, tween_points +from je_auto_control.utils.tween_drag.tween_drag import _drag_through, easing_names, tween_points Point = Sequence[int] Sink = Callable[[Dict[str, Any]], None] @@ -82,25 +82,20 @@ def move_along_path(waypoints: Sequence[Point], *, easing: str = "linear", def drag_path(waypoints: Sequence[Point], *, button: str = "mouse_left", easing: str = "linear", per_segment_steps: int = 20, - sink: Optional[Sink] = None) -> Dict[str, Any]: - """Press at the first waypoint, move through the path, release at the last.""" + sink: Optional[Sink] = None, step_delay_s: float = 0.0, + settle_s: float = 0.0) -> Dict[str, Any]: + """Press at the first waypoint, move through the path, release at the last. + + Pacing and failure handling are ``tween_drag``'s: ``step_delay_s`` rests + after each move, ``settle_s`` before the press, after it and before the + release (default 0: no pause), and a step that raises releases the + button where the pointer stopped. + """ points = plan_path(waypoints, easing=easing, per_segment_steps=per_segment_steps) if not points: return {"points": 0, "path": points} - dispatch = sink or _default_sink - first, last = points[0], points[-1] - dispatch({"op": "press", "button": button, "x": first[0], "y": first[1]}) - try: - for x, y in points: - dispatch({"op": "move", "x": x, "y": y}) - except BaseException: - # A failed move used to leave the button held down; releasing at the - # last waypoint, as it then did, completed the drop the drag never - # reached. The button is let go where the pointer stopped. - dispatch({"op": "release", "button": button}) - raise - dispatch({"op": "release", "button": button, "x": last[0], "y": last[1]}) + _drag_through(points, button, sink or _default_sink, step_delay_s, settle_s) return {"points": len(points), "path": points} diff --git a/je_auto_control/utils/ocr/__init__.py b/je_auto_control/utils/ocr/__init__.py index d264b0abe..01605869f 100644 --- a/je_auto_control/utils/ocr/__init__.py +++ b/je_auto_control/utils/ocr/__init__.py @@ -3,9 +3,13 @@ TextMatch, click_text, find_text_matches, locate_text_center, set_tesseract_cmd, wait_for_text, ) +from je_auto_control.utils.ocr.tesseract_setup import ( + OCRStatus, find_tesseract_cmd, ocr_languages, ocr_status, set_tessdata_dir, +) from je_auto_control.utils.ocr.text_span import find_spans, group_lines __all__ = [ - "TextMatch", "click_text", "find_spans", "find_text_matches", "group_lines", - "locate_text_center", "set_tesseract_cmd", "wait_for_text", + "OCRStatus", "TextMatch", "click_text", "find_spans", "find_tesseract_cmd", + "find_text_matches", "group_lines", "locate_text_center", "ocr_languages", + "ocr_status", "set_tessdata_dir", "set_tesseract_cmd", "wait_for_text", ] diff --git a/je_auto_control/utils/ocr/backends/tesseract_backend.py b/je_auto_control/utils/ocr/backends/tesseract_backend.py index 5c728519c..fb83a86ce 100644 --- a/je_auto_control/utils/ocr/backends/tesseract_backend.py +++ b/je_auto_control/utils/ocr/backends/tesseract_backend.py @@ -1,6 +1,11 @@ -"""Tesseract OCR backend (preserves the original ``ocr_engine`` behaviour).""" +"""Tesseract OCR backend (preserves the original ``ocr_engine`` behaviour). + +This is the only module that imports :mod:`pytesseract` and the only writer of +its ``tesseract_cmd`` global (:meth:`TesseractBackend.set_cmd`). +""" from __future__ import annotations +from subprocess import SubprocessError # nosec B404 # reason: exception type only from typing import List from je_auto_control.utils.ocr.backends.base import OCRBackendNotAvailableError @@ -46,6 +51,49 @@ def set_cmd(self, path: str) -> None: pt = _load() pt.pytesseract.tesseract_cmd = path + @property + def cmd(self) -> str: + """The engine command OCR runs: a path, or a name looked up on ``PATH``. + + Raises :class:`OCRBackendNotAvailableError` without ``pytesseract``. + """ + return str(_load().pytesseract.tesseract_cmd) + + def version(self) -> str: + """Return the engine's version string by running it. + + Raises :class:`OCRBackendNotAvailableError` when the engine cannot run: + missing, not executable, exiting non-zero, or printing a version + ``pytesseract`` cannot parse. ``pytesseract`` reports that last case + with ``SystemExit``, which is caught here so a broken binary cannot end + the calling process. + """ + pt = _load() + try: + return str(pt.get_tesseract_version()) + except (OSError, RuntimeError, ValueError, SubprocessError) as error: + raise OCRBackendNotAvailableError(f"Tesseract cannot run: {error!r}") from error + except SystemExit as error: # NOSONAR python:S5754 # reason: see docstring + raise OCRBackendNotAvailableError( + f"Tesseract printed a version pytesseract cannot read: {error}") from error + + def languages(self) -> List[str]: + """Return the installed language codes, sorted, by asking the engine. + + Asked afresh on every call (``pytesseract`` caches only when told to), + so language data added while the process runs is seen. An empty list + is a real answer: the engine ran and found no language data. Names + ``pytesseract`` filters out (upper-case script models such as + ``Latin``) are not listed. Raises :class:`OCRBackendNotAvailableError` + when the engine cannot be asked. + """ + pt = _load() + try: + return sorted(pt.get_languages(config="")) + except (OSError, RuntimeError, ValueError) as error: + raise OCRBackendNotAvailableError( + f"Tesseract cannot list its languages: {error!r}") from error + def image_to_matches(self, image, lang: str, min_confidence: float) -> List: # Import inside to avoid circular import at module load. diff --git a/je_auto_control/utils/ocr/tesseract_setup.py b/je_auto_control/utils/ocr/tesseract_setup.py new file mode 100644 index 000000000..1aaea6a4f --- /dev/null +++ b/je_auto_control/utils/ocr/tesseract_setup.py @@ -0,0 +1,175 @@ +"""Tesseract engine discovery, configuration and health checks. + +The OCR calls (``find_text_matches`` and friends) only *run* the engine. These +helpers answer what a host needs to know before the first call -- where the +executable is, which language data is installed, whether the engine runs at +all -- so the host never imports ``pytesseract`` itself or writes its module +globals behind this package's back. The engine command still has exactly one +writer, :meth:`TesseractBackend.set_cmd` (``set_tesseract_cmd``). + +Typical start-up, for an engine that may not be on ``PATH``:: + + command = find_tesseract_cmd() + if command: + set_tesseract_cmd(command) + ok, reason = ocr_status() + +Imports no ``PySide6``. +""" +import os +import shutil +from typing import List, NamedTuple, Optional + +from je_auto_control.utils.exception.exceptions import AutoControlActionException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.ocr.backends.base import OCRBackendNotAvailableError +from je_auto_control.utils.ocr.backends.tesseract_backend import TesseractBackend +from je_auto_control.utils.platform_id import is_macos, is_windows + +#: Environment variable naming the engine executable; read by +#: :func:`find_tesseract_cmd` before ``PATH``. +TESSERACT_CMD_ENV = "TESSERACT_CMD" +#: Environment variable the engine reads for its language-data directory. +TESSDATA_ENV = "TESSDATA_PREFIX" + +#: :func:`ocr_status` reasons. Stable strings, safe to branch on. +OCR_READY = "ready" +OCR_MISSING_PACKAGE = "missing_package" +OCR_MISSING_ENGINE = "missing_engine" +OCR_ENGINE_UNUSABLE = "engine_unusable" +OCR_NO_LANGUAGE_DATA = "no_language_data" + +#: Where the installers put the engine when they do not add it to ``PATH`` +#: (the Windows installer does not by default). +_MACOS_CANDIDATES = ("/opt/homebrew/bin/tesseract", "/usr/local/bin/tesseract", + "/opt/local/bin/tesseract") +_UNIX_CANDIDATES = ("/usr/bin/tesseract", "/usr/local/bin/tesseract") + + +class OCRStatus(NamedTuple): + """Whether OCR can work here, and if not, which part is missing. + + ``reason`` is one of the ``OCR_*`` constants in this module. + """ + ok: bool + reason: str + + +def _windows_candidates() -> List[str]: + roots = [os.environ.get("ProgramFiles") or r"C:\Program Files", + os.environ.get("ProgramFiles(x86)") or r"C:\Program Files (x86)"] + local = os.environ.get("LOCALAPPDATA") + if local: + roots.append(os.path.join(local, "Programs")) + return [os.path.join(root, "Tesseract-OCR", "tesseract.exe") for root in roots] + + +def _install_candidates(platform: str = "") -> List[str]: + """Conventional install locations for ``platform`` (default: this one).""" + if is_windows(platform): + return _windows_candidates() + if is_macos(platform): + return list(_MACOS_CANDIDATES) + return list(_UNIX_CANDIDATES) + + +def find_tesseract_cmd() -> Optional[str]: + """Return the absolute path of the Tesseract executable, or ``None``. + + Looks in order at ``$TESSERACT_CMD`` (when it names an existing file), + ``PATH``, then the installers' conventional locations. Only looks: + pass the result to ``set_tesseract_cmd`` to make OCR use it. + """ + override = os.environ.get(TESSERACT_CMD_ENV, "").strip() + if override: + if os.path.isfile(override): + return os.path.abspath(override) + autocontrol_logger.warning( + "%s names no file (%r); searching PATH instead", TESSERACT_CMD_ENV, override) + found = shutil.which("tesseract") + if found: + return os.path.abspath(found) + for candidate in _install_candidates(): + if os.path.isfile(candidate): + return candidate + return None + + +def set_tessdata_dir(path: str | os.PathLike[str] | None) -> str | None: + """Point the engine at the directory holding ``*.traineddata``; return it resolved. + + Sets ``$TESSDATA_PREFIX`` for this process, which every later engine run + inherits, so it also changes what :func:`ocr_languages` reports. ``None`` + removes the variable and the engine falls back to its built-in location. + Raises ``AutoControlActionException`` when ``path`` is not an existing + directory; an empty directory is accepted (``ocr_status`` then reports + ``no_language_data``). + """ + if path is None: + os.environ.pop(TESSDATA_ENV, None) + return None + resolved = os.path.realpath(os.fspath(path)) + if not os.path.isdir(resolved): + raise AutoControlActionException(f"tessdata directory does not exist: {resolved}") + os.environ[TESSDATA_ENV] = resolved + return resolved + + +def ocr_languages() -> Optional[List[str]]: + """Return the installed Tesseract language codes, sorted, or ``None``. + + ``None`` and ``[]`` mean different things and call for opposite handling: + + * ``[]`` -- the engine answered and has **no language data**, so every OCR + call will fail; + * ``None`` -- the engine **could not be asked** (``pytesseract`` or the + executable is missing, or listing failed), so nothing is known about + the languages. + + Asked afresh on every call, never cached. + """ + try: + return TesseractBackend().languages() + except OCRBackendNotAvailableError as error: + autocontrol_logger.info("ocr_languages: the engine cannot be asked: %r", error) + return None + + +def _command_exists(command: str) -> bool: + return shutil.which(command) is not None or os.path.isfile(command) + + +def ocr_status() -> OCRStatus: + """Report whether Tesseract OCR can work here: ``(ok, reason)``. + + Checks, in order, and stops at the first missing part: the ``pytesseract`` + package (``missing_package``), the configured engine command + (``missing_engine``), running it (``engine_unusable``), and its language + data (``no_language_data``, only when the engine *answered* with an empty + list). Otherwise ``(True, "ready")``. It checks the command OCR would use + and configures nothing; see the module docstring for the start-up recipe. + """ + backend = TesseractBackend() + try: + command = backend.cmd + except OCRBackendNotAvailableError: + return OCRStatus(False, OCR_MISSING_PACKAGE) + if not _command_exists(command): + return OCRStatus(False, OCR_MISSING_ENGINE) + try: + backend.version() + except OCRBackendNotAvailableError as error: + autocontrol_logger.warning("ocr_status: the engine does not run: %r", error) + return OCRStatus(False, OCR_ENGINE_UNUSABLE) + languages = ocr_languages() + if languages is not None and not languages: + return OCRStatus(False, OCR_NO_LANGUAGE_DATA) + return OCRStatus(True, OCR_READY) + + +__all__ = [ + "OCRStatus", "OCR_ENGINE_UNUSABLE", "OCR_MISSING_ENGINE", + "OCR_MISSING_PACKAGE", "OCR_NO_LANGUAGE_DATA", "OCR_READY", + "TESSDATA_ENV", "TESSERACT_CMD_ENV", + "find_tesseract_cmd", "ocr_languages", "ocr_status", "set_tessdata_dir", +] diff --git a/je_auto_control/utils/tween_drag/tween_drag.py b/je_auto_control/utils/tween_drag/tween_drag.py index 099c41de0..411dd91b2 100644 --- a/je_auto_control/utils/tween_drag/tween_drag.py +++ b/je_auto_control/utils/tween_drag/tween_drag.py @@ -9,8 +9,20 @@ The point math is pure and unit-testable; dispatch goes through an injectable ``sink`` so the drag is tested without real input. Imports no ``PySide6``. + +The press / move / release sequence itself is :func:`_drag_through`, shared +with ``mouse_path.drag_path``: optional pacing (``step_delay_s`` after each +move, ``settle_s`` before the press, after it and before the release) and a +release that runs in ``finally``, at the last point the pointer reached, if +any step raises. """ -from typing import Any, Callable, Dict, List, Optional, Tuple +import math +import time +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +Sink = Callable[[Dict[str, Any]], None] def _linear(t: float) -> float: @@ -76,20 +88,89 @@ def _default_sink(event: Dict[str, Any]) -> None: release_mouse(event.get("button", "mouse_left"), x, y) -def tween_drag(start: Tuple[int, int], end: Tuple[int, int], *, - steps: int = 30, easing: str = "ease_in_out_quad", - button: str = "mouse_left", - sink: Optional[Callable[[Dict[str, Any]], None]] = None - ) -> Dict[str, Any]: - """Drag from ``start`` to ``end`` along an eased path; return point count.""" - points = tween_points(start, end, steps, easing) - dispatch = sink or _default_sink +def _pause_seconds(value: Any, name: str) -> float: + """``value`` as a finite, non-negative number of seconds, else ValueError. + + A NaN would pass a plain ``< 0`` test and then make ``time.sleep`` raise + with the button already held, so finiteness is checked up front. + """ + try: + seconds = float(value) + except (TypeError, ValueError) as error: + raise ValueError(f"{name} must be a number of seconds, got {value!r}") from error + if not math.isfinite(seconds) or seconds < 0: + raise ValueError(f"{name} must be a finite number >= 0, got {value!r}") + return seconds + + +def _pause(seconds: float) -> None: + if seconds: + time.sleep(seconds) + + +def _release_quietly(dispatch: Sink, button: str, point: Sequence[int]) -> None: + """Release ``button`` at ``point`` from a ``finally``; never raises. + + An exception here would replace the one that ended the drag, which is + what the caller needs to see, so a failed release is logged instead. + ``Exception`` rather than a list: the default sink raises the + ``AutoControlException`` family, which no builtin type covers. + """ + try: + dispatch({"op": "release", "button": button, "x": point[0], "y": point[1]}) + except Exception as error: # noqa: BLE001 # pylint: disable=broad-except # reason: see docstring + autocontrol_logger.error("failed to release %r after an aborted drag: %r", button, error) + + +def _drag_through(points: List[List[int]], button: str, dispatch: Sink, + step_delay_s: Any = 0.0, settle_s: Any = 0.0) -> None: + """Press at ``points[0]``, move through every point, release at ``points[-1]``. + + ``settle_s`` (when non-zero) first moves to the start and rests there, + then rests after the press and again before the release; ``step_delay_s`` + rests after each move. With both at 0 the events are exactly press, + moves, release. If any step raises, the button is released in + ``finally`` at the last point the pointer reached -- not at the end, + which would complete a drop the drag never got to -- and a failing + cleanup release is logged rather than raised over the original error. + Both pauses are checked before anything is dispatched. + """ + step_delay = _pause_seconds(step_delay_s, "step_delay_s") + settle = _pause_seconds(settle_s, "settle_s") first, last = points[0], points[-1] + if settle: + dispatch({"op": "move", "x": first[0], "y": first[1]}) + time.sleep(settle) dispatch({"op": "press", "button": button, "x": first[0], "y": first[1]}) + reached: Sequence[int] = first + held = True try: + _pause(settle) for x, y in points: dispatch({"op": "move", "x": x, "y": y}) - finally: - # A failed move used to leave the button held down. + reached = (x, y) + _pause(step_delay) + _pause(settle) dispatch({"op": "release", "button": button, "x": last[0], "y": last[1]}) + held = False + finally: + if held: + _release_quietly(dispatch, button, reached) + + +def tween_drag(start: Tuple[int, int], end: Tuple[int, int], *, + steps: int = 30, easing: str = "ease_in_out_quad", + button: str = "mouse_left", + sink: Optional[Sink] = None, + step_delay_s: float = 0.0, settle_s: float = 0.0, + ) -> Dict[str, Any]: + """Drag from ``start`` to ``end`` along an eased path; return point count. + + ``step_delay_s`` rests after each move and ``settle_s`` before the press, + after it and before the release (seconds, default 0: no pause). Apps + that tell a drag from a click by the pointer's motion need both. If a + step raises, the button is released where the pointer stopped. + """ + points = tween_points(start, end, steps, easing) + _drag_through(points, button, sink or _default_sink, step_delay_s, settle_s) return {"points": len(points), "path": points} diff --git a/je_auto_control/windows/core/utils/win32_vk.py b/je_auto_control/windows/core/utils/win32_vk.py index ef7a34e28..33bc4fc3b 100644 --- a/je_auto_control/windows/core/utils/win32_vk.py +++ b/je_auto_control/windows/core/utils/win32_vk.py @@ -175,6 +175,7 @@ WIN32_VK_BROWSER_STOP: int = 0xA9 # Browser Stop key WIN32_VK_BROWSER_SEARCH: int = 0xAA # Browser Search key WIN32_VK_BROWSER_FAVORITES: int = 0xAB # Browser Favorites key +WIN32_VK_BROWSER_HOME: int = 0xAC # Browser Start and Home key WIN32_VK_VOLUME_MUTE: int = 0xAD WIN32_VK_VOLUME_DOWN: int = 0xAE WIN32_VK_VOLUME_UP: int = 0xAF @@ -185,4 +186,20 @@ WIN32_VK_LAUNCH_MAIL: int = 0xB4 WIN32_VK_LAUNCH_MEDIA_SELECT: int = 0xB5 WIN32_VK_LAUNCH_APP1: int = 0xB6 -WIN32_VK_LAUNCH_APP2: int = 0xB7 \ No newline at end of file +WIN32_VK_LAUNCH_APP2: int = 0xB7 +# OEM keys (Microsoft "Virtual-Key Codes"). OEM_PLUS / COMMA / MINUS / PERIOD +# are the same key on every layout; OEM_1 .. OEM_8 and OEM_102 vary by layout. +WIN32_VK_OEM_1: int = 0xBA +WIN32_VK_OEM_PLUS: int = 0xBB +WIN32_VK_OEM_COMMA: int = 0xBC +WIN32_VK_OEM_MINUS: int = 0xBD +WIN32_VK_OEM_PERIOD: int = 0xBE +WIN32_VK_OEM_2: int = 0xBF +WIN32_VK_OEM_3: int = 0xC0 +WIN32_VK_OEM_4: int = 0xDB +WIN32_VK_OEM_5: int = 0xDC +WIN32_VK_OEM_6: int = 0xDD +WIN32_VK_OEM_7: int = 0xDE +WIN32_VK_OEM_8: int = 0xDF +WIN32_VK_OEM_102: int = 0xE2 +WIN32_VK_OEM_CLEAR: int = 0xFE \ No newline at end of file diff --git a/je_auto_control/wrapper/_platform_linux.py b/je_auto_control/wrapper/_platform_linux.py index cf66a07b2..5e9d90a7a 100644 --- a/je_auto_control/wrapper/_platform_linux.py +++ b/je_auto_control/wrapper/_platform_linux.py @@ -1,4 +1,4 @@ -from typing import Tuple +from typing import Dict, Tuple from je_auto_control.linux_with_x11.core.utils.x11_linux_vk import ( x11_linux_key_backspace, x11_linux_key_slash_b, x11_linux_key_tab, @@ -271,5 +271,8 @@ def _select_input_backend() -> Tuple[X11UnixKeyboardBackend, screen: ScreenBackend = x11_linux_screen recorder: RecorderBackend = x11_linux_recorder +#: Aliases of canonical key names; none on this backend (see ``_platform_windows``). +keyboard_key_aliases: Dict[str, str] = {} + if None in [keyboard_keys_table, mouse_keys_table, special_mouse_keys_table, keyboard, mouse, screen, recorder]: raise AutoControlException("Can't init auto control") diff --git a/je_auto_control/wrapper/_platform_osx.py b/je_auto_control/wrapper/_platform_osx.py index 0a44983ce..9da36234b 100644 --- a/je_auto_control/wrapper/_platform_osx.py +++ b/je_auto_control/wrapper/_platform_osx.py @@ -1,3 +1,5 @@ +from typing import Dict + from je_auto_control.osx.core.utils.osx_vk import ( osx_key_a, osx_key_A, osx_key_b, osx_key_B, osx_key_c, osx_key_C, osx_key_d, osx_key_D, osx_key_e, osx_key_E, osx_key_f, osx_key_F, @@ -153,5 +155,8 @@ screen: ScreenBackend = osx_screen recorder: RecorderBackend = osx_recorder +#: Aliases of canonical key names; none on this backend (see ``_platform_windows``). +keyboard_key_aliases: Dict[str, str] = {} + if None in [keyboard_keys_table, mouse_keys_table, keyboard_check, keyboard, mouse, screen, recorder]: raise AutoControlException("Can't init auto control") diff --git a/je_auto_control/wrapper/_platform_wayland.py b/je_auto_control/wrapper/_platform_wayland.py index 9c147eab1..d5cbd5f3c 100644 --- a/je_auto_control/wrapper/_platform_wayland.py +++ b/je_auto_control/wrapper/_platform_wayland.py @@ -9,6 +9,8 @@ ydotool BTN_* hex codes are converted in :mod:`linux_wayland.mouse` so the public mouse_keys_table stays consistent across platforms. """ +from typing import Dict + from je_auto_control.linux_wayland import keyboard as wayland_keyboard from je_auto_control.linux_wayland import listener as wayland_listener from je_auto_control.linux_wayland import mouse as wayland_mouse @@ -46,13 +48,16 @@ recorder: RecorderBackend = wayland_record.wayland_recorder +#: Aliases of canonical key names; none on this backend (see ``_platform_windows``). +keyboard_key_aliases: Dict[str, str] = {} + if None in [keyboard_keys_table, mouse_keys_table, special_mouse_keys_table, keyboard, mouse, screen, recorder]: raise AutoControlException("Can't init auto control (Wayland)") __all__ = [ - "keyboard", "keyboard_check", "keyboard_keys_table", + "keyboard", "keyboard_check", "keyboard_key_aliases", "keyboard_keys_table", "mouse", "mouse_keys_table", "recorder", "screen", "special_mouse_keys_table", ] diff --git a/je_auto_control/wrapper/_platform_windows.py b/je_auto_control/wrapper/_platform_windows.py index c5e1b239c..ea89419ce 100644 --- a/je_auto_control/wrapper/_platform_windows.py +++ b/je_auto_control/wrapper/_platform_windows.py @@ -1,5 +1,5 @@ import os -from typing import Tuple +from typing import Dict, Tuple from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -11,7 +11,7 @@ from je_auto_control.windows.core.utils.win32_vk import ( WIN32_VK_ACCEPT, WIN32_VK_ADD, WIN32_VK_APPS, WIN32_VK_BACK, WIN32_VK_BROWSER_BACK, WIN32_VK_BROWSER_FAVORITES, - WIN32_VK_BROWSER_FORWARD, WIN32_VK_BROWSER_REFRESH, + WIN32_VK_BROWSER_FORWARD, WIN32_VK_BROWSER_HOME, WIN32_VK_BROWSER_REFRESH, WIN32_VK_BROWSER_SEARCH, WIN32_VK_BROWSER_STOP, WIN32_VK_CANCEL, WIN32_VK_CAPITAL, WIN32_VK_CLEAR, WIN32_VK_CONTROL, WIN32_VK_CONVERT, WIN32_VK_DECIMAL, WIN32_VK_DELETE, WIN32_VK_DIVIDE, WIN32_VK_DOWN, @@ -31,6 +31,10 @@ WIN32_VK_MEDIA_PREV_TRACK, WIN32_VK_MEDIA_STOP, WIN32_VK_MODECHANGE, WIN32_VK_MULTIPLY, WIN32_VK_Menu, WIN32_VK_NEXT, WIN32_VK_NONCONVERT, WIN32_VK_NUMLOCK, + WIN32_VK_OEM_1, WIN32_VK_OEM_2, WIN32_VK_OEM_3, WIN32_VK_OEM_4, + WIN32_VK_OEM_5, WIN32_VK_OEM_6, WIN32_VK_OEM_7, WIN32_VK_OEM_8, + WIN32_VK_OEM_102, WIN32_VK_OEM_CLEAR, WIN32_VK_OEM_COMMA, + WIN32_VK_OEM_MINUS, WIN32_VK_OEM_PERIOD, WIN32_VK_OEM_PLUS, WIN32_VK_NUMPAD0, WIN32_VK_NUMPAD1, WIN32_VK_NUMPAD2, WIN32_VK_NUMPAD3, WIN32_VK_NUMPAD4, WIN32_VK_NUMPAD5, WIN32_VK_NUMPAD6, WIN32_VK_NUMPAD7, WIN32_VK_NUMPAD8, WIN32_VK_NUMPAD9, @@ -132,6 +136,7 @@ def _build_mouse_keys_table(mouse_module) -> dict: "browser_back": WIN32_VK_BROWSER_BACK, "browser_favorites": WIN32_VK_BROWSER_FAVORITES, "browser_forward": WIN32_VK_BROWSER_FORWARD, + "browser_home": WIN32_VK_BROWSER_HOME, "browser_refresh": WIN32_VK_BROWSER_REFRESH, "browser_search": WIN32_VK_BROWSER_SEARCH, "browser_stop": WIN32_VK_BROWSER_STOP, @@ -139,9 +144,6 @@ def _build_mouse_keys_table(mouse_module) -> dict: "capital": WIN32_VK_CAPITAL, "clear": WIN32_VK_CLEAR, "control": WIN32_VK_CONTROL, - # Linux and macOS call it "ctrl", and so do the defaults of type_unicode, - # set_field_text and with_modifiers; without the alias they failed here. - "ctrl": WIN32_VK_CONTROL, "convert": WIN32_VK_CONVERT, "decimal": WIN32_VK_DECIMAL, "delete": WIN32_VK_DELETE, @@ -184,7 +186,10 @@ def _build_mouse_keys_table(mouse_module) -> dict: "junja": WIN32_VK_JUNJA, "kana": WIN32_VK_KANA, "launch_app1": WIN32_VK_LAUNCH_APP1, + # The upper-case spelling stays for scripts that use it; a lower-case + # caller (or anything that lower-cases a typed name) needs the other. "LAUNCH_APP2": WIN32_VK_LAUNCH_APP2, + "launch_app2": WIN32_VK_LAUNCH_APP2, "launch_mail": WIN32_VK_LAUNCH_MAIL, "launch_media_select": WIN32_VK_LAUNCH_MEDIA_SELECT, "lbutton": WIN32_VK_LBUTTON, @@ -203,6 +208,22 @@ def _build_mouse_keys_table(mouse_module) -> dict: "menu": WIN32_VK_Menu, "next": WIN32_VK_NEXT, "nonconvert": WIN32_VK_NONCONVERT, + # OEM keys are named after Microsoft's constants on purpose: oem_1 is ";" + # on a US layout and something else elsewhere, so "semicolon" would lie. + "oem_1": WIN32_VK_OEM_1, + "oem_2": WIN32_VK_OEM_2, + "oem_3": WIN32_VK_OEM_3, + "oem_4": WIN32_VK_OEM_4, + "oem_5": WIN32_VK_OEM_5, + "oem_6": WIN32_VK_OEM_6, + "oem_7": WIN32_VK_OEM_7, + "oem_8": WIN32_VK_OEM_8, + "oem_102": WIN32_VK_OEM_102, + "oem_clear": WIN32_VK_OEM_CLEAR, + "oem_comma": WIN32_VK_OEM_COMMA, + "oem_minus": WIN32_VK_OEM_MINUS, + "oem_period": WIN32_VK_OEM_PERIOD, + "oem_plus": WIN32_VK_OEM_PLUS, "numlock": WIN32_VK_NUMLOCK, "num0": WIN32_VK_NUMPAD0, "num1": WIN32_VK_NUMPAD1, @@ -304,6 +325,33 @@ def _build_mouse_keys_table(mouse_module) -> dict: "z": WIN32_keyZ, } +# 常見的別名 → 上表的標準鍵名。別名也查得到同一個鍵碼，但反查（`keyboard_key_name`） +# 一律回標準名，所以加別名不會改變錄製寫出來的鍵名。 +# Common aliases -> the canonical names above. An alias resolves to the same +# code, but a reverse lookup (``keyboard_key_name``) never answers with one, so +# adding an alias cannot change the key names a recorder writes. +keyboard_key_aliases: Dict[str, str] = { + # Linux and macOS call it "ctrl", and so do the defaults of type_unicode, + # set_field_text and with_modifiers; without the alias they failed here. + "ctrl": "control", "lctrl": "lcontrol", "rctrl": "rcontrol", + "alt": "menu", "lalt": "lmenu", "ralt": "rmenu", + "enter": "return", "esc": "escape", + "win": "lwin", "super": "lwin", "cmd": "lwin", "meta": "lwin", + "backspace": "back", "bksp": "back", + "del": "delete", "ins": "insert", + "pgup": "prior", "pageup": "prior", "pgdn": "next", "pagedown": "next", + "caps": "capital", "capslock": "capital", + "printscreen": "snapshot", "prtsc": "snapshot", "prtscr": "snapshot", + "scrolllock": "scroll", + **{f"numpad{digit}": f"num{digit}" for digit in range(10)}, + # Microsoft defines these four as the same key on every layout, so the + # readable name does not lie; oem_1 .. oem_8 get no alias for that reason. + "plus": "oem_plus", "comma": "oem_comma", "minus": "oem_minus", + "period": "oem_period", +} +keyboard_keys_table.update( + {alias: keyboard_keys_table[name] for alias, name in keyboard_key_aliases.items()}) + special_mouse_keys_table = None keyboard, mouse = _select_input_backend() # 必須在挑選後端後才建表，否則 Interception 後端會拿到 SendInput 的旗標。 diff --git a/je_auto_control/wrapper/auto_control_keyboard.py b/je_auto_control/wrapper/auto_control_keyboard.py index 31373ce8e..48cb50feb 100644 --- a/je_auto_control/wrapper/auto_control_keyboard.py +++ b/je_auto_control/wrapper/auto_control_keyboard.py @@ -8,6 +8,7 @@ and the only form a type checker can prune, and raise on a platform that matches neither. """ +import re import sys import warnings from typing import Optional, Union, Tuple @@ -25,7 +26,9 @@ from je_auto_control.utils.platform_id import is_windows, is_x11_unix from je_auto_control.utils.test_record.record_test_class import record_action_to_list from je_auto_control.utils.text_unicode.text_unicode import unicode_code_units -from je_auto_control.wrapper.platform_wrapper import keyboard, keyboard_keys_table, keyboard_check +from je_auto_control.wrapper.platform_wrapper import ( + keyboard, keyboard_check, keyboard_key_aliases, keyboard_keys_table, +) def get_keyboard_keys_table() -> dict: """ @@ -35,6 +38,31 @@ def get_keyboard_keys_table() -> dict: return keyboard_keys_table +#: A key name a user can type back: lower case, digits, underscore. +_TYPABLE_KEY_NAME = re.compile(r"[a-z0-9_]+") + + +def keyboard_key_name(keycode: int) -> Optional[str]: + """ + 鍵碼反查標準鍵名；查不到回 None + Return the canonical key name for ``keycode``, or ``None`` if it has none. + + A code often has several names in the table: aliases (``esc`` for + ``escape``), a legacy upper-case spelling (``LAUNCH_APP2``), a letter's + capital (``A``), two spellings of one key (``down`` / ``vk_down``). + Aliases never answer (``keyboard_key_aliases`` lists them), so adding one + cannot change the name a recorder writes. Among the rest, a name made of + lower-case letters, digits and underscores wins (it can be typed back), + then the shortest, then the alphabetically first, so every machine with + the same table gives the same answer. + """ + names = [name for name, code in keyboard_keys_table.items() + if code == keycode and name not in keyboard_key_aliases] + if not names: + return None + return min(names, key=lambda name: (_TYPABLE_KEY_NAME.fullmatch(name) is None, len(name), name)) + + def _resolve_keycode(keycode: Union[int, str]) -> int: """ 將字串鍵名轉換成對應的 keycode diff --git a/je_auto_control/wrapper/auto_control_mouse.py b/je_auto_control/wrapper/auto_control_mouse.py index 881a967df..f6f23e15a 100644 --- a/je_auto_control/wrapper/auto_control_mouse.py +++ b/je_auto_control/wrapper/auto_control_mouse.py @@ -29,9 +29,11 @@ belt-and-braces half of the same statement. """ import ctypes +import math import sys +import time import warnings -from typing import Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union from je_auto_control.utils.exception.exception_tags import ( mouse_click_mouse_error_message, mouse_get_position_error_message, mouse_press_mouse_error_message, @@ -251,30 +253,74 @@ def release_mouse(mouse_keycode: Union[int, str], x: Optional[int] = None, raise -def click_mouse(mouse_keycode: Union[int, str], x: Optional[int] = None, - y: Optional[int] = None) -> Tuple[MouseKeycode, int, int]: +def _click_plan(clicks: object, interval: object) -> Tuple[int, float]: + """``(clicks, interval)`` checked, else raise ``AutoControlMouseException``. + + ``clicks`` takes an int or an integer string (an action file's variable + may arrive as text) but not a float, which would be silently truncated, + nor a ``bool``: ``clicks=True`` is a caller's mistake, not one click. A + NaN interval would pass a plain ``< 0`` test, so finiteness is checked + explicitly. + """ + problem = f"clicks must be an integer >= 1, got {clicks!r}" + if isinstance(clicks, bool): + raise AutoControlMouseException(problem) + try: + count = int(str(clicks).strip()) + except ValueError as error: + raise AutoControlMouseException(problem) from error + if count < 1: + raise AutoControlMouseException(problem) + try: + pause = float(interval) # type: ignore[arg-type] + except (TypeError, ValueError) as error: + raise AutoControlMouseException(f"interval must be a number, got {interval!r}") from error + if not math.isfinite(pause) or pause < 0: + raise AutoControlMouseException(f"interval must be a finite number >= 0, got {interval!r}") + return count, pause + + +def click_mouse(mouse_keycode: int | str, x: int | None = None, + y: int | None = None, clicks: int = 1, + interval: float = 0.0) -> Tuple[MouseKeycode, int, int]: """ - 在指定座標按下並放開滑鼠按鍵 - Click mouse button at given position + 在指定座標按下並放開滑鼠按鍵；``clicks=2`` 就是雙擊 + Click mouse button at given position; ``clicks=2`` double-clicks + + All clicks land on the same point, ``interval`` seconds apart. Windows and + X11 recognise a double-click from the timing and distance of two clicks, + so keep ``interval`` under the system double-click time (500 ms by + default on Windows). macOS apps read a click count this backend does not + set, so there the clicks arrive as separate single clicks. :param mouse_keycode: 滑鼠按鍵代碼 Mouse keycode :param x: X 座標 X position :param y: Y 座標 Y position + :param clicks: 連點次數 Number of clicks (>= 1) + :param interval: 兩次點擊之間的秒數 Seconds between clicks (>= 0) :return: (keycode, x, y) """ - autocontrol_logger.info(f"click_mouse, keycode={mouse_keycode}, x={x}, y={y}") - param = {"keycode": mouse_keycode, "x": x, "y": y} + autocontrol_logger.info( + f"click_mouse, keycode={mouse_keycode}, x={x}, y={y}, clicks={clicks}, interval={interval}") + param: Dict[str, Any] = {"keycode": mouse_keycode, "x": x, "y": y} + if clicks != 1 or interval: + # Only when used, so a single click records exactly what it always did. + param.update(clicks=clicks, interval=interval) try: + count, pause = _click_plan(clicks, interval) keycode, x, y = mouse_preprocess(mouse_keycode, x, y) - # macOS orders its mouse backend as (x, y, button) — same convention as - # press_mouse/release_mouse above. Without this branch the arguments - # bind as x=, y=, button=; the osx button table holds - # strings, so the int never matches any branch and the click is - # silently dropped with no exception. - if sys.platform == "darwin": - mouse.click_mouse(x, y, keycode) - else: - mouse.click_mouse(keycode, x, y) + for index in range(count): + if index and pause: + time.sleep(pause) + # macOS orders its mouse backend as (x, y, button) — same convention as + # press_mouse/release_mouse above. Without this branch the arguments + # bind as x=, y=, button=; the osx button table holds + # strings, so the int never matches any branch and the click is + # silently dropped with no exception. + if sys.platform == "darwin": + mouse.click_mouse(x, y, keycode) + else: + mouse.click_mouse(keycode, x, y) record_action_to_list("click_mouse", param) return keycode, x, y except AutoControlMouseException as error: diff --git a/je_auto_control/wrapper/platform_wrapper.py b/je_auto_control/wrapper/platform_wrapper.py index f4a4c4054..199d9cf57 100644 --- a/je_auto_control/wrapper/platform_wrapper.py +++ b/je_auto_control/wrapper/platform_wrapper.py @@ -15,6 +15,9 @@ # target. keyboard_check: KeyboardCheckBackend keyboard_keys_table: Dict[str, int] +#: Alias -> canonical name for entries of ``keyboard_keys_table`` that are +#: aliases (``esc`` for ``escape``); reverse lookups skip them. +keyboard_key_aliases: Dict[str, str] #: Values are platform-specific button codes: a plain int on X11, Wayland and #: macOS, a tuple of three Win32 event flags on Windows. mouse_keys_table: Dict[str, MouseKeycode] @@ -41,13 +44,13 @@ if is_windows(): from je_auto_control.wrapper._platform_windows import ( # noqa: F401 # reason: facade re-export - keyboard as _keyboard, keyboard_check, keyboard_keys_table, + keyboard as _keyboard, keyboard_check, keyboard_key_aliases, keyboard_keys_table, mouse as _mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) elif is_macos(): from je_auto_control.wrapper._platform_osx import ( # noqa: F401 # reason: facade re-export - keyboard as _keyboard, keyboard_check, keyboard_keys_table, + keyboard as _keyboard, keyboard_check, keyboard_key_aliases, keyboard_keys_table, mouse as _mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) @@ -60,7 +63,7 @@ if _DISPLAY_SERVER == "wayland": try: from je_auto_control.wrapper._platform_wayland import ( # noqa: F401 # reason: facade re-export - keyboard as _keyboard, keyboard_check, keyboard_keys_table, + keyboard as _keyboard, keyboard_check, keyboard_key_aliases, keyboard_keys_table, mouse as _mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) @@ -72,13 +75,13 @@ "this warning.", _wayland_error, ) from je_auto_control.wrapper._platform_linux import ( # noqa: F401 # reason: facade re-export - keyboard as _keyboard, keyboard_check, keyboard_keys_table, + keyboard as _keyboard, keyboard_check, keyboard_key_aliases, keyboard_keys_table, mouse as _mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) else: from je_auto_control.wrapper._platform_linux import ( # noqa: F401 # reason: facade re-export - keyboard as _keyboard, keyboard_check, keyboard_keys_table, + keyboard as _keyboard, keyboard_check, keyboard_key_aliases, keyboard_keys_table, mouse as _mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) @@ -89,7 +92,7 @@ # whose socket and portal are Linux desktop infrastructure; a BSD running # Wayland reaches this through XWayland like any other X11 client. from je_auto_control.wrapper._platform_linux import ( # noqa: F401 # reason: facade re-export - keyboard as _keyboard, keyboard_check, keyboard_keys_table, + keyboard as _keyboard, keyboard_check, keyboard_key_aliases, keyboard_keys_table, mouse as _mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) @@ -110,7 +113,7 @@ __all__ = [ - "keyboard", "keyboard_check", "keyboard_keys_table", + "keyboard", "keyboard_check", "keyboard_key_aliases", "keyboard_keys_table", "mouse", "mouse_keys_table", "special_mouse_keys_table", "screen", "recorder", ] diff --git a/pyproject.toml b/pyproject.toml index aaf9bf06e..20a511191 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,9 +74,9 @@ je_auto_control_mcp = "je_auto_control.utils.mcp_server.__main__:main" je_auto_control = "je_auto_control.utils.pytest_plugin.plugin" [project.urls] -Homepage = "https://github.com/Intergration-Automation-Testing/AutoControl" +Homepage = "https://github.com/Integration-Automation/AutoControlGUI" Documentation = "https://autocontrol.readthedocs.io/en/latest/" -Code = "https://github.com/Intergration-Automation-Testing/AutoControl" +Code = "https://github.com/Integration-Automation/AutoControlGUI" [project.readme] file = "README.md" diff --git a/requirements.txt b/requirements.txt index abfdb63c8..a9b90dc3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # Floor, not a pin: releases before 0.0.216 resolve pillow==12.2.0, which # carries 13 CVEs fixed in 12.3.0. A floor keeps the dependency graph honest # without needing a bump here on every release. -je_auto_control>=0.0.216 +je_auto_control>=0.0.217 # Stated directly as well: the dependency graph recorded Pillow for this file # when je_auto_control was still unbounded here, and the floor above never # replaced that entry. A direct requirement does. diff --git a/test/unit_test/headless/test_agent_executor_mcp_wiring.py b/test/unit_test/headless/test_agent_executor_mcp_wiring.py index 132426e14..e9c991e86 100644 --- a/test/unit_test/headless/test_agent_executor_mcp_wiring.py +++ b/test/unit_test/headless/test_agent_executor_mcp_wiring.py @@ -80,3 +80,26 @@ def test_unknown_backend_raises(): import pytest with pytest.raises(ValueError, match="unknown agent backend"): _run_agent(goal="x", backend="bogus") + + +def test_openai_agent_uses_focused_toolset(monkeypatch): + captured = {} + + class StubBackend(FakeAgentBackend): + def __init__(self, *, tools, **kwargs): + captured["tools"] = tools + super().__init__([{"stop": True, "message": "focused"}]) + + import je_auto_control.utils.agent.backends as backends_pkg + monkeypatch.setattr(backends_pkg, "OpenAIAgentBackend", StubBackend) + from je_auto_control.utils.executor.action_executor import _run_agent + + result = _run_agent(goal="probe", backend="openai", max_steps=1, wall_seconds=5.0) + + names = {item["function"]["name"] for item in captured["tools"]} + assert result["succeeded"] is True + assert "AC_screenshot" in names + assert "AC_click_mouse" in names + assert "AC_shell_command" not in names + assert "AC_execute_process" not in names + assert len(names) <= 128 diff --git a/test/unit_test/headless/test_click_mouse_multi.py b/test/unit_test/headless/test_click_mouse_multi.py new file mode 100644 index 000000000..646cfabd7 --- /dev/null +++ b/test/unit_test/headless/test_click_mouse_multi.py @@ -0,0 +1,124 @@ +"""``click_mouse(clicks=, interval=)``: multi-click in one call. + +The backend and the clock are stand-ins, so no real click is sent and nothing +sleeps. No Qt imports. +""" +import math +import sys +import types + +import pytest + +from je_auto_control.utils.exception.exceptions import AutoControlMouseException +from je_auto_control.wrapper import auto_control_mouse + + +class _Backend: + """Records each backend click; mirrors the Windows / X11 signature.""" + + def __init__(self): + self.clicks = [] + + def click_mouse(self, mouse_keycode, x=None, y=None): + self.clicks.append((mouse_keycode, x, y)) + + +@pytest.fixture +def env(monkeypatch): + backend = _Backend() + sleeps = [] + records = [] + monkeypatch.setattr(auto_control_mouse, "mouse", backend) + monkeypatch.setattr(auto_control_mouse, "time", types.SimpleNamespace(sleep=sleeps.append)) + monkeypatch.setattr(auto_control_mouse, "mouse_keys_table", {"mouse_left": 1, "mouse_right": 2}) + monkeypatch.setattr(auto_control_mouse, "record_action_to_list", + lambda name, param, error=None: records.append((name, dict(param or {}), error))) + monkeypatch.setattr(auto_control_mouse, "get_mouse_position", lambda: (7, 8)) + monkeypatch.setattr(sys, "platform", "win32") + return types.SimpleNamespace(backend=backend, sleeps=sleeps, records=records) + + +def test_the_default_is_one_click_with_no_pause_and_the_old_record(env): + assert auto_control_mouse.click_mouse("mouse_left", 10, 20) == (1, 10, 20) + assert env.backend.clicks == [(1, 10, 20)] + assert env.sleeps == [] + assert env.records == [("click_mouse", {"keycode": "mouse_left", "x": 10, "y": 20}, None)] + + +def test_a_double_click_is_two_clicks_on_one_point_with_one_pause(env): + auto_control_mouse.click_mouse("mouse_left", 10, 20, clicks=2, interval=0.06) + assert env.backend.clicks == [(1, 10, 20), (1, 10, 20)] + assert env.sleeps == [0.06] + name, param, error = env.records[-1] + assert (name, error) == ("click_mouse", None) + assert param["clicks"] == 2 + assert param["interval"] == 0.06 + + +def test_no_pause_between_clicks_when_interval_is_zero(env): + auto_control_mouse.click_mouse("mouse_right", clicks=3) + assert env.backend.clicks == [(2, 7, 8)] * 3 # the cursor, resolved once + assert env.sleeps == [] + + +def test_an_integer_string_is_accepted(env): + auto_control_mouse.click_mouse("mouse_left", 1, 2, clicks="2") + assert len(env.backend.clicks) == 2 + + +def test_macos_binds_every_click_xy_first(env, monkeypatch): + calls = [] + monkeypatch.setattr(auto_control_mouse, "mouse", types.SimpleNamespace( + click_mouse=lambda x, y, mouse_button: calls.append((x, y, mouse_button)))) + monkeypatch.setattr(sys, "platform", "darwin") + auto_control_mouse.click_mouse("mouse_left", 5, 6, clicks=2) + assert calls == [(5, 6, 1), (5, 6, 1)] + + +@pytest.mark.parametrize("clicks", [0, -1, True, False, 1.5, 2.0, "x", None]) +def test_an_invalid_click_count_clicks_nothing(env, clicks): + with pytest.raises(AutoControlMouseException): + auto_control_mouse.click_mouse("mouse_left", 1, 2, clicks=clicks) + assert env.backend.clicks == [] + + +@pytest.mark.parametrize("interval", [-0.1, math.nan, math.inf, "soon", None]) +def test_an_invalid_interval_clicks_nothing(env, interval): + with pytest.raises(AutoControlMouseException): + auto_control_mouse.click_mouse("mouse_left", 1, 2, clicks=2, interval=interval) + assert env.backend.clicks == [] + assert env.records[-1][2] is not None # the failure is recorded + + +def test_the_executor_command_forwards_both_parameters(env): + from je_auto_control.utils.executor.action_executor import executor + executor.event_dict["AC_click_mouse"]("mouse_left", 3, 4, clicks=2, interval=0.05) + assert env.backend.clicks == [(1, 3, 4), (1, 3, 4)] + assert env.sleeps == [0.05] + + +def test_the_mcp_handler_forwards_both_parameters(env): + from je_auto_control.utils.mcp_server.tools import _handlers_input + assert _handlers_input.click_mouse("mouse_left", 3, 4, clicks=2, interval=0.05) == [1, 3, 4] + assert len(env.backend.clicks) == 2 + + +def test_the_mcp_fake_backend_records_each_click(): + from je_auto_control.utils.mcp_server import fake_backend + from je_auto_control.utils.mcp_server.tools import _handlers_input + fake_backend.reset_fake_state() + fake_backend.install_fake_backend() + try: + _handlers_input.click_mouse("mouse_left", 3, 4, clicks=2) + assert fake_backend.fake_state().mouse_actions == [("click", "mouse_left", 3, 4)] * 2 + finally: + fake_backend.uninstall_fake_backend() + + +def test_the_mcp_tool_and_script_builder_declare_both_parameters(): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + tool = next(t for t in build_default_tool_registry() if t.name == "ac_click_mouse") + assert {"clicks", "interval"} <= set(tool.input_schema["properties"]) + from je_auto_control.gui.script_builder.command_schema import _build_specs + spec = next(s for s in _build_specs() if s.command == "AC_click_mouse") + assert {"clicks", "interval"} <= {field.name for field in spec.fields} diff --git a/test/unit_test/headless/test_cross_project_contracts.py b/test/unit_test/headless/test_cross_project_contracts.py index 3b5b949e6..1db3960db 100644 --- a/test/unit_test/headless/test_cross_project_contracts.py +++ b/test/unit_test/headless/test_cross_project_contracts.py @@ -104,6 +104,39 @@ def test_create_project(tmp_path, flag): assert keywords, result.stderr[-2000:] +def _failing_actions(target: Path) -> list: + """An action that fails when it runs, then one that writes ``target``. + + The first writes below a path that is a file, so it passes the check of the + script and raises only at run time. + """ + not_a_directory = target.with_name(target.name + ".blocker") + not_a_directory.write_text("x", encoding="utf-8") + return [["AC_export_sarif", {"findings": [], "path": str(not_a_directory / "out.sarif")}], + *_actions(target)] + + +@pytest.mark.parametrize("flag", ["-e", "--execute_file"]) +def test_a_failed_action_exits_1_after_the_rest_ran(tmp_path, flag): + target = tmp_path / "out.sarif" + action_file = tmp_path / "actions.json" + action_file.write_text(json.dumps(_failing_actions(target)), encoding="utf-8") + result = _run_cli(tmp_path, flag, str(action_file)) + assert result.returncode == 1, result.stderr[-2000:] + assert "error: 1 action(s) failed" in result.stderr + assert target.is_file(), result.stderr[-2000:] + + +def test_a_failed_action_in_a_directory_exits_1(tmp_path): + action_dir = tmp_path / "actions" + action_dir.mkdir() + (action_dir / "a_ok.json").write_text( + json.dumps(_actions(tmp_path / "a.sarif")), encoding="utf-8") + (action_dir / "b_bad.json").write_text( + json.dumps(_failing_actions(tmp_path / "b.sarif")), encoding="utf-8") + assert _run_cli(tmp_path, "--execute_dir", str(action_dir)).returncode == 1 + + def test_no_flag_exits_non_zero(tmp_path): """A caller that forgot its flag must see a failure, not a silent 0.""" assert _run_cli(tmp_path).returncode != 0 @@ -125,6 +158,9 @@ def test_no_flag_exits_non_zero(tmp_path): "save_window_layout", "screen_size", "set_clipboard", "set_mouse_position", "snap_window", "wait_until_clipboard_changes", "wait_until_port", "wait_until_process", "write", + # Jeffrey_RPA: added so it can drop its own copies of the same logic. + "find_tesseract_cmd", "ocr_languages", "ocr_status", "set_tessdata_dir", + "keyboard_key_name", ]) def test_facade_name_other_repositories_call(name): assert name in je_auto_control.__all__, f"{name} left the facade" @@ -154,6 +190,18 @@ def test_internal_names_jeffrey_rpa_imports(): assert isinstance(WRITE_CONTROL_KEYS, dict) and WRITE_CONTROL_KEYS +@pytest.mark.parametrize("name, keywords", [ + ("click_mouse", {"clicks", "interval"}), + ("tween_drag", {"step_delay_s", "settle_s"}), + ("drag_path", {"step_delay_s", "settle_s"}), +]) +def test_keywords_jeffrey_rpa_passes(name, keywords): + """Keyword parameters added so Jeffrey_RPA can drop its own loops.""" + import inspect + parameters = inspect.signature(getattr(je_auto_control, name)).parameters + assert keywords <= set(parameters), (name, keywords - set(parameters)) + + def test_gui_widget_pybreeze_embeds_is_still_there(): """Read, not imported: importing it needs PySide6, which CI lacks here.""" source = (REPO_ROOT / "je_auto_control" / "gui" / "main_widget.py" @@ -183,6 +231,8 @@ def test_key_tables_jeffrey_rpa_reads(): for name in ("up", "down", "left", "right", "space", "tab", "shift", "a", "z", "0", "9", "f1", "f12"): assert name in keyboard, name + # Reverse lookups skip these, so a recorder never writes an alias. + assert isinstance(platform_wrapper.keyboard_key_aliases, dict) @pytest.mark.skipif(not sys.platform.startswith("win"), @@ -199,6 +249,10 @@ def test_windows_key_names_jeffrey_rpa_sends(): for name in ("return", "escape", "control", "menu", "back", "delete", "home", "end", "insert", "capital", "vk_down"): assert name in platform_wrapper.keyboard_keys_table, name + # Added so it can drop its own alias and extra-key tables. + for name in ("ctrl", "alt", "enter", "esc", "win", "pgup", "numpad0", "plus", + "oem_1", "oem_plus", "oem_102", "oem_clear", "browser_home", "launch_app2"): + assert name in platform_wrapper.keyboard_keys_table, name def test_executor_api_webrunner_calls(): diff --git a/test/unit_test/headless/test_dev_release.py b/test/unit_test/headless/test_dev_release.py index 5d61a2882..af53407a4 100644 --- a/test/unit_test/headless/test_dev_release.py +++ b/test/unit_test/headless/test_dev_release.py @@ -159,7 +159,7 @@ def test_the_workflow_publishes_only_a_tested_push_to_dev(): def test_the_workflow_uploads_only_a_changed_build_and_keeps_no_credentials(): job = _publish_job() upload = job.index("twine upload") - assert job.index("dev_release.py prepare") < job.index("python -m build") < upload + assert job.index("dev_release.py prepare") < job.index("python -m build --no-isolation") < upload assert job.index("dev_release.py changed dist") < upload assert job.index("git ls-remote origin refs/heads/dev") < upload assert "if: steps.compare.outputs.changed == 'true' && steps.tip.outputs.current == 'true'" in job @@ -177,4 +177,8 @@ def test_dev_is_tested_the_way_main_is(): def test_the_dev_package_is_built_with_the_tooling_the_stable_one_is(): - assert _line_with(_publish_job(), "build==") == _line_with(_workflow("stable.yml"), "build==") + # Both jobs install the same hash-locked file and build with the backend it pins + # (test_publish_tooling_lock.py says what those two lines must be). + job, stable = _publish_job(), _workflow("stable.yml") + assert _line_with(job, "--require-hashes") == _line_with(stable, "--require-hashes") + assert _line_with(job, "python -m build") == _line_with(stable, "python -m build") diff --git a/test/unit_test/headless/test_drag_pacing.py b/test/unit_test/headless/test_drag_pacing.py new file mode 100644 index 000000000..80096fde5 --- /dev/null +++ b/test/unit_test/headless/test_drag_pacing.py @@ -0,0 +1,193 @@ +"""Drag pacing (``step_delay_s`` / ``settle_s``) and the release on failure. + +``tween_drag`` and ``drag_path`` share one press / move / release sequence. +Every event goes to a recording sink and every pause to a recording clock, so +no real input is sent and nothing sleeps. No Qt imports. +""" +import importlib +import math +import types + +import pytest + +from je_auto_control.utils.mouse_path import drag_path +from je_auto_control.utils.tween_drag import tween_drag + +tween_module = importlib.import_module("je_auto_control.utils.tween_drag.tween_drag") +path_module = importlib.import_module("je_auto_control.utils.mouse_path.mouse_path") + + +class _Timeline: + """One ordered record of dispatched events and pauses.""" + + def __init__(self, fail_on=None, error=RuntimeError): + self.entries = [] + self.fail_on = fail_on or (lambda event, index: False) + self.error = error + + def sink(self, event): + index = sum(1 for kind, *_ in self.entries if kind == event["op"]) + self.entries.append((event["op"], event.get("x"), event.get("y"))) + if self.fail_on(event, index): + raise self.error(f"{event['op']} #{index} failed") + + def sleep(self, seconds): + self.entries.append(("sleep", seconds, None)) + + def ops(self): + return [kind for kind, *_ in self.entries] + + +@pytest.fixture +def timeline(monkeypatch): + line = _Timeline() + monkeypatch.setattr(tween_module, "time", types.SimpleNamespace(sleep=line.sleep)) + return line + + +def _linear(start, end, **kwargs): + return tween_drag(start, end, steps=2, easing="linear", **kwargs) + + +# --- defaults: the old event sequence, no pauses ------------------------------- + +def test_defaults_dispatch_press_moves_release_and_never_sleep(timeline): + out = _linear((0, 0), (10, 0), sink=timeline.sink) + assert timeline.entries == [("press", 0, 0), ("move", 0, 0), ("move", 5, 0), + ("move", 10, 0), ("release", 10, 0)] + assert out["points"] == 3 + + +def test_drag_path_defaults_are_unchanged_too(timeline): + drag_path([(0, 0), (4, 0)], per_segment_steps=2, sink=timeline.sink) + assert timeline.ops() == ["press", "move", "move", "move", "release"] + + +# --- pacing ---------------------------------------------------------------------- + +def test_step_delay_pauses_after_every_move(timeline): + _linear((0, 0), (10, 0), sink=timeline.sink, step_delay_s=0.012) + assert timeline.ops() == ["press", "move", "sleep", "move", "sleep", + "move", "sleep", "release"] + assert {seconds for kind, seconds, _ in timeline.entries if kind == "sleep"} == {0.012} + + +def test_settle_rests_on_the_start_and_around_the_press_and_release(timeline): + _linear((0, 0), (10, 0), sink=timeline.sink, settle_s=0.08) + assert timeline.entries == [ + ("move", 0, 0), ("sleep", 0.08, None), # hover on the start + ("press", 0, 0), ("sleep", 0.08, None), # after the press + ("move", 0, 0), ("move", 5, 0), ("move", 10, 0), + ("sleep", 0.08, None), # before the release + ("release", 10, 0)] + + +def test_drag_path_takes_the_same_pacing(timeline): + drag_path([(0, 0), (2, 0), (2, 2)], per_segment_steps=1, sink=timeline.sink, + step_delay_s=0.01, settle_s=0.05) + assert timeline.ops() == ["move", "sleep", "press", "sleep", + "move", "sleep", "move", "sleep", "move", "sleep", + "sleep", "release"] + + +@pytest.mark.parametrize("name", ["step_delay_s", "settle_s"]) +@pytest.mark.parametrize("value", [-0.01, math.nan, math.inf, "soon", None]) +def test_an_invalid_pause_is_refused_before_anything_is_sent(timeline, name, value): + with pytest.raises(ValueError): + _linear((0, 0), (10, 0), sink=timeline.sink, **{name: value}) + with pytest.raises(ValueError): + drag_path([(0, 0), (4, 0)], sink=timeline.sink, **{name: value}) + assert timeline.entries == [] + + +def test_an_empty_path_sends_nothing_even_with_pacing(timeline): + assert drag_path([], sink=timeline.sink, settle_s=1.0)["points"] == 0 + assert timeline.entries == [] + + +# --- the release when a step raises ---------------------------------------------- + +def test_a_failed_move_releases_at_the_last_point_reached(timeline): + timeline.fail_on = lambda event, index: event["op"] == "move" and index == 2 + with pytest.raises(RuntimeError, match="move #2"): + _linear((0, 0), (10, 0), sink=timeline.sink) + # The move to (10, 0) failed: let go at (5, 0), not at the drop target. + assert timeline.entries[-1] == ("release", 5, 0) + assert timeline.ops().count("release") == 1 + + +def test_a_failure_on_the_first_move_releases_where_it_was_pressed(timeline): + timeline.fail_on = lambda event, index: event["op"] == "move" and index == 0 + with pytest.raises(RuntimeError): + drag_path([(3, 4), (9, 9)], per_segment_steps=2, sink=timeline.sink) + assert timeline.entries[-1] == ("release", 3, 4) + + +def test_a_failed_press_releases_nothing(timeline): + timeline.fail_on = lambda event, index: event["op"] == "press" + with pytest.raises(RuntimeError): + _linear((0, 0), (10, 0), sink=timeline.sink) + assert "release" not in timeline.ops() + + +def test_an_interrupted_pause_still_releases(timeline, monkeypatch): + def interrupt(seconds): + timeline.entries.append(("sleep", seconds, None)) + if timeline.ops().count("move") == 2: + raise KeyboardInterrupt + monkeypatch.setattr(tween_module, "time", types.SimpleNamespace(sleep=interrupt)) + with pytest.raises(KeyboardInterrupt): + _linear((0, 0), (10, 0), sink=timeline.sink, step_delay_s=0.01) + assert timeline.entries[-1] == ("release", 5, 0) + + +def test_a_failing_cleanup_release_does_not_hide_the_original_error(timeline): + def fail(event, index): + return (event["op"] == "move" and index == 1) or event["op"] == "release" + timeline.fail_on = fail + with pytest.raises(RuntimeError, match="move #1"): + _linear((0, 0), (10, 0), sink=timeline.sink) + assert timeline.entries[-1] == ("release", 0, 0) + + +def test_a_failing_final_release_is_raised_and_retried_once(timeline): + timeline.fail_on = lambda event, index: event["op"] == "release" and index == 0 + with pytest.raises(RuntimeError, match="release #0"): + _linear((0, 0), (10, 0), sink=timeline.sink) + assert timeline.entries[-2:] == [("release", 10, 0), ("release", 10, 0)] + + +# --- wiring ---------------------------------------------------------------------- + +def test_the_executor_adapters_forward_the_pacing(timeline, monkeypatch): + from je_auto_control.utils.executor import action_executor + monkeypatch.setattr(tween_module, "_default_sink", timeline.sink) + monkeypatch.setattr(path_module, "_default_sink", timeline.sink) + action_executor._tween_drag([0, 0], [10, 0], steps=1, easing="linear", + step_delay_s=0.02, settle_s="0.03") + assert ("sleep", 0.02, None) in timeline.entries + assert ("sleep", 0.03, None) in timeline.entries + timeline.entries.clear() + action_executor._drag_path([[0, 0], [5, 5]], per_segment_steps=1, step_delay_s=0.04) + assert ("sleep", 0.04, None) in timeline.entries + + +def test_the_mcp_handlers_forward_the_pacing(timeline, monkeypatch): + from je_auto_control.utils.mcp_server.tools import _handlers, _handlers_executor_bridge + monkeypatch.setattr(tween_module, "_default_sink", timeline.sink) + monkeypatch.setattr(path_module, "_default_sink", timeline.sink) + assert _handlers.tween_drag([0, 0], [10, 0], steps=1, settle_s=0.05) == {"points": 2} + assert ("sleep", 0.05, None) in timeline.entries + timeline.entries.clear() + _handlers_executor_bridge.drag_path([[0, 0], [5, 5]], per_segment_steps=1, step_delay_s=0.06) + assert ("sleep", 0.06, None) in timeline.entries + + +def test_the_mcp_tools_and_script_builder_declare_the_pacing(): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + from je_auto_control.gui.script_builder.command_schema import _build_specs + tools = {tool.name: tool for tool in build_default_tool_registry()} + specs = {spec.command: spec for spec in _build_specs()} + for tool, command in (("ac_tween_drag", "AC_tween_drag"), ("ac_drag_path", "AC_drag_path")): + assert {"step_delay_s", "settle_s"} <= set(tools[tool].input_schema["properties"]) + assert {"step_delay_s", "settle_s"} <= {field.name for field in specs[command].fields} diff --git a/test/unit_test/headless/test_key_names_and_aliases.py b/test/unit_test/headless/test_key_names_and_aliases.py new file mode 100644 index 000000000..5b90a98d0 --- /dev/null +++ b/test/unit_test/headless/test_key_names_and_aliases.py @@ -0,0 +1,151 @@ +"""Key aliases, the added Windows keys, and the canonical reverse lookup. + +``keyboard_key_name`` answers "which name does this virtual key have?" for +recorders. Aliases resolve to the same code but must never be the answer, or +adding one would change the key names a recording writes. Reads tables only; +no key is sent. No Qt imports. +""" +import re +import sys + +import pytest + +import je_auto_control as ac +from je_auto_control.wrapper import auto_control_keyboard +from je_auto_control.wrapper import platform_wrapper + +windows_only = pytest.mark.skipif(not sys.platform.startswith("win"), + reason="the Win32 virtual-key table loads on Windows only") + +#: Microsoft "Virtual-Key Codes (Winuser.h)" for the keys added here. +_ADDED_WINDOWS_KEYS = { + "browser_home": 0xAC, "launch_app2": 0xB7, + "oem_1": 0xBA, "oem_plus": 0xBB, "oem_comma": 0xBC, "oem_minus": 0xBD, + "oem_period": 0xBE, "oem_2": 0xBF, "oem_3": 0xC0, "oem_4": 0xDB, + "oem_5": 0xDC, "oem_6": 0xDD, "oem_7": 0xDE, "oem_8": 0xDF, + "oem_102": 0xE2, "oem_clear": 0xFE, +} + +#: What a recorder must keep writing for keys that gained aliases: the name a +#: shortest-lower-case lookup picked before any alias existed. +_CANONICAL_BEFORE_ALIASES = { + 0x11: "control", 0xA2: "lcontrol", 0xA3: "rcontrol", + 0x12: "menu", 0xA4: "lmenu", 0xA5: "rmenu", + 0x0D: "return", 0x1B: "escape", 0x5B: "lwin", 0x08: "back", + 0x2E: "delete", 0x2D: "insert", 0x21: "prior", 0x22: "next", + 0x14: "capital", 0x2C: "snapshot", 0x91: "scroll", + **{0x60 + digit: f"num{digit}" for digit in range(10)}, + 0x28: "down", 0x41: "a", +} + + +def _shortest_lower_case(table, code, skip=()): + names = [name for name, value in table.items() if value == code and name not in skip] + return min(names, key=lambda name: (name != name.lower(), len(name), name), default=None) + + +# --- the Windows table ------------------------------------------------------- + +@windows_only +@pytest.mark.parametrize("name, code", sorted(_ADDED_WINDOWS_KEYS.items())) +def test_the_added_keys_carry_microsofts_codes(name, code): + assert platform_wrapper.keyboard_keys_table[name] == code + + +@windows_only +def test_the_upper_case_launch_app2_spelling_stays(): + assert platform_wrapper.keyboard_keys_table["LAUNCH_APP2"] == 0xB7 + + +@windows_only +def test_every_alias_resolves_to_its_targets_code(): + table = platform_wrapper.keyboard_keys_table + aliases = platform_wrapper.keyboard_key_aliases + assert len(aliases) >= 40, "the alias map is empty; the checks below would pass vacuously" + for alias, target in aliases.items(): + assert target in table, (alias, target) + assert target not in aliases, (alias, target) + assert table[alias] == table[target], alias + assert auto_control_keyboard._resolve_keycode(alias) == table[target], alias + + +@windows_only +@pytest.mark.parametrize("alias, target", [ + ("ctrl", "control"), ("alt", "menu"), ("enter", "return"), ("esc", "escape"), + ("win", "lwin"), ("backspace", "back"), ("del", "delete"), ("pgup", "prior"), + ("pagedown", "next"), ("capslock", "capital"), ("prtsc", "snapshot"), + ("numpad5", "num5"), ("plus", "oem_plus"), ("period", "oem_period"), +]) +def test_common_aliases(alias, target): + assert platform_wrapper.keyboard_key_aliases[alias] == target + + +@windows_only +def test_every_name_is_typable_except_the_legacy_capitals(): + """Callers lower-case what a user types; every added name survives that.""" + typable = re.compile(r"[a-z0-9_]+") + added = set(_ADDED_WINDOWS_KEYS) | set(platform_wrapper.keyboard_key_aliases) + assert all(typable.fullmatch(name) for name in added) + + +@windows_only +def test_layout_dependent_oem_keys_get_no_layout_specific_alias(): + """``oem_1`` is ";" on a US layout only; a name like "semicolon" would lie.""" + layout_bound = {0xBA, 0xBF, 0xC0, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE2} + table = platform_wrapper.keyboard_keys_table + for alias in platform_wrapper.keyboard_key_aliases: + assert table[alias] not in layout_bound, alias + + +# --- reverse lookup: canonical names only ---------------------------------------- + +@windows_only +@pytest.mark.parametrize("code, name", sorted(_CANONICAL_BEFORE_ALIASES.items())) +def test_reverse_lookup_keeps_the_pre_alias_name(code, name): + assert ac.keyboard_key_name(code) == name + + +@windows_only +def test_no_alias_changes_any_reverse_lookup(): + """For every code, the answer is the old lookup over the table minus aliases.""" + table = platform_wrapper.keyboard_keys_table + aliases = platform_wrapper.keyboard_key_aliases + for code in set(table.values()): + assert ac.keyboard_key_name(code) == _shortest_lower_case(table, code, aliases), hex(code) + + +@windows_only +def test_reverse_lookup_names_the_added_keys(): + for name, code in _ADDED_WINDOWS_KEYS.items(): + assert ac.keyboard_key_name(code) == name + assert ac.keyboard_key_name(0xFC) is None # VK_NONAME: no entry + + +def test_the_preference_order_on_a_synthetic_table(monkeypatch): + """Platform-independent: aliases lose, then untypable names, then length, then order.""" + monkeypatch.setattr(auto_control_keyboard, "keyboard_keys_table", { + "esc": 27, "escape": 27, # alias vs canonical + "\b": 8, "backspace": 8, # untypable vs typable + "LAUNCH_APP2": 183, "launch_app2": 183, + "vk_down": 40, "down": 40, # length + "next": 34, "pgdn": 34, # same length: alphabetical + "only_alias": 99, + }) + monkeypatch.setattr(auto_control_keyboard, "keyboard_key_aliases", + {"esc": "escape", "pgdn": "next", "only_alias": "x"}) + name = auto_control_keyboard.keyboard_key_name + assert [name(27), name(8), name(183), name(40), name(34)] == [ + "escape", "backspace", "launch_app2", "down", "next"] + assert name(99) is None + assert name(12345) is None + + +def test_every_platform_publishes_an_alias_map(): + aliases = platform_wrapper.keyboard_key_aliases + assert isinstance(aliases, dict) + assert all(alias in platform_wrapper.keyboard_keys_table for alias in aliases) + + +def test_facade_export(): + assert "keyboard_key_name" in ac.__all__ + assert callable(ac.keyboard_key_name) diff --git a/test/unit_test/headless/test_match_rbac_bundle_audit.py b/test/unit_test/headless/test_match_rbac_bundle_audit.py index bdd67a789..64c1712a3 100644 --- a/test/unit_test/headless/test_match_rbac_bundle_audit.py +++ b/test/unit_test/headless/test_match_rbac_bundle_audit.py @@ -125,7 +125,8 @@ def sink(event): with pytest.raises(KeyboardInterrupt): drag_path([[0, 0], [100, 0], [100, 100]], per_segment_steps=4, sink=sink) - assert events[-1] == {"op": "release", "button": "mouse_left"} + # Let go at the last point the pointer reached, not at the end of the path. + assert events[-1] == {"op": "release", "button": "mouse_left", "x": 0, "y": 0} completed = [] drag_path([[0, 0], [10, 0]], per_segment_steps=2, sink=completed.append) assert completed[-1] == {"op": "release", "button": "mouse_left", "x": 10, "y": 0} diff --git a/test/unit_test/headless/test_ocr_tesseract_setup.py b/test/unit_test/headless/test_ocr_tesseract_setup.py new file mode 100644 index 000000000..9cd94286b --- /dev/null +++ b/test/unit_test/headless/test_ocr_tesseract_setup.py @@ -0,0 +1,270 @@ +"""Tesseract discovery, configuration and health checks (``ocr/tesseract_setup``). + +A stand-in ``pytesseract`` replaces the real one, so no engine is run and no +screen is read. No Qt imports. +""" +import os +import subprocess # nosec B404 # reason: only CalledProcessError is used, as a fake engine failure +import sys +import types + +import pytest + +import je_auto_control as ac +from je_auto_control.utils.exception.exceptions import AutoControlActionException +from je_auto_control.utils.ocr import tesseract_setup as setup +from je_auto_control.utils.ocr.backends import tesseract_backend +from je_auto_control.utils.ocr.backends.base import OCRBackendNotAvailableError +from je_auto_control.utils.ocr.backends.tesseract_backend import TesseractBackend + + +class _NotFound(OSError): + """What pytesseract raises when the executable cannot be started.""" + + +class _FakePytesseract: + """The slice of ``pytesseract`` the setup helpers reach.""" + + def __init__(self, languages=("eng",), cmd="tesseract"): + self.pytesseract = types.SimpleNamespace(tesseract_cmd=cmd) + self.languages = list(languages) + self.version_error = None + self.languages_error = None + self.language_calls = [] + + def get_tesseract_version(self): + if self.version_error is not None: + raise self.version_error + return "5.3.0" + + def get_languages(self, config=""): + self.language_calls.append(config) + if self.languages_error is not None: + raise self.languages_error + return list(self.languages) + + +@pytest.fixture +def fake(monkeypatch, tmp_path): + """A fake engine whose command is an existing file.""" + engine = tmp_path / "tesseract.exe" + engine.write_bytes(b"") + module = _FakePytesseract(languages=("jpn", "eng", "chi_tra"), cmd=str(engine)) + monkeypatch.setattr(tesseract_backend, "_pytesseract", module) + return module + + +@pytest.fixture +def no_pytesseract(monkeypatch): + monkeypatch.setattr(tesseract_backend, "_pytesseract", None) + monkeypatch.setitem(sys.modules, "pytesseract", None) # import -> ImportError + + +# --- ocr_languages: None and [] are different answers ----------------------- + +def test_languages_are_sorted_and_asked_with_an_empty_config(fake): + assert setup.ocr_languages() == ["chi_tra", "eng", "jpn"] + assert fake.language_calls == [""] + + +def test_an_engine_with_no_language_data_answers_an_empty_list(fake): + fake.languages = [] + assert setup.ocr_languages() == [] + + +def test_languages_are_none_when_the_package_is_missing(no_pytesseract): + assert setup.ocr_languages() is None + + +def test_languages_are_none_when_the_engine_cannot_be_asked(fake): + fake.languages_error = _NotFound("tesseract is not installed") + assert setup.ocr_languages() is None + + +def test_languages_are_asked_afresh_on_every_call(fake): + """Language data added while the process runs must be seen.""" + fake.languages = [] + assert setup.ocr_languages() == [] + fake.languages = ["eng"] + assert setup.ocr_languages() == ["eng"] + assert len(fake.language_calls) == 2 + + +# --- ocr_status -------------------------------------------------------------- + +def test_status_ready(fake): + status = setup.ocr_status() + assert status == (True, setup.OCR_READY) + ok, reason = status + assert ok is True + assert reason == "ready" + assert status.ok is True + assert status.reason == "ready" + + +def test_status_missing_package(no_pytesseract): + assert setup.ocr_status() == (False, setup.OCR_MISSING_PACKAGE) + + +def test_status_missing_engine(fake, tmp_path): + fake.pytesseract.tesseract_cmd = str(tmp_path / "absent" / "tesseract.exe") + assert setup.ocr_status() == (False, setup.OCR_MISSING_ENGINE) + + +@pytest.mark.parametrize("error", [ + _NotFound("wrong architecture"), + subprocess.CalledProcessError(1, ["tesseract", "--version"]), + UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), +], ids=["os-error", "non-zero-exit", "undecodable-output"]) +def test_status_engine_unusable(fake, error): + fake.version_error = error + assert setup.ocr_status() == (False, setup.OCR_ENGINE_UNUSABLE) + + +def test_an_unreadable_version_does_not_end_the_process(fake): + """pytesseract raises SystemExit for a version string it cannot parse.""" + fake.version_error = SystemExit('Invalid tesseract version: "garbage"') + assert setup.ocr_status() == (False, setup.OCR_ENGINE_UNUSABLE) + backend = TesseractBackend() + with pytest.raises(OCRBackendNotAvailableError): + backend.version() + + +def test_status_no_language_data_only_when_the_engine_answered_empty(fake): + fake.languages = [] + assert setup.ocr_status() == (False, setup.OCR_NO_LANGUAGE_DATA) + # An engine that runs but cannot list is not "no language data". + fake.languages_error = _NotFound("listing failed") + assert setup.ocr_status() == (True, setup.OCR_READY) + + +def test_status_finds_a_bare_command_on_path(fake, monkeypatch): + fake.pytesseract.tesseract_cmd = "tesseract" + monkeypatch.setattr(setup, "shutil", types.SimpleNamespace( + which=lambda name: r"C:\bin\tesseract.exe" if name == "tesseract" else None)) + assert setup.ocr_status() == (True, setup.OCR_READY) + + +# --- the engine command has one writer --------------------------------------- + +def test_set_tesseract_cmd_is_what_status_checks(fake, tmp_path): + other = tmp_path / "other-tesseract.exe" + other.write_bytes(b"") + ac.set_tesseract_cmd(str(other)) + assert fake.pytesseract.tesseract_cmd == str(other) + assert TesseractBackend().cmd == str(other) + assert setup.ocr_status().ok is True + + +# --- set_tessdata_dir ---------------------------------------------------------- + +def test_set_tessdata_dir_sets_the_resolved_directory(monkeypatch, tmp_path): + monkeypatch.delenv(setup.TESSDATA_ENV, raising=False) + data = tmp_path / "tessdata" + data.mkdir() + resolved = setup.set_tessdata_dir(data) + assert resolved == os.path.realpath(data) + assert os.environ[setup.TESSDATA_ENV] == resolved + + +def test_set_tessdata_dir_none_restores_the_engine_default(monkeypatch, tmp_path): + monkeypatch.setenv(setup.TESSDATA_ENV, str(tmp_path)) + assert setup.set_tessdata_dir(None) is None + assert setup.TESSDATA_ENV not in os.environ + + +def test_set_tessdata_dir_refuses_what_is_not_a_directory(monkeypatch, tmp_path): + monkeypatch.setenv(setup.TESSDATA_ENV, "kept") + a_file = tmp_path / "eng.traineddata" + a_file.write_bytes(b"") + for bad in (tmp_path / "absent", a_file): + with pytest.raises(AutoControlActionException): + setup.set_tessdata_dir(bad) + assert os.environ[setup.TESSDATA_ENV] == "kept" + + +# --- find_tesseract_cmd ---------------------------------------------------------- + +def _no_path(monkeypatch): + monkeypatch.setattr(setup, "shutil", types.SimpleNamespace(which=lambda name: None)) + + +def test_find_prefers_the_environment_override(monkeypatch, tmp_path): + engine = tmp_path / "custom-tesseract.exe" + engine.write_bytes(b"") + monkeypatch.setenv(setup.TESSERACT_CMD_ENV, f" {engine} ") + monkeypatch.setattr(setup, "shutil", types.SimpleNamespace( + which=lambda name: pytest.fail("PATH must not be searched"))) + assert setup.find_tesseract_cmd() == os.path.abspath(engine) + + +def test_find_ignores_an_override_naming_no_file(monkeypatch, tmp_path): + monkeypatch.setenv(setup.TESSERACT_CMD_ENV, str(tmp_path / "absent.exe")) + found = tmp_path / "on-path" / "tesseract.exe" + monkeypatch.setattr(setup, "shutil", types.SimpleNamespace(which=lambda name: str(found))) + assert setup.find_tesseract_cmd() == os.path.abspath(found) + + +def test_find_falls_back_to_the_install_locations(monkeypatch, tmp_path): + monkeypatch.delenv(setup.TESSERACT_CMD_ENV, raising=False) + _no_path(monkeypatch) + installed = tmp_path / "Tesseract-OCR" / "tesseract.exe" + installed.parent.mkdir() + installed.write_bytes(b"") + monkeypatch.setattr(setup, "_install_candidates", + lambda: [str(tmp_path / "missing.exe"), str(installed)]) + assert setup.find_tesseract_cmd() == str(installed) + + +def test_find_returns_none_when_nothing_is_installed(monkeypatch, tmp_path): + monkeypatch.delenv(setup.TESSERACT_CMD_ENV, raising=False) + _no_path(monkeypatch) + monkeypatch.setattr(setup, "_install_candidates", lambda: [str(tmp_path / "missing.exe")]) + assert setup.find_tesseract_cmd() is None + + +def test_install_locations_per_platform(monkeypatch): + monkeypatch.setenv("ProgramFiles", r"D:\Apps") + monkeypatch.delenv("ProgramFiles(x86)", raising=False) + monkeypatch.setenv("LOCALAPPDATA", r"D:\Users\me\AppData\Local") + windows = setup._install_candidates("win32") + assert os.path.join(r"D:\Apps", "Tesseract-OCR", "tesseract.exe") in windows + assert os.path.join(r"C:\Program Files (x86)", "Tesseract-OCR", "tesseract.exe") in windows + assert os.path.join(r"D:\Users\me\AppData\Local", "Programs", "Tesseract-OCR", + "tesseract.exe") in windows + assert "/opt/homebrew/bin/tesseract" in setup._install_candidates("darwin") + assert "/usr/bin/tesseract" in setup._install_candidates("linux") + assert "/usr/bin/tesseract" in setup._install_candidates("freebsd14") + + +# --- wiring ------------------------------------------------------------------------ + +def test_executor_commands_return_json_friendly_dicts(fake): + from je_auto_control.utils.executor.action_executor import executor + assert executor.event_dict["AC_ocr_status"]() == {"ok": True, "reason": "ready"} + assert executor.event_dict["AC_ocr_languages"]() == {"languages": ["chi_tra", "eng", "jpn"]} + fake.languages_error = _NotFound("listing failed") + assert executor.event_dict["AC_ocr_languages"]() == {"languages": None} + + +def test_mcp_tools_and_script_builder_specs(): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + tools = {tool.name: tool for tool in build_default_tool_registry()} + assert {"ac_ocr_status", "ac_ocr_languages"} <= set(tools) + assert tools["ac_ocr_status"].annotations.read_only is True + from je_auto_control.gui.script_builder.command_schema import _build_specs + assert {"AC_ocr_status", "AC_ocr_languages"} <= {s.command for s in _build_specs()} + + +def test_mcp_handlers_return_json_friendly_dicts(fake): + from je_auto_control.utils.mcp_server.tools import _handlers_screen + assert _handlers_screen.ocr_status() == {"ok": True, "reason": "ready"} + fake.languages = [] + assert _handlers_screen.ocr_languages() == {"languages": []} + + +def test_facade_exports(): + for name in ("OCRStatus", "find_tesseract_cmd", "ocr_languages", "ocr_status", + "set_tessdata_dir"): + assert hasattr(ac, name), name + assert name in ac.__all__, name diff --git a/test/unit_test/headless/test_publish_tooling_lock.py b/test/unit_test/headless/test_publish_tooling_lock.py new file mode 100644 index 000000000..c6e955131 --- /dev/null +++ b/test/unit_test/headless/test_publish_tooling_lock.py @@ -0,0 +1,150 @@ +"""The jobs that hold the PyPI token install and build with hash-locked tooling only. + +``publish`` in ``stable.yml`` and ``publish-dev`` in ``dev.yml`` receive ``secrets.PYPI_API_TOKEN``, +so whatever they install runs next to it. Both install one file, ``.github/requirements/publish.txt``: +wheels only, at recorded hashes, generated from ``publish.in`` beside it. They build with +``python -m build --no-isolation``, so the build backend is the locked ``setuptools`` as well, not the +newest one downloaded into a fresh build environment. + +Everything here is read from the files as text; nothing is installed or built. +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +import pytest +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10, where pytest itself depends on tomli + tomllib = pytest.importorskip("tomli") + +REPO_ROOT = Path(__file__).resolve().parents[3] +WORKFLOWS = REPO_ROOT / ".github" / "workflows" +REQUIREMENTS = REPO_ROOT / ".github" / "requirements" +# The metadata files a publish job can build from: scripts/dev_release.py writes dev.toml over pyproject.toml. +METADATA = ("pyproject.toml", "dev.toml") +LOCKED_INSTALL = "python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt" + +_JOB_NAME = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$", re.MULTILINE) +_PIP_INSTALL = re.compile(r"(?:python3? -m )?\bpip3? install\b.*") +_BUILD = re.compile(r"\bpython3? -m build\b.*") +_RUN_OR_IMPORT = re.compile(r"python3? -m ([A-Za-z_]\w*)|^\s*import ([A-Za-z_]\w*)", re.MULTILINE) +_PIN = re.compile(r"^([A-Za-z0-9][\w.-]*)==(\S+)", re.MULTILINE) + + +def _jobs(workflow: Path) -> list[tuple[str, str]]: + """Return ``(job name, job text without comment lines)`` for each job of a workflow.""" + jobs = workflow.read_text(encoding="utf-8").split("\njobs:\n", 1)[1] + names = list(_JOB_NAME.finditer(jobs)) + ends = [name.start() for name in names[1:]] + [len(jobs)] + return [(name.group(1), _without_comments(jobs[name.end():end])) for name, end in zip(names, ends)] + + +def _without_comments(text: str) -> str: + return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) + + +def _token_jobs() -> list[tuple[str, str]]: + """Return ``(workflow:job, job text)`` for each job that reads the PyPI token.""" + return [(f"{workflow.name}:{name}", body) + for workflow in sorted(WORKFLOWS.glob("*.yml")) + for name, body in _jobs(workflow) + if "secrets.PYPI_API_TOKEN" in body] + + +TOKEN_JOBS = _token_jobs() +token_job = pytest.mark.parametrize( + "body", [body for _name, body in TOKEN_JOBS], ids=[name for name, _body in TOKEN_JOBS]) + + +def _named_in_publish_in() -> set[str]: + """Return the distributions ``publish.in`` names, one at the start of each line that is not a comment.""" + text = (REQUIREMENTS / "publish.in").read_text(encoding="utf-8") + return {canonicalize_name(found) for found in re.findall(r"^([A-Za-z0-9][\w.-]*)", text, re.MULTILINE)} + + +def _locked() -> dict[str, str]: + """Return ``{distribution: version}`` for every pin in ``publish.txt``.""" + text = (REQUIREMENTS / "publish.txt").read_text(encoding="utf-8") + return {canonicalize_name(name): version for name, version in _PIN.findall(text)} + + +def _build_requires(metadata: str) -> list[Requirement]: + with (REPO_ROOT / metadata).open("rb") as handle: + return [Requirement(item) for item in tomllib.load(handle)["build-system"]["requires"]] + + +def _tools(body: str) -> set[str]: + """Return what a job runs with ``python -m`` or imports in an inline script, less pip and the stdlib.""" + named = {module or imported for module, imported in _RUN_OR_IMPORT.findall(body)} + return {canonicalize_name(name) for name in named - {"pip"} - set(sys.stdlib_module_names)} + + +def test_the_jobs_that_hold_the_pypi_token_are_the_two_publish_jobs(): + # A third job that gains the token is covered by the tests below, but should be a decision. + assert [name for name, _body in TOKEN_JOBS] == ["dev.yml:publish-dev", "stable.yml:publish"] + + +@token_job +def test_a_job_with_the_pypi_token_installs_only_the_hash_locked_tooling(body): + # An unpinned install, a second install or a pip upgrade takes whatever was uploaded that day. + assert [command.strip().strip("\"'") for command in _PIP_INSTALL.findall(body)] == [LOCKED_INSTALL] + + +@token_job +def test_a_job_with_the_pypi_token_builds_with_the_locked_backend(body): + # An isolated build downloads the newest setuptools each time, outside the lock. + builds = _BUILD.findall(body) + assert builds, "the job no longer runs python -m build: update this guard" + assert all("--no-isolation" in command for command in builds) + + +@pytest.mark.parametrize("metadata", METADATA) +def test_the_lock_satisfies_build_system_requires(metadata): + # --no-isolation checks the requirement instead of installing it, so a floor raised without + # regenerating the lock has to fail here, not in the job that is about to upload. + locked = _locked() + requires = _build_requires(metadata) + assert requires, f"{metadata} names no build backend" + for requirement in requires: + name = canonicalize_name(requirement.name) + assert name in locked, f"{metadata} needs {requirement}, which publish.txt does not pin" + assert requirement.specifier.contains(locked[name], prereleases=True), ( + f"{metadata} needs {requirement}, publish.txt pins {locked[name]}: regenerate the lock") + + +def test_publish_in_names_the_tools_the_jobs_run_and_the_build_backend(): + # A tool a job starts using has to be locked first, or the release fails at that step. + used = set().union(*(_tools(body) for _name, body in TOKEN_JOBS)) + backend = {canonicalize_name(requirement.name) + for metadata in METADATA for requirement in _build_requires(metadata)} + assert _named_in_publish_in() == used | backend + + +def test_the_lock_pins_everything_publish_in_names(): + # publish.txt is generated; editing publish.in alone changes nothing the jobs install. + assert _named_in_publish_in() <= set(_locked()) + + +def test_the_lock_is_resolved_for_the_python_the_jobs_set_up(): + # The lock holds the wheels of one Python version; a job on another one may find none that match. + header = (REQUIREMENTS / "publish.txt").read_text(encoding="utf-8").splitlines()[1] + assert "--generate-hashes" in header + assert "--only-binary :all:" in header + locked_for = re.search(r"--python-version (\S+)", header).group(1) + set_up = {version for _name, body in TOKEN_JOBS + for version in re.findall(r"python-version:\s*\"([^\"]+)\"", body)} + assert set_up == {locked_for} + + +def test_dependabot_watches_the_hash_locked_requirements(): + # From "/" Dependabot does not look as deep as .github/requirements, so the lock would never be updated. + text = (REPO_ROOT / ".github" / "dependabot.yml").read_text(encoding="utf-8") + entries = re.split(r"^\s*-\s*package-ecosystem:", text, flags=re.MULTILINE)[1:] + pip = next(entry for entry in entries if entry.split()[0].strip("\"'") == "pip") + assert set(re.findall(r"^\s*-\s*\"(/[^\"]*)\"", pip, re.MULTILINE)) == {"/", "/.github/requirements"} diff --git a/test/unit_test/headless/test_report_path.py b/test/unit_test/headless/test_report_path.py new file mode 100644 index 000000000..a47716f4e --- /dev/null +++ b/test/unit_test/headless/test_report_path.py @@ -0,0 +1,85 @@ +"""Under TestPioneer a report asked for under a relative name goes below ``TEST_PIONEER_ARTIFACT_DIR``.""" +from pathlib import Path + +import pytest + +from je_auto_control.utils.generate_report.generate_html_report import generate_html_report +from je_auto_control.utils.generate_report.generate_json_report import generate_json_report +from je_auto_control.utils.generate_report.generate_xml_report import generate_xml_report +from je_auto_control.utils.generate_report.report_path import ARTIFACT_DIR_VARIABLE, report_path +from je_auto_control.utils.test_record.record_test_class import test_record_instance + + +@pytest.fixture(autouse=True) +def _records(): + saved = list(test_record_instance.test_record_list) + test_record_instance.test_record_list[:] = [ + {"function_name": "ok_func", "local_param": None, "time": "t", "program_exception": "None"}, + {"function_name": "bad_func", "local_param": None, "time": "t", "program_exception": "boom"}, + ] + yield + test_record_instance.test_record_list[:] = saved + + +@pytest.fixture +def artifact_dir(tmp_path, monkeypatch): + """A TestPioneer run: the variable is set, and the working directory is somewhere else.""" + directory = tmp_path / "artifacts" / "run-1" / "runners" / "gui-runner" / "01-gui" + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + monkeypatch.setenv(ARTIFACT_DIR_VARIABLE, str(directory)) + return directory + + +def test_a_name_is_returned_as_given_outside_testpioneer(monkeypatch): + monkeypatch.delenv(ARTIFACT_DIR_VARIABLE, raising=False) + assert report_path("reports/gui") == "reports/gui" + + +def test_an_empty_variable_counts_as_unset(monkeypatch): + monkeypatch.setenv(ARTIFACT_DIR_VARIABLE, " ") + assert report_path("reports/gui") == "reports/gui" + + +def test_a_relative_name_goes_below_the_artifact_directory(artifact_dir): + target = Path(report_path("reports/gui_success.json")) + assert target == artifact_dir.resolve() / "reports" / "gui_success.json" + assert target.parent.is_dir() + + +def test_an_absolute_name_is_left_alone(artifact_dir, tmp_path): + name = str(tmp_path / "elsewhere" / "gui.json") + assert report_path(name) == name + assert not artifact_dir.exists() + + +def test_a_name_that_leaves_the_directory_is_left_alone(artifact_dir): + assert report_path("../outside.json") == "../outside.json" + assert not artifact_dir.exists() + + +def test_a_directory_that_cannot_be_created_falls_back_to_the_name(tmp_path, monkeypatch): + not_a_directory = tmp_path / "file" + not_a_directory.write_text("x", encoding="utf-8") + monkeypatch.setenv(ARTIFACT_DIR_VARIABLE, str(not_a_directory)) + assert report_path("reports/gui.json") == "reports/gui.json" + + +@pytest.mark.parametrize("generate, written", [ + (generate_json_report, ["reports/gui_success.json", "reports/gui_failure.json"]), + (generate_xml_report, ["reports/gui_success.xml", "reports/gui_failure.xml"]), + (generate_html_report, ["reports/gui.html"]), +], ids=["json", "xml", "html"]) +def test_a_report_is_written_below_the_artifact_directory(artifact_dir, generate, written): + generate("reports/gui") + assert [(artifact_dir / path).is_file() for path in written] == [True] * len(written) + # Nothing is left in the working directory, where the name used to point. + assert not Path("reports").exists() + + +def test_outside_testpioneer_a_report_stays_in_the_working_directory(tmp_path, monkeypatch): + monkeypatch.delenv(ARTIFACT_DIR_VARIABLE, raising=False) + monkeypatch.chdir(tmp_path) + generate_json_report("gui") + assert (tmp_path / "gui_success.json").is_file() diff --git a/uv.lock b/uv.lock index bd9be686f..ffe46599d 100644 --- a/uv.lock +++ b/uv.lock @@ -364,7 +364,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -451,7 +451,7 @@ wheels = [ [[package]] name = "je-auto-control" -version = "0.0.221" +version = "0.0.224" source = { editable = "." } dependencies = [ { name = "cryptography", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, @@ -1273,165 +1273,166 @@ version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyobjc-core" }, - { name = "pyobjc-framework-accessibility", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-accounts", marker = "platform_release >= '12.0'" }, + { name = "pyobjc-framework-accessibility", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-accounts", marker = "platform_release >= '12'" }, { name = "pyobjc-framework-addressbook" }, - { name = "pyobjc-framework-adservices", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-adsupport", marker = "platform_release >= '18.0'" }, + { name = "pyobjc-framework-adservices", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-adsupport", marker = "platform_release >= '18'" }, { name = "pyobjc-framework-applescriptkit" }, - { name = "pyobjc-framework-applescriptobjc", marker = "platform_release >= '10.0'" }, + { name = "pyobjc-framework-applescriptobjc", marker = "platform_release >= '10'" }, { name = "pyobjc-framework-applicationservices" }, - { name = "pyobjc-framework-apptrackingtransparency", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-arkit", marker = "platform_release >= '25.0'" }, - { name = "pyobjc-framework-audiovideobridging", marker = "platform_release >= '12.0'" }, - { name = "pyobjc-framework-authenticationservices", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-automaticassessmentconfiguration", marker = "platform_release >= '19.0'" }, + { name = "pyobjc-framework-apptrackingtransparency", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-arkit", marker = "platform_release >= '25'" }, + { name = "pyobjc-framework-audiovideobridging", marker = "platform_release >= '12'" }, + { name = "pyobjc-framework-authenticationservices", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-automaticassessmentconfiguration", marker = "platform_release >= '19'" }, { name = "pyobjc-framework-automator" }, - { name = "pyobjc-framework-avfoundation", marker = "platform_release >= '11.0'" }, - { name = "pyobjc-framework-avkit", marker = "platform_release >= '13.0'" }, - { name = "pyobjc-framework-avrouting", marker = "platform_release >= '22.0'" }, - { name = "pyobjc-framework-backgroundassets", marker = "platform_release >= '22.0'" }, + { name = "pyobjc-framework-avfoundation", marker = "platform_release >= '11'" }, + { name = "pyobjc-framework-avkit", marker = "platform_release >= '13'" }, + { name = "pyobjc-framework-avrouting", marker = "platform_release >= '22'" }, + { name = "pyobjc-framework-backgroundassets", marker = "platform_release >= '22'" }, { name = "pyobjc-framework-browserenginekit", marker = "platform_release >= '23.4'" }, - { name = "pyobjc-framework-businesschat", marker = "platform_release >= '18.0'" }, - { name = "pyobjc-framework-calendarstore", marker = "platform_release >= '9.0'" }, - { name = "pyobjc-framework-callkit", marker = "platform_release >= '20.0'" }, + { name = "pyobjc-framework-businesschat", marker = "platform_release >= '18'" }, + { name = "pyobjc-framework-calendarstore", marker = "platform_release >= '9'" }, + { name = "pyobjc-framework-callkit", marker = "platform_release >= '20'" }, { name = "pyobjc-framework-carbon" }, { name = "pyobjc-framework-cfnetwork" }, - { name = "pyobjc-framework-cinematic", marker = "platform_release >= '23.0'" }, - { name = "pyobjc-framework-classkit", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-cloudkit", marker = "platform_release >= '14.0'" }, - { name = "pyobjc-framework-cocoa" }, - { name = "pyobjc-framework-collaboration", marker = "platform_release >= '9.0'" }, - { name = "pyobjc-framework-colorsync", marker = "platform_release >= '17.0'" }, - { name = "pyobjc-framework-compositorservices", marker = "platform_release >= '25.0'" }, - { name = "pyobjc-framework-contacts", marker = "platform_release >= '15.0'" }, - { name = "pyobjc-framework-contactsui", marker = "platform_release >= '15.0'" }, + { name = "pyobjc-framework-cinematic", marker = "platform_release >= '23'" }, + { name = "pyobjc-framework-classkit", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-cloudkit", marker = "platform_release >= '14'" }, + { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-framework-collaboration", marker = "platform_release >= '9'" }, + { name = "pyobjc-framework-colorsync", marker = "platform_release >= '17'" }, + { name = "pyobjc-framework-compositorservices", marker = "platform_release >= '25'" }, + { name = "pyobjc-framework-contacts", marker = "platform_release >= '15'" }, + { name = "pyobjc-framework-contactsui", marker = "platform_release >= '15'" }, { name = "pyobjc-framework-coreaudio" }, { name = "pyobjc-framework-coreaudiokit" }, - { name = "pyobjc-framework-corebluetooth", marker = "platform_release >= '14.0'" }, + { name = "pyobjc-framework-corebluetooth", marker = "platform_release >= '14'" }, { name = "pyobjc-framework-coredata" }, - { name = "pyobjc-framework-corehaptics", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-corelocation", marker = "platform_release >= '10.0'" }, - { name = "pyobjc-framework-coremedia", marker = "platform_release >= '11.0'" }, - { name = "pyobjc-framework-coremediaio", marker = "platform_release >= '11.0'" }, + { name = "pyobjc-framework-corehaptics", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-corelocation", marker = "platform_release >= '10'" }, + { name = "pyobjc-framework-coremedia", marker = "platform_release >= '11'" }, + { name = "pyobjc-framework-coremediaio", marker = "platform_release >= '11'" }, { name = "pyobjc-framework-coremidi" }, - { name = "pyobjc-framework-coreml", marker = "platform_release >= '17.0'" }, - { name = "pyobjc-framework-coremotion", marker = "platform_release >= '19.0'" }, + { name = "pyobjc-framework-coreml", marker = "platform_release >= '17'" }, + { name = "pyobjc-framework-coremotion", marker = "platform_release >= '19'" }, { name = "pyobjc-framework-coreservices" }, - { name = "pyobjc-framework-corespotlight", marker = "platform_release >= '17.0'" }, + { name = "pyobjc-framework-corespotlight", marker = "platform_release >= '17'" }, { name = "pyobjc-framework-coretext" }, - { name = "pyobjc-framework-corewlan", marker = "platform_release >= '10.0'" }, - { name = "pyobjc-framework-cryptotokenkit", marker = "platform_release >= '14.0'" }, - { name = "pyobjc-framework-datadetection", marker = "platform_release >= '21.0'" }, - { name = "pyobjc-framework-devicecheck", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-devicediscoveryextension", marker = "platform_release >= '24.0'" }, - { name = "pyobjc-framework-dictionaryservices", marker = "platform_release >= '9.0'" }, + { name = "pyobjc-framework-corewlan", marker = "platform_release >= '10'" }, + { name = "pyobjc-framework-cryptotokenkit", marker = "platform_release >= '14'" }, + { name = "pyobjc-framework-datadetection", marker = "platform_release >= '21'" }, + { name = "pyobjc-framework-devicecheck", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-devicediscoveryextension", marker = "platform_release >= '24'" }, + { name = "pyobjc-framework-dictionaryservices", marker = "platform_release >= '9'" }, { name = "pyobjc-framework-discrecording" }, { name = "pyobjc-framework-discrecordingui" }, { name = "pyobjc-framework-diskarbitration" }, { name = "pyobjc-framework-dvdplayback" }, - { name = "pyobjc-framework-eventkit", marker = "platform_release >= '12.0'" }, + { name = "pyobjc-framework-eventkit", marker = "platform_release >= '12'" }, { name = "pyobjc-framework-exceptionhandling" }, - { name = "pyobjc-framework-executionpolicy", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-extensionkit", marker = "platform_release >= '22.0'" }, - { name = "pyobjc-framework-externalaccessory", marker = "platform_release >= '17.0'" }, - { name = "pyobjc-framework-fileprovider", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-fileproviderui", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-findersync", marker = "platform_release >= '14.0'" }, - { name = "pyobjc-framework-fsevents", marker = "platform_release >= '9.0'" }, + { name = "pyobjc-framework-executionpolicy", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-extensionkit", marker = "platform_release >= '22'" }, + { name = "pyobjc-framework-externalaccessory", marker = "platform_release >= '17'" }, + { name = "pyobjc-framework-fileprovider", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-fileproviderui", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-findersync", marker = "platform_release >= '14'" }, + { name = "pyobjc-framework-fsevents", marker = "platform_release >= '9'" }, { name = "pyobjc-framework-fskit", marker = "platform_release >= '24.4'" }, - { name = "pyobjc-framework-gamecenter", marker = "platform_release >= '12.0'" }, - { name = "pyobjc-framework-gamecontroller", marker = "platform_release >= '13.0'" }, - { name = "pyobjc-framework-gamekit", marker = "platform_release >= '12.0'" }, - { name = "pyobjc-framework-gameplaykit", marker = "platform_release >= '15.0'" }, - { name = "pyobjc-framework-gamesave", marker = "platform_release >= '25.0'" }, - { name = "pyobjc-framework-healthkit", marker = "platform_release >= '22.0'" }, - { name = "pyobjc-framework-imagecapturecore", marker = "platform_release >= '10.0'" }, - { name = "pyobjc-framework-inputmethodkit", marker = "platform_release >= '9.0'" }, + { name = "pyobjc-framework-gamecenter", marker = "platform_release >= '12'" }, + { name = "pyobjc-framework-gamecontroller", marker = "platform_release >= '13'" }, + { name = "pyobjc-framework-gamekit", marker = "platform_release >= '12'" }, + { name = "pyobjc-framework-gameplaykit", marker = "platform_release >= '15'" }, + { name = "pyobjc-framework-gamesave", marker = "platform_release >= '25'" }, + { name = "pyobjc-framework-healthkit", marker = "platform_release >= '22'" }, + { name = "pyobjc-framework-imagecapturecore", marker = "platform_release >= '10'" }, + { name = "pyobjc-framework-inputmethodkit", marker = "platform_release >= '9'" }, { name = "pyobjc-framework-installerplugins" }, - { name = "pyobjc-framework-instantmessage", marker = "platform_release >= '9.0'" }, - { name = "pyobjc-framework-intents", marker = "platform_release >= '16.0'" }, - { name = "pyobjc-framework-intentsui", marker = "platform_release >= '21.0'" }, + { name = "pyobjc-framework-instantmessage", marker = "platform_release >= '9'" }, + { name = "pyobjc-framework-intents", marker = "platform_release >= '16'" }, + { name = "pyobjc-framework-intentsui", marker = "platform_release >= '21'" }, { name = "pyobjc-framework-iobluetooth" }, { name = "pyobjc-framework-iobluetoothui" }, - { name = "pyobjc-framework-iosurface", marker = "platform_release >= '10.0'" }, - { name = "pyobjc-framework-ituneslibrary", marker = "platform_release >= '10.0'" }, - { name = "pyobjc-framework-kernelmanagement", marker = "platform_release >= '20.0'" }, + { name = "pyobjc-framework-iosurface", marker = "platform_release >= '10'" }, + { name = "pyobjc-framework-ituneslibrary", marker = "platform_release >= '10'" }, + { name = "pyobjc-framework-kernelmanagement", marker = "platform_release >= '20'" }, { name = "pyobjc-framework-latentsemanticmapping" }, { name = "pyobjc-framework-launchservices" }, - { name = "pyobjc-framework-libdispatch", marker = "platform_release >= '12.0'" }, - { name = "pyobjc-framework-libxpc", marker = "platform_release >= '12.0'" }, - { name = "pyobjc-framework-linkpresentation", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-localauthentication", marker = "platform_release >= '14.0'" }, - { name = "pyobjc-framework-localauthenticationembeddedui", marker = "platform_release >= '21.0'" }, - { name = "pyobjc-framework-mailkit", marker = "platform_release >= '21.0'" }, - { name = "pyobjc-framework-mapkit", marker = "platform_release >= '13.0'" }, - { name = "pyobjc-framework-mediaaccessibility", marker = "platform_release >= '13.0'" }, - { name = "pyobjc-framework-mediaextension", marker = "platform_release >= '24.0'" }, - { name = "pyobjc-framework-medialibrary", marker = "platform_release >= '13.0'" }, - { name = "pyobjc-framework-mediaplayer", marker = "platform_release >= '16.0'" }, - { name = "pyobjc-framework-mediatoolbox", marker = "platform_release >= '13.0'" }, - { name = "pyobjc-framework-metal", marker = "platform_release >= '15.0'" }, - { name = "pyobjc-framework-metalfx", marker = "platform_release >= '22.0'" }, - { name = "pyobjc-framework-metalkit", marker = "platform_release >= '15.0'" }, - { name = "pyobjc-framework-metalperformanceshaders", marker = "platform_release >= '17.0'" }, - { name = "pyobjc-framework-metalperformanceshadersgraph", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-metrickit", marker = "platform_release >= '21.0'" }, - { name = "pyobjc-framework-mlcompute", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-modelio", marker = "platform_release >= '15.0'" }, - { name = "pyobjc-framework-multipeerconnectivity", marker = "platform_release >= '14.0'" }, - { name = "pyobjc-framework-naturallanguage", marker = "platform_release >= '18.0'" }, - { name = "pyobjc-framework-netfs", marker = "platform_release >= '10.0'" }, - { name = "pyobjc-framework-network", marker = "platform_release >= '18.0'" }, - { name = "pyobjc-framework-networkextension", marker = "platform_release >= '15.0'" }, - { name = "pyobjc-framework-notificationcenter", marker = "platform_release >= '14.0'" }, - { name = "pyobjc-framework-opendirectory", marker = "platform_release >= '10.0'" }, + { name = "pyobjc-framework-libdispatch", marker = "platform_release >= '12'" }, + { name = "pyobjc-framework-libxpc", marker = "platform_release >= '12'" }, + { name = "pyobjc-framework-linkpresentation", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-localauthentication", marker = "platform_release >= '14'" }, + { name = "pyobjc-framework-localauthenticationembeddedui", marker = "platform_release >= '21'" }, + { name = "pyobjc-framework-mailkit", marker = "platform_release >= '21'" }, + { name = "pyobjc-framework-mapkit", marker = "platform_release >= '13'" }, + { name = "pyobjc-framework-mediaaccessibility", marker = "platform_release >= '13'" }, + { name = "pyobjc-framework-mediaextension", marker = "platform_release >= '24'" }, + { name = "pyobjc-framework-medialibrary", marker = "platform_release >= '13'" }, + { name = "pyobjc-framework-mediaplayer", marker = "platform_release >= '16'" }, + { name = "pyobjc-framework-mediatoolbox", marker = "platform_release >= '13'" }, + { name = "pyobjc-framework-metal", marker = "platform_release >= '15'" }, + { name = "pyobjc-framework-metalfx", marker = "platform_release >= '22'" }, + { name = "pyobjc-framework-metalkit", marker = "platform_release >= '15'" }, + { name = "pyobjc-framework-metalperformanceshaders", marker = "platform_release >= '17'" }, + { name = "pyobjc-framework-metalperformanceshadersgraph", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-metrickit", marker = "platform_release >= '21'" }, + { name = "pyobjc-framework-mlcompute", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-modelio", marker = "platform_release >= '15'" }, + { name = "pyobjc-framework-multipeerconnectivity", marker = "platform_release >= '14'" }, + { name = "pyobjc-framework-naturallanguage", marker = "platform_release >= '18'" }, + { name = "pyobjc-framework-netfs", marker = "platform_release >= '10'" }, + { name = "pyobjc-framework-network", marker = "platform_release >= '18'" }, + { name = "pyobjc-framework-networkextension", marker = "platform_release >= '15'" }, + { name = "pyobjc-framework-notificationcenter", marker = "platform_release >= '14'" }, + { name = "pyobjc-framework-opendirectory", marker = "platform_release >= '10'" }, { name = "pyobjc-framework-osakit" }, - { name = "pyobjc-framework-oslog", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-passkit", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-pencilkit", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-phase", marker = "platform_release >= '21.0'" }, - { name = "pyobjc-framework-photos", marker = "platform_release >= '15.0'" }, - { name = "pyobjc-framework-photosui", marker = "platform_release >= '15.0'" }, + { name = "pyobjc-framework-oslog", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-passkit", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-pencilkit", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-phase", marker = "platform_release >= '21'" }, + { name = "pyobjc-framework-photos", marker = "platform_release >= '15'" }, + { name = "pyobjc-framework-photosui", marker = "platform_release >= '15'" }, { name = "pyobjc-framework-preferencepanes" }, - { name = "pyobjc-framework-pushkit", marker = "platform_release >= '19.0'" }, + { name = "pyobjc-framework-pubsub", marker = "platform_release >= '9' and platform_release < '18'" }, + { name = "pyobjc-framework-pushkit", marker = "platform_release >= '19'" }, { name = "pyobjc-framework-quartz" }, - { name = "pyobjc-framework-quicklookthumbnailing", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-replaykit", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-safariservices", marker = "platform_release >= '16.0'" }, - { name = "pyobjc-framework-safetykit", marker = "platform_release >= '22.0'" }, - { name = "pyobjc-framework-scenekit", marker = "platform_release >= '11.0'" }, + { name = "pyobjc-framework-quicklookthumbnailing", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-replaykit", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-safariservices", marker = "platform_release >= '16'" }, + { name = "pyobjc-framework-safetykit", marker = "platform_release >= '22'" }, + { name = "pyobjc-framework-scenekit", marker = "platform_release >= '11'" }, { name = "pyobjc-framework-screencapturekit", marker = "platform_release >= '21.4'" }, { name = "pyobjc-framework-screensaver" }, - { name = "pyobjc-framework-screentime", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-scriptingbridge", marker = "platform_release >= '9.0'" }, + { name = "pyobjc-framework-screentime", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-scriptingbridge", marker = "platform_release >= '9'" }, { name = "pyobjc-framework-searchkit" }, { name = "pyobjc-framework-security" }, { name = "pyobjc-framework-securityfoundation" }, { name = "pyobjc-framework-securityinterface" }, { name = "pyobjc-framework-securityui", marker = "platform_release >= '24.4'" }, - { name = "pyobjc-framework-sensitivecontentanalysis", marker = "platform_release >= '23.0'" }, - { name = "pyobjc-framework-servicemanagement", marker = "platform_release >= '10.0'" }, - { name = "pyobjc-framework-sharedwithyou", marker = "platform_release >= '22.0'" }, - { name = "pyobjc-framework-sharedwithyoucore", marker = "platform_release >= '22.0'" }, - { name = "pyobjc-framework-shazamkit", marker = "platform_release >= '21.0'" }, - { name = "pyobjc-framework-social", marker = "platform_release >= '12.0'" }, - { name = "pyobjc-framework-soundanalysis", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-speech", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-spritekit", marker = "platform_release >= '13.0'" }, - { name = "pyobjc-framework-storekit", marker = "platform_release >= '11.0'" }, - { name = "pyobjc-framework-symbols", marker = "platform_release >= '23.0'" }, + { name = "pyobjc-framework-sensitivecontentanalysis", marker = "platform_release >= '23'" }, + { name = "pyobjc-framework-servicemanagement", marker = "platform_release >= '10'" }, + { name = "pyobjc-framework-sharedwithyou", marker = "platform_release >= '22'" }, + { name = "pyobjc-framework-sharedwithyoucore", marker = "platform_release >= '22'" }, + { name = "pyobjc-framework-shazamkit", marker = "platform_release >= '21'" }, + { name = "pyobjc-framework-social", marker = "platform_release >= '12'" }, + { name = "pyobjc-framework-soundanalysis", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-speech", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-spritekit", marker = "platform_release >= '13'" }, + { name = "pyobjc-framework-storekit", marker = "platform_release >= '11'" }, + { name = "pyobjc-framework-symbols", marker = "platform_release >= '23'" }, { name = "pyobjc-framework-syncservices" }, { name = "pyobjc-framework-systemconfiguration" }, - { name = "pyobjc-framework-systemextensions", marker = "platform_release >= '19.0'" }, - { name = "pyobjc-framework-threadnetwork", marker = "platform_release >= '22.0'" }, - { name = "pyobjc-framework-uniformtypeidentifiers", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-usernotifications", marker = "platform_release >= '18.0'" }, - { name = "pyobjc-framework-usernotificationsui", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-videosubscriberaccount", marker = "platform_release >= '18.0'" }, - { name = "pyobjc-framework-videotoolbox", marker = "platform_release >= '12.0'" }, - { name = "pyobjc-framework-virtualization", marker = "platform_release >= '20.0'" }, - { name = "pyobjc-framework-vision", marker = "platform_release >= '17.0'" }, + { name = "pyobjc-framework-systemextensions", marker = "platform_release >= '19'" }, + { name = "pyobjc-framework-threadnetwork", marker = "platform_release >= '22'" }, + { name = "pyobjc-framework-uniformtypeidentifiers", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-usernotifications", marker = "platform_release >= '18'" }, + { name = "pyobjc-framework-usernotificationsui", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-videosubscriberaccount", marker = "platform_release >= '18'" }, + { name = "pyobjc-framework-videotoolbox", marker = "platform_release >= '12'" }, + { name = "pyobjc-framework-virtualization", marker = "platform_release >= '20'" }, + { name = "pyobjc-framework-vision", marker = "platform_release >= '17'" }, { name = "pyobjc-framework-webkit" }, ] sdist = { url = "https://files.pythonhosted.org/packages/73/ef/b4e64fe87051e72608ed4134072e832e9eae28d97e9c9bb0870f01a18ac5/pyobjc-12.2.1.tar.gz", hash = "sha256:0b2cf49d24213e7604620c31863e7b4e42770c4442c10e59b18ad951cd200cd3", size = 12148, upload-time = "2026-06-19T16:19:38.283Z" } @@ -3617,6 +3618,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/b1/bc42da075faf6e4201bfe2ec4bfda95b71877a15df4a501977ef739b7128/pyobjc_framework_preferencepanes-12.2.1-py2.py3-none-any.whl", hash = "sha256:026ff87afba2d381e3d9dadc58957580558e1a705291da6794107220a3cd8102", size = 4829, upload-time = "2026-06-19T16:15:51.915Z" }, ] +[[package]] +name = "pyobjc-framework-pubsub" +version = "12.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/d1/944c37789f5f24638dd087d30a11b7806333a67114c8ebefaccc50a4a75a/pyobjc_framework_pubsub-12.2.1.tar.gz", hash = "sha256:44a9507ff51f678fc9460ebb8620df592aeed85fc48130b6c6595cd457f6f429", size = 14857, upload-time = "2026-06-19T16:21:27.292Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/6d/4516eb9e55ccf610d9892caeeaee493bf4c8415465ffe0718453af414457/pyobjc_framework_pubsub-12.2.1-py2.py3-none-any.whl", hash = "sha256:5fd1d1a2f66551a75527284f133e1085fb11773003fd2c60ba26236caa9542a9", size = 4391, upload-time = "2026-06-19T16:15:52.834Z" }, +] + [[package]] name = "pyobjc-framework-pushkit" version = "12.2.1" @@ -4690,11 +4704,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]]