From 2ef566ed271ed32a305ea481de275f82233bd2c2 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Sun, 1 Nov 2026 12:03:36 -0800 Subject: [PATCH 1/2] devops(bots): publish the agent's GitHub writes as typed calls The triage and revision workflows switch to the publish-calls action of microsoft/playwright. The agent leaves what must happen on GitHub as typed calls in calls.json, and one job checks and makes them with the app token. It replaces publish-patch and triage-comment. Depends on: https://github.com/microsoft/playwright/pull/43838 --- .claude/skills/playwright-triage/SKILL.md | 38 ++++--- .github/workflows/revision.yml | 26 ++--- .github/workflows/triage.yml | 126 +++++++++------------- 3 files changed, 92 insertions(+), 98 deletions(-) diff --git a/.claude/skills/playwright-triage/SKILL.md b/.claude/skills/playwright-triage/SKILL.md index a64412382..08f4834f0 100644 --- a/.claude/skills/playwright-triage/SKILL.md +++ b/.claude/skills/playwright-triage/SKILL.md @@ -59,18 +59,32 @@ too, the root cause is in the driver or in a browser, and the fix lands in `micr which the family treats as one team. When Node.js passes, the bug is in the Java client: compare the `*Impl.java` with its JavaScript counterpart, and fix it here. -For an upstream bug, create `$OUTPUT_DIR/upstream/` with: - -- `report.md`: the report for a maintainer to file in `microsoft/playwright`, for an engineer who - did not read the Java issue: what fails, expected versus actual, the driver version, the browsers - and the platform, the root cause when you found it, and the condensed Node.js test as the repro. -- `report.json`: `{"title": "..."}`, at most 256 characters, for example - `route.fulfill() ignores the charset in Content-Type`. - -The comment job appends the title and the body to your comment, for a maintainer to file. The bot -never files it. Still write the comment as the note: the verdict, the evidence, and that the bug is -upstream. Do not paste the report into it. Write `report.json` last, when `report.md` and the -comment are done. Like `proposal.json`, it marks the report as ready. +For an upstream bug, write a report for a maintainer to file in `microsoft/playwright`, for an +engineer who did not read the Java issue: what fails, expected versus actual, the driver version, +the browsers and the platform, the root cause when you found it, and the condensed Node.js test as +the repro. The title has at most 256 characters, for example +`route.fulfill() ignores the charset in Content-Type`. + +The bot never files it: this run allows no `issue` call (the calls are in +`$AGENT_GUIDES_DIR/calls.d.ts`). Put the report at the end of your comment, for a maintainer to +file, in this form: + +`````markdown +Upstream bug report, for a maintainer to file in microsoft/playwright: + +**Title:** + +<details> +<summary>Body</summary> + +````markdown +<the report> +```` + +</details> +````` + +Write the rest of the comment as the note: the verdict, the evidence, and that the bug is upstream. The public API is generated from the upstream docs: `scripts/generate_api.sh` runs upstream's `utils/doclint/generateApiJson.js` at the commit of the driver, and `tools/api-generator` writes diff --git a/.github/workflows/revision.yml b/.github/workflows/revision.yml index 89010a3ae..d8aa55ca0 100644 --- a/.github/workflows/revision.yml +++ b/.github/workflows/revision.yml @@ -1,8 +1,9 @@ # Copilot addresses the maintainers' review comments on a draft PR that the triage workflow opened, -# without write credentials. A fresh job pushes the new commit to the PR's branch using the app, so -# CI runs on it. The steps that are the same for every Playwright repo are actions in -# microsoft/playwright, and the shared guide they carry says how to revise; the prompt below has -# what is specific to this repo. +# without write credentials. The agent leaves the calls to make on GitHub in calls.json. A fresh job +# checks them and makes them using the app: it pushes the new commit to the PR's branch, so CI runs +# on it, and replies in the review threads. The steps that are the same for every Playwright repo +# are actions in microsoft/playwright, and the shared guide they carry says how to revise; the +# prompt below has what is specific to this repo. name: PR revision on: @@ -97,14 +98,14 @@ jobs: Pull request: https://github.com/microsoft/playwright-java/pull/${{ github.event.pull_request.number }} output-dir: ${{ env.OUTPUT_DIR }} - result-file: commit/commit.json + result-file: calls.json artifact-name: revision exec: ${{ env.CI_ENV_EXEC }} push: name: Push the revision needs: [prepare, revise] - # commit.json is written last, so a finished revision is pushed even when the job failed later. + # calls.json is written when the revision is done, so it is pushed even if the job failed later. if: ${{ !cancelled() && needs.revise.outputs.has_revision == 'true' }} runs-on: ubuntu-latest steps: @@ -113,6 +114,8 @@ jobs: with: app-id: ${{ vars.PLAYWRIGHT_APP_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.prepare.outputs.head }} @@ -121,12 +124,9 @@ jobs: with: name: revision path: ${{ runner.temp }}/revision - # Never execute code or scripts from the proposed changes in the app job. - - name: Validate and push - uses: microsoft/playwright/.github/actions/publish-patch@main + # Never execute code or scripts from the artifact in the app job. The calls are data. + - name: Check and make the calls + uses: microsoft/playwright/.github/actions/publish-calls@main with: token: ${{ steps.app-token.outputs.token }} - dir: ${{ runner.temp }}/revision/commit - patch: revision.patch - metadata: commit.json - branch: ${{ needs.prepare.outputs.branch }} + dir: ${{ runner.temp }}/revision diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml index b0b7594f1..16ebb7f4d 100644 --- a/.github/workflows/triage.yml +++ b/.github/workflows/triage.yml @@ -1,8 +1,9 @@ # Copilot triages and fixes an issue, or works on a task given by hand, without write credentials. -# Fresh jobs post the run link on the issue, open the completed fix as a draft PR using the app, so -# CI runs on it, and comment on the issue. The steps that are the same for every Playwright repo are -# actions in microsoft/playwright. The skill in .claude/skills/playwright-triage/ has what is -# specific to this repo. +# The agent leaves the calls to make on GitHub in calls.json. A fresh job posts the run link on the +# issue first. After the agent, a fresh job checks the calls and makes them using the app: it opens +# the fix as a draft PR, so CI runs on it, and comments on the issue. The steps that are the same +# for every Playwright repo are actions in microsoft/playwright. The skill in +# .claude/skills/playwright-triage/ has what is specific to this repo. name: Issue triage on: @@ -86,7 +87,6 @@ jobs: base: ${{ steps.base.outputs.sha }} setup: ${{ steps.setup.outcome }} agent: ${{ steps.agent.outputs.outcome }} - has_fix: ${{ steps.agent.outputs.has-result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -130,98 +130,78 @@ jobs: with: prompt: ${{ steps.prompt.outputs.prompt }} output-dir: ${{ env.OUTPUT_DIR }} - result-file: fix/proposal.json + result-file: calls.json artifact-name: triage github-tools: true exec: ${{ env.CI_ENV_EXEC }} - - name: Check the upstream bug report - run: | - if [[ -f "$OUTPUT_DIR/upstream/report.json" ]]; then - title=$(node -p 'String(JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).title ?? "")' "$OUTPUT_DIR/upstream/report.json" || true) - if [[ -z "$title" ]]; then - echo "::warning::upstream/report.json has no title, so the report is not appended to the comment" - else - echo "Upstream bug report: $title" >> "$GITHUB_STEP_SUMMARY" - fi - fi publish: - name: Open draft pull request - needs: triage - # proposal.json is written last, so a finished proposal is published even when the job failed later. - if: ${{ !cancelled() && needs.triage.outputs.has_fix == 'true' }} + name: Publish the results + needs: [start, triage] + # Runs after every triage run, also a failed one: without a comment from the agent, the comment + # says why the bot could not finish. + if: ${{ !cancelled() && needs.triage.result != 'skipped' }} runs-on: ubuntu-latest - outputs: - pr: ${{ steps.publish.outputs.pull-request }} steps: - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 id: app-token with: app-id: ${{ vars.PLAYWRIGHT_APP_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} + permission-contents: write + permission-issues: write + permission-pull-requests: write - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.triage.outputs.base }} token: ${{ steps.app-token.outputs.token }} - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: triage - path: ${{ runner.temp }}/triage - # Never execute code or scripts from the proposed changes in the app job. - - name: Validate, push and open draft - id: publish - uses: microsoft/playwright/.github/actions/publish-patch@main - with: - token: ${{ steps.app-token.outputs.token }} - dir: ${{ runner.temp }}/triage/fix - patch: fix.patch - metadata: proposal.json - body-file: pr-body.md - branch: triage/${{ github.event.issue.number || inputs.issue || 'task' }}/${{ github.run_id }}-${{ github.run_attempt }} - commit-body: "${{ (github.event.issue.number || inputs.issue) && format('Fixes: https://github.com/{0}/issues/{1}', github.repository, github.event.issue.number || inputs.issue) || '' }}" - pull-request: true - - comment: - name: Comment on the issue - needs: [start, triage, publish] - # Runs after every triage run that has an issue, including failed ones, and whatever publish did. - if: ${{ !cancelled() && needs.triage.result != 'skipped' && (github.event_name == 'issues' || inputs.issue != '') }} - runs-on: ubuntu-latest - permissions: - issues: write - steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 # A failed triage run may not have uploaded anything. continue-on-error: true with: name: triage path: ${{ runner.temp }}/triage - # The bot never files upstream; the comment hands the report to a maintainer. - - name: Describe the upstream bug report - id: upstream + # Without a comment from the agent on the issue, for example after a failed run, the issue gets + # one that says that the bot could not finish. + - name: Add a comment when the agent left none + if: github.event.issue.number || inputs.issue env: - OUTPUT_DIR: ${{ runner.temp }}/triage + CALLS: ${{ runner.temp }}/triage/calls.json + ISSUE: ${{ github.event.issue.number || inputs.issue }} + SETUP: ${{ needs.triage.outputs.setup }} + AGENT: ${{ needs.triage.outputs.agent }} + FOOTER: Triaged on ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }} by the Playwright bot + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | - extra="" - if [[ -s "$OUTPUT_DIR/upstream/report.md" && -f "$OUTPUT_DIR/upstream/report.json" ]]; then - title=$(jq -r '.title // empty' "$OUTPUT_DIR/upstream/report.json") - if [[ -n "$title" ]]; then - extra=$(printf 'Upstream bug report, for a maintainer to file in microsoft/playwright:\n\n**Title:** %s\n\n<details>\n<summary>Body</summary>\n\n````markdown\n%s\n````\n\n</details>' "$title" "$(cat "$OUTPUT_DIR/upstream/report.md")") - fi + mkdir -p "$(dirname "$CALLS")" + if jq -e --argjson issue "$ISSUE" 'any(.[]; .call == "comment" and .issue == $issue)' "$CALLS" > /dev/null 2>&1; then + exit 0 fi - delimiter="EXTRA_$RANDOM$RANDOM" - { echo "extra<<$delimiter"; echo "$extra"; echo "$delimiter"; } >> "$GITHUB_OUTPUT" - - name: Post the triage comment - uses: microsoft/playwright/.github/actions/triage-comment@main + if [[ "$SETUP" == "failure" ]]; then + reason="setting up the runner failed" + elif [[ "$AGENT" == "failure" ]]; then + reason="the Copilot session failed or ran out of budget" + else + reason="the session ended without a comment" + fi + body=$(printf "Hi, I'm the Playwright bot. I couldn't finish triaging this issue: %s. The agent run has the details.\n\n<sub>%s - [agent run](%s)</sub>\n<!-- playwright-ai-triage -->" "$reason" "$FOOTER" "$RUN_URL") + comment=$(jq -n --arg repo "$GITHUB_REPOSITORY" --argjson issue "$ISSUE" --arg body "$body" '{call: "comment", repo: $repo, issue: $issue, body: $body}') + # A calls.json that is not an array is replaced: its calls would be rejected anyway. + jq --argjson comment "$comment" '. + [$comment]' "$CALLS" > "$CALLS.tmp" 2> /dev/null \ + || jq -n --argjson comment "$comment" '[$comment]' > "$CALLS.tmp" + mv "$CALLS.tmp" "$CALLS" + # Never execute code or scripts from the artifact in the app job. The calls are data. + - name: Check and make the calls + id: publish + uses: microsoft/playwright/.github/actions/publish-calls@main with: - issue: ${{ github.event.issue.number || inputs.issue }} + token: ${{ steps.app-token.outputs.token }} dir: ${{ runner.temp }}/triage - footer: Triaged on ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }} by the Playwright bot - job-result: ${{ needs.triage.result }} - setup-outcome: ${{ needs.triage.outputs.setup }} - agent-outcome: ${{ needs.triage.outputs.agent }} - setup-failure-reason: setting up the runner, building the project or downloading the browsers failed - pull-request: ${{ needs.publish.outputs.pr }} - publish-result: ${{ needs.publish.result }} - extra: ${{ steps.upstream.outputs.extra }} - start-comment: ${{ needs.start.outputs.comment }} + - name: Hide the comment with the run link + if: ${{ !cancelled() && steps.publish.outputs.comment == 'true' && needs.start.outputs.comment != '' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ID: ${{ needs.start.outputs.comment }} + run: | + gh api graphql -f query='mutation($id: ID!) { minimizeComment(input: { subjectId: $id, classifier: OUTDATED }) { clientMutationId } }' -f id="$ID" \ + || echo "::warning::Could not hide the comment with the run link" From 18d11303b490baad1d74b2d59b045f2ca3bd3b37 Mon Sep 17 00:00:00 2001 From: Pavel Feldman <pavel.feldman@gmail.com> Date: Sun, 1 Nov 2026 14:08:01 -0800 Subject: [PATCH 2/2] devops(bots): let collect-review read the issue of the PR --- .github/workflows/revision.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/revision.yml b/.github/workflows/revision.yml index d8aa55ca0..63f3bd374 100644 --- a/.github/workflows/revision.yml +++ b/.github/workflows/revision.yml @@ -25,6 +25,7 @@ jobs: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest permissions: + issues: read pull-requests: write env: PLATFORM: ${{ fromJSON('{"needs-revision":"linux","needs-revision-mac":"mac","needs-revision-win":"windows"}')[github.event.label.name] }}