From cda6efa1567e27390ab579e23b57bb6d0da8fdab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Sun, 18 Oct 2026 10:26:43 +0200 Subject: [PATCH] ci: harden the pull request analysis CI-Analysis runs with repository secrets on artifacts that a pull request controls, and it trusted the pull request number, branch name and comment body in them. A pull request could therefore comment on any pull request, overwrite the mutation score of `main` or mention users. The pull request and the dashboard version now come from the `workflow_run` event alone, artifacts are checked for symbolic links, the report is validated before the upload, comment content is defused, and only comments of the pipeline are updated. The mutation job of CI-Analysis now also publishes pull request reports to the dashboard, which the comment's badge links to. A release tag that is no valid version now fails the build instead of producing an odd package version. Stryker is pinned to 5.0.0 through the build project so that Dependabot updates it, GitVersion and ReportGenerator run on .NET 10 like in the other repositories, Sonar reads the generic coverage format, which keeps the lines of lambdas in generic classes, the mutation targets follow the names of aweXpect, and all actions are pinned to a commit. Dependabot no longer bumps Roslyn, because the analyzers and the source generator only load in compilers at least as new as the Roslyn they reference. --- .fallout/build.schema.json | 5 +- .github/dependabot.yml | 3 + .github/workflows/build.yml | 2 +- .github/workflows/ci-analysis.yml | 32 +++- .github/workflows/ci.yml | 32 ++-- .github/workflows/notify-docs-site.yml | 2 +- Pipeline/Build.Benchmarks.cs | 69 ++++--- Pipeline/Build.CodeAnalysis.cs | 2 +- Pipeline/Build.CodeCoverage.cs | 6 +- Pipeline/Build.Compile.cs | 8 + Pipeline/Build.MutationTests.cs | 244 ++++++++++++++++--------- Pipeline/Build.cs | 2 +- Pipeline/Build.csproj | 2 +- Pipeline/BuildExtensions.cs | 101 ++++++++++ 14 files changed, 355 insertions(+), 155 deletions(-) diff --git a/.fallout/build.schema.json b/.fallout/build.schema.json index 8d700115..5ef3e111 100644 --- a/.fallout/build.schema.json +++ b/.fallout/build.schema.json @@ -37,10 +37,9 @@ "CodeCoverage", "Compile", "DotNetUnitTests", - "MutationTestComment", - "MutationTestDashboard", - "MutationTestExecution", "MutationTests", + "MutationTestsComment", + "MutationTestsDashboard", "Pack", "PublishBenchmarkReport", "Restore", diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 39b08e4b..29277b1a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,3 +29,6 @@ updates: aweXpect: patterns: - "aweXpect*" + # The analyzers and generators only load in compilers at least as new as the Roslyn they reference. + ignore: + - dependency-name: "Microsoft.CodeAnalysis*" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df74b88f..358cedcb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,7 +149,7 @@ jobs: 8.0.x 10.0.x - name: Run mutation tests - run: ./build.sh MutationTests MutationTestDashboard + run: ./build.sh MutationTests MutationTestsDashboard static-code-analysis: name: "Static code analysis" diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index d1483f63..f8da99c7 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -12,39 +12,55 @@ jobs: name: "Mutation tests" runs-on: ubuntu-latest permissions: + actions: read + contents: read pull-requests: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 10.0.x - - name: Create mutation test comment - run: ./build.sh MutationTestComment + - name: Upload mutation dashboard and create comment + run: ./build.sh MutationTestsComment + # The artifacts of the workflow run are untrusted, because a pull request controls the code that + # produces them, so the build identifies the pull request by these values of the event alone. env: GithubToken: ${{ secrets.GITHUB_TOKEN }} + STRYKER_DASHBOARD_API_KEY: ${{ secrets.STRYKER_DASHBOARD_API_KEY }} WorkflowRunId: ${{ github.event.workflow_run.id }} - + WorkflowRunEvent: ${{ github.event.workflow_run.event }} + WorkflowRunHeadOwner: ${{ github.event.workflow_run.head_repository.owner.login }} + WorkflowRunHeadBranch: ${{ github.event.workflow_run.head_branch }} + WorkflowRunHeadSha: ${{ github.event.workflow_run.head_sha }} + benchmarks: name: "Benchmarks" runs-on: ubuntu-latest permissions: - contents: write + actions: read + contents: read pull-requests: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 10.0.x - name: Create benchmark comment run: ./build.sh BenchmarkComment + # The artifacts of the workflow run are untrusted, because a pull request controls the code that + # produces them, so the build identifies the pull request by these values of the event alone. env: GithubToken: ${{ secrets.GITHUB_TOKEN }} WorkflowRunId: ${{ github.event.workflow_run.id }} + WorkflowRunEvent: ${{ github.event.workflow_run.event }} + WorkflowRunHeadOwner: ${{ github.event.workflow_run.head_repository.owner.login }} + WorkflowRunHeadBranch: ${{ github.event.workflow_run.head_branch }} + WorkflowRunHeadSha: ${{ github.event.workflow_run.head_sha }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3611296c..5d610b6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,11 +14,11 @@ jobs: os: [ ubuntu-latest, windows-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -31,7 +31,7 @@ jobs: run: ./build.sh CodeCoverage - name: Upload artifacts if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ runner.os }}-artifacts path: | @@ -44,11 +44,11 @@ jobs: env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -57,7 +57,7 @@ jobs: run: ./build.sh ApiChecks - name: Upload artifacts if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: API-tests path: | @@ -70,11 +70,11 @@ jobs: env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -83,7 +83,7 @@ jobs: run: ./build.sh MutationTests - name: Upload artifacts if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: MutationTests path: | @@ -105,11 +105,11 @@ jobs: env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -118,7 +118,7 @@ jobs: run: ./build.sh Benchmarks --benchmark-filter "*${{ matrix.benchmark }}*" - name: Upload artifacts if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: Benchmarks-${{ matrix.benchmark }} path: | @@ -134,11 +134,11 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -156,11 +156,11 @@ jobs: if: always() steps: - name: Download Artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: path: artifacts - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 + uses: EnricoMi/publish-unit-test-result-action@7622c5bf3d112a72ab4fa3951c2aa5723d8ab92d # v2.25.0 with: comment_mode: always files: "artifacts/**/**/*.trx" diff --git a/.github/workflows/notify-docs-site.yml b/.github/workflows/notify-docs-site.yml index d9b37588..2b7a10ec 100644 --- a/.github/workflows/notify-docs-site.yml +++ b/.github/workflows/notify-docs-site.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Dispatch extension-documentation-updated-event to Testably/Testably.Site - uses: peter-evans/repository-dispatch@v4 + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: token: ${{ secrets.SITE_DISPATCH_TOKEN }} repository: Testably/Testably.Site diff --git a/Pipeline/Build.Benchmarks.cs b/Pipeline/Build.Benchmarks.cs index 641852ec..22d4597c 100644 --- a/Pipeline/Build.Benchmarks.cs +++ b/Pipeline/Build.Benchmarks.cs @@ -63,16 +63,20 @@ partial class Build string fileContent = await File.ReadAllTextAsync(file); Log.Information("Report ({FileName}):\n {FileContent}", Path.GetFileName(file), fileContent); } - - if (GitHubActions?.IsPullRequest == true) - { - File.WriteAllText(ArtifactsDirectory / "PR.txt", GitHubActions.PullRequestNumber.ToString()); - } }); Target BenchmarkComment => _ => _ .Executes(async () => { + // The artifacts are untrusted, because the pull request controls the code that produces them, so the pull + // request to comment on comes from the event of the workflow run. + int? prId = await BuildExtensions.ResolvePullRequestOfWorkflowRun(GithubToken); + if (prId == null) + { + Log.Information("Skip writing a comment, as the workflow run belongs to no pull request."); + return; + } + await "Benchmarks-".DownloadArtifactsStartingWith(ArtifactsDirectory, GithubToken); if (!Directory.Exists(ArtifactsDirectory / "Benchmarks" / "results")) { @@ -87,46 +91,35 @@ partial class Build return; } - if (!File.Exists(ArtifactsDirectory / "PR.txt")) - { - Log.Information("Skip writing a comment, as no PR number was specified."); - return; - } - AbsolutePath baselineDirectory = ArtifactsDirectory / "Baseline"; string baselineResultsDirectory = await DownloadBaselineBenchmarks(baselineDirectory); - string prNumber = File.ReadAllText(ArtifactsDirectory / "PR.txt"); - string body = CreateBenchmarkCommentBody(files, baselineResultsDirectory); - Log.Debug("Pull request number: {PullRequestId}", prNumber); - if (int.TryParse(prNumber, out int prId)) + string body = CreateBenchmarkCommentBody(files, baselineResultsDirectory).AsUntrustedCommentContent(); + GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")); + Credentials tokenAuth = new(GithubToken); + gitHubClient.Credentials = tokenAuth; + IReadOnlyList comments = + await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value); + long? commentId = null; + Log.Information($"Found {comments.Count} comments"); + foreach (IssueComment comment in comments) { - GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")); - Credentials tokenAuth = new(GithubToken); - gitHubClient.Credentials = tokenAuth; - IReadOnlyList comments = - await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId); - long? commentId = null; - Log.Information($"Found {comments.Count} comments"); - foreach (IssueComment comment in comments) + if (comment.IsPipelineComment("## :rocket: Benchmark Results")) { - if (comment.Body.Contains("## :rocket: Benchmark Results")) - { - Log.Information($"Found comment: {comment.Body}"); - commentId = comment.Id; - } + Log.Information($"Found comment: {comment.Body}"); + commentId = comment.Id; } + } - if (commentId == null) - { - Log.Information($"Create comment:\n{body}"); - await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId, body); - } - else - { - Log.Information($"Update comment:\n{body}"); - await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body); - } + if (commentId == null) + { + Log.Information($"Create comment:\n{body}"); + await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value, body); + } + else + { + Log.Information($"Update comment:\n{body}"); + await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body); } }); diff --git a/Pipeline/Build.CodeAnalysis.cs b/Pipeline/Build.CodeAnalysis.cs index ec5c7019..a947ed23 100644 --- a/Pipeline/Build.CodeAnalysis.cs +++ b/Pipeline/Build.CodeAnalysis.cs @@ -19,7 +19,7 @@ partial class Build .SetOrganization("testably") .SetProjectKey("Testably_Mockolate") .AddVSTestReports(TestResultsDirectory / "*.trx") - .AddOpenCoverPaths(TestResultsDirectory / "reports" / "OpenCover.xml") + .AddGenericCoveragePaths(TestResultsDirectory / "reports" / "SonarQube.xml") .SetPullRequestOrBranchName(GitHubActions, GitVersion) .SetVersion(GitVersion.SemVer) .SetToken(SonarToken)); diff --git a/Pipeline/Build.CodeCoverage.cs b/Pipeline/Build.CodeCoverage.cs index 7631d451..0efea962 100644 --- a/Pipeline/Build.CodeCoverage.cs +++ b/Pipeline/Build.CodeCoverage.cs @@ -15,10 +15,12 @@ partial class Build { ReportGenerator(s => s .SetProcessToolPath(NuGetToolPathResolver.GetPackageExecutable("ReportGenerator", "ReportGenerator.dll", - framework: "net8.0")) + framework: "net10.0")) .SetTargetDirectory(TestResultsDirectory / "reports") .AddReports(TestResultsDirectory / "**/coverage.cobertura.xml") - .AddReportTypes(ReportTypes.OpenCover) + // The generic format of Sonar lists the lines per file. A format that lists them per method leaves out the + // lines of a lambda in a generic class, because ReportGenerator keeps them in a class without methods. + .AddReportTypes(ReportTypes.SonarQube) .AddFileFilters("-*.g.cs") .SetAssemblyFilters("+Mockolate*")); }); diff --git a/Pipeline/Build.Compile.cs b/Pipeline/Build.Compile.cs index 1cd95957..973c502b 100644 --- a/Pipeline/Build.Compile.cs +++ b/Pipeline/Build.Compile.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Text.RegularExpressions; using Fallout.Common; using Fallout.Common.IO; using Fallout.Common.Tools.DotNet; @@ -30,6 +31,13 @@ partial class Build } else if (GitHubActions.Ref.StartsWith("refs/tags/", StringComparison.OrdinalIgnoreCase)) { + if (!Regex.IsMatch(GitHubActions.Ref, + @"\Arefs/tags/v\d+\.\d+\.\d+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?\z")) + { + Assert.Fail( + $"The tag '{GitHubActions.Ref["refs/tags/".Length..]}' is no valid release tag (expected 'v1.2.3', optionally with a pre-release suffix like '-pre.1')"); + } + int preReleaseIndex = GitHubActions.Ref.IndexOf('-'); preRelease = preReleaseIndex > 0 ? GitHubActions.Ref[preReleaseIndex..] : ""; } diff --git a/Pipeline/Build.MutationTests.cs b/Pipeline/Build.MutationTests.cs index dedfd0bb..f25b7613 100644 --- a/Pipeline/Build.MutationTests.cs +++ b/Pipeline/Build.MutationTests.cs @@ -5,6 +5,10 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Xml.Linq; using Fallout.Common; using Fallout.Common.IO; using Fallout.Common.Tooling; @@ -23,13 +27,19 @@ namespace Build; partial class Build { private static readonly bool DisableMutationTests = false; - private static string MutationCommentBody = ""; + private const long MaxMutationReportSize = 100 * 1024 * 1024; - Target MutationTests => _ => _ - .DependsOn(MutationTestExecution) - .DependsOn(MutationTestComment); + Dictionary MutationTestProjects => new() + { + { + Solution.Mockolate, [ + Solution.Tests.Mockolate_Tests, + Solution.Tests.Mockolate_Internal_Tests, + ] + }, + }; - Target MutationTestExecution => _ => _ + Target MutationTests => _ => _ .DependsOn(Compile) .OnlyWhenDynamic(() => !DisableMutationTests) .Executes(() => @@ -42,20 +52,10 @@ partial class Build DotNetToolInstall(_ => _ .SetPackageName("dotnet-stryker") - .SetVersion("4.10.0") + .SetVersion(GetStrykerVersion()) .SetToolInstallationPath(toolPath)); - Dictionary projects = new() - { - { - Solution.Mockolate, [ - Solution.Tests.Mockolate_Tests, - Solution.Tests.Mockolate_Internal_Tests, - ] - }, - }; - - foreach (KeyValuePair project in projects) + foreach (KeyValuePair project in MutationTestProjects) { string branchName = BranchName; if (GitHubActions?.Ref.StartsWith("refs/tags/", StringComparison.OrdinalIgnoreCase) == true) @@ -65,8 +65,6 @@ partial class Build Log.Information("Use release branch analysis for '{BranchName}'", branchName); } - File.WriteAllText(ArtifactsDirectory / "BranchName.txt", branchName); - string configText = $$""" { "stryker-config": { @@ -109,103 +107,173 @@ partial class Build $"Stryker did not execute successfully for {project.Key.Name}: (exit code {process.ExitCode})."); } - MutationCommentBody += Environment.NewLine + CreateMutationCommentBody(project.Key.Name); + File.WriteAllText(ArtifactsDirectory / $"MutationTest_{project.Key.Name}.md", + CreateMutationCommentBody(project.Key.Name)); } }); - Target MutationTestComment => _ => _ - .After(MutationTestExecution) + Target MutationTestsComment => _ => _ + .After(MutationTests) + .DependsOn(MutationTestsDashboard) .OnlyWhenDynamic(() => !DisableMutationTests) - .OnlyWhenDynamic(() => GitHubActions.IsPullRequest) - .Executes(() => + .Executes(async () => { - int? prId = GitHubActions.PullRequestNumber; - Log.Debug("Pull request number: {PullRequestId}", prId); - if (string.IsNullOrWhiteSpace(MutationCommentBody)) + // The artifacts are untrusted, because the pull request controls the code that produces them, so the pull + // request to comment on comes from the event of the workflow run. + int? prId = await BuildExtensions.ResolvePullRequestOfWorkflowRun(GithubToken); + if (prId == null) + { + Log.Information( + "The workflow run belongs to no pull request, so there is no mutation comment to write"); + return; + } + + List mutationCommentBodies = []; + foreach (AbsolutePath file in ArtifactsDirectory.GetFiles("MutationTest_*.md", 2)) + { + mutationCommentBodies.Add(await File.ReadAllTextAsync(file)); + } + + if (mutationCommentBodies.Count == 0) { + Log.Warning("No files matching \"MutationTest_*.md\" found"); return; } + GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")); + Credentials tokenAuth = new(GithubToken); + gitHubClient.Credentials = tokenAuth; + IReadOnlyList comments = + await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value); + long? commentId = null; + Log.Information($"Found {comments.Count} comments"); + foreach (IssueComment comment in comments) + { + if (comment.IsPipelineComment("## :alien: Mutation Results")) + { + Log.Information($"Found comment: {comment.Body}"); + commentId = comment.Id; + } + } + string body = "## :alien: Mutation Results" + Environment.NewLine + $"[![Mutation testing badge](https://img.shields.io/endpoint?style=flat&url=https%3A%2F%2Fbadge-api.stryker-mutator.io%2Fgithub.com%2F{BuildExtensions.Owner}%2F{BuildExtensions.Repo}%2Fpull/{prId}/merge)](https://dashboard.stryker-mutator.io/reports/github.com/{BuildExtensions.Owner}/{BuildExtensions.Repo}/pull/{prId}/merge)" + Environment.NewLine - + MutationCommentBody; - File.WriteAllText(ArtifactsDirectory / "PR_Comment.md", body); - - if (prId != null) + + string.Join(Environment.NewLine, mutationCommentBodies).AsUntrustedCommentContent(); + if (commentId == null) { - File.WriteAllText(ArtifactsDirectory / "PR.txt", prId.Value.ToString()); + Log.Information($"Create comment:\n{body}"); + await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value, body); + } + else + { + Log.Information($"Update comment:\n{body}"); + await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body); } }); - Target MutationTestDashboard => _ => _ - .After(MutationTestExecution) + Target MutationTestsDashboard => _ => _ + .After(MutationTests) .OnlyWhenDynamic(() => !DisableMutationTests) .Executes(async () => { + string version = await GetMutationDashboardVersion(); + if (version == null) + { + Log.Information("The run has no version on the mutation dashboard, so there is nothing to publish"); + return; + } + await "MutationTests".DownloadArtifactTo(ArtifactsDirectory, GithubToken); - Dictionary projects = new() - { - { - Solution.Mockolate, [ - Solution.Tests.Mockolate_Tests, - Solution.Tests.Mockolate_Internal_Tests, - ] - }, - }; string apiKey = Environment.GetEnvironmentVariable("STRYKER_DASHBOARD_API_KEY"); - string branchName = File.ReadAllText(ArtifactsDirectory / "BranchName.txt"); - foreach (KeyValuePair project in projects) + foreach (Project project in MutationTestProjects.Keys) { - string reportComment = - File.ReadAllText(ArtifactsDirectory / "Stryker" / "reports" / "mutation-report.json"); + AbsolutePath reportFile = ArtifactsDirectory / "Stryker" / "reports" / "mutation-report.json"; + if (!File.Exists(reportFile)) + { + Log.Information( + "Skip uploading the mutation report of {ProjectName}, as Stryker found nothing to mutate", + project.Name); + continue; + } + + if (new FileInfo(reportFile).Length > MaxMutationReportSize) + { + Assert.Fail($"The {project.Name} mutation report is larger than {MaxMutationReportSize} bytes"); + } + + string reportComment = File.ReadAllText(reportFile); + if (JsonNode.Parse(reportComment) is not JsonObject) + { + Assert.Fail($"The {project.Name} mutation report is no JSON object"); + } + using HttpClient client = new(); client.DefaultRequestHeaders.Add("X-Api-Key", apiKey); // https://stryker-mutator.io/docs/General/dashboard/#send-a-report-via-curl - await client.PutAsync( - $"https://dashboard.stryker-mutator.io/api/reports/github.com/{BuildExtensions.Owner}/{BuildExtensions.Repo}/{branchName}?module={project.Key.Name}", + HttpResponseMessage response = await client.PutAsync( + $"https://dashboard.stryker-mutator.io/api/reports/github.com/{BuildExtensions.Owner}/{BuildExtensions.Repo}/{version}?module={project.Name}", new StringContent(reportComment, new MediaTypeHeaderValue("application/json"))); - } - - if (File.Exists(ArtifactsDirectory / "PR.txt")) - { - string prNumber = File.ReadAllText(ArtifactsDirectory / "PR.txt"); - Log.Debug("Pull request number: {PullRequestId}", prNumber); - string body = File.ReadAllText(ArtifactsDirectory / "PR_Comment.md"); - if (int.TryParse(prNumber, out int prId)) + string responseContent = await response.Content.ReadAsStringAsync(); + if (response.IsSuccessStatusCode) { - GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")); - Credentials tokenAuth = new(GithubToken); - gitHubClient.Credentials = tokenAuth; - IReadOnlyList comments = - await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId); - long? commentId = null; - Log.Information($"Found {comments.Count} comments"); - foreach (IssueComment comment in comments) - { - if (comment.Body.Contains("## :alien: Mutation Results")) - { - Log.Information($"Found comment: {comment.Body}"); - commentId = comment.Id; - } - } - - if (commentId == null) - { - Log.Information($"Create comment:\n{body}"); - await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId, body); - } - else - { - Log.Information($"Update comment:\n{body}"); - await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body); - } + Log.Information("Uploaded the {Module} mutation report ({Size} bytes): {Response}", + project.Name, reportComment.Length, responseContent); + } + else + { + // Without this the job stays green while the dashboard keeps showing the previous score. + Assert.Fail( + $"Could not upload the {project.Name} mutation report ({reportComment.Length} bytes), " + + $"the dashboard answered {(int)response.StatusCode} {response.ReasonPhrase}: {responseContent}"); } } }); + /// + /// The version under which the dashboard stores the mutation reports of the analysed run, or + /// when it has none. + /// + /// + /// The artifacts are untrusted, because a pull request controls the code that produces them, so the version + /// comes from the event alone, and a run that a pull request triggered can only publish to the version of + /// that pull request. + /// + private async Task GetMutationDashboardVersion() + { + string version = null; + if (GitHubActions?.EventName != "workflow_run") + { + // The reports come from this very run, so its own ref identifies them. + string gitRef = GitHubActions?.Ref ?? ""; + if (gitRef.StartsWith("refs/tags/", StringComparison.Ordinal)) + { + version = "release/" + gitRef.Substring("refs/tags/".Length); + } + else if (gitRef.StartsWith("refs/heads/", StringComparison.Ordinal)) + { + version = gitRef.Substring("refs/heads/".Length); + } + } + else if (Environment.GetEnvironmentVariable("WorkflowRunEvent") == "pull_request") + { + int? prId = await BuildExtensions.ResolvePullRequestOfWorkflowRun(GithubToken); + return prId == null ? null : $"pull/{prId}/merge"; + } + else + { + version = Environment.GetEnvironmentVariable("WorkflowRunHeadBranch"); + } + + // The version becomes a part of the path that the report is sent to. + return version != null && + Regex.IsMatch(version, @"\A[A-Za-z0-9][A-Za-z0-9._-]*(/[A-Za-z0-9][A-Za-z0-9._-]*)*\z") + ? version + : null; + } + string CreateMutationCommentBody(string projectName) { string[] fileContent = File.ReadAllLines(ArtifactsDirectory / "Stryker" / "reports" / "mutation-report.md"); @@ -249,5 +317,15 @@ string CreateMutationCommentBody(string projectName) return body; } + /// + /// The version of Stryker to install, which is the one that the PackageDownload of the build project + /// pins, so that Dependabot keeps it up to date. + /// + static string GetStrykerVersion() + => XDocument.Load(RootDirectory / "Pipeline" / "Build.csproj") + .Descendants("PackageDownload") + .Single(package => package.Attribute("Include")?.Value == "dotnet-stryker") + .Attribute("Version")!.Value.Trim('[', ']'); + static string PathForJson(Project project) => $"\"{project.Path.ToString().Replace(@"\", @"\\")}\""; } diff --git a/Pipeline/Build.cs b/Pipeline/Build.cs index 8f0a9ad2..3d0c665a 100644 --- a/Pipeline/Build.cs +++ b/Pipeline/Build.cs @@ -18,7 +18,7 @@ partial class Build : FalloutBuild [Parameter("Github Token")] readonly string GithubToken; - [Required] [GitVersion(Framework = "net8.0", NoCache = true, NoFetch = true)] readonly GitVersion GitVersion; + [Required] [GitVersion(Framework = "net10.0", NoCache = true, NoFetch = true)] readonly GitVersion GitVersion; [Solution(GenerateProjects = true)] readonly Solution Solution; diff --git a/Pipeline/Build.csproj b/Pipeline/Build.csproj index cd7dc33f..52471879 100644 --- a/Pipeline/Build.csproj +++ b/Pipeline/Build.csproj @@ -29,7 +29,7 @@ - + diff --git a/Pipeline/BuildExtensions.cs b/Pipeline/BuildExtensions.cs index 2c1decfa..57ff01e3 100644 --- a/Pipeline/BuildExtensions.cs +++ b/Pipeline/BuildExtensions.cs @@ -10,7 +10,9 @@ using Fallout.Common.CI.GitHubActions; using Fallout.Common.Tools.GitVersion; using Fallout.Common.Tools.SonarScanner; +using Octokit; using Serilog; +using ProductHeaderValue = Octokit.ProductHeaderValue; namespace Build; @@ -46,6 +48,97 @@ public static SonarScannerBeginSettings SetPullRequestOrBranchName( return settings.SetBranchName(gitVersion.BranchName); } + /// + /// Resolves the pull request that triggered the workflow run under analysis, or + /// when there is none. + /// + /// + /// The artifacts of that run are untrusted, because the pull request controls the code that produces them, so + /// the pull request is identified by the workflow_run event alone. The event lists no pull requests for + /// a fork, which is why the number is looked up by the head branch and verified against the head commit. + /// + /// A pull request that was merged or closed while its run was analysed is still found. When the same commit + /// of the same branch belongs to several pull requests, the open one is taken. + /// + public static async Task ResolvePullRequestOfWorkflowRun(string githubToken) + { + string workflowRunEvent = Environment.GetEnvironmentVariable("WorkflowRunEvent"); + string headOwner = Environment.GetEnvironmentVariable("WorkflowRunHeadOwner"); + string headBranch = Environment.GetEnvironmentVariable("WorkflowRunHeadBranch"); + string headSha = Environment.GetEnvironmentVariable("WorkflowRunHeadSha"); + if (workflowRunEvent != "pull_request" || + string.IsNullOrEmpty(headOwner) || string.IsNullOrEmpty(headBranch) || string.IsNullOrEmpty(headSha)) + { + Log.Information("The workflow run was not triggered by a pull request"); + return null; + } + + GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")) + { + Credentials = new Credentials(githubToken), + }; + IReadOnlyList pullRequests = await gitHubClient.PullRequest.GetAllForRepository( + Owner, Repo, + new PullRequestRequest + { + State = ItemStateFilter.All, + Head = $"{headOwner}:{headBranch}", + }); + // GitHub ignores a head filter it cannot interpret and then lists every pull request. + PullRequest[] matches = pullRequests + .Where(pullRequest => pullRequest.Head.Sha == headSha && + pullRequest.Head.Ref == headBranch && + string.Equals(pullRequest.Head.User?.Login, headOwner, + StringComparison.OrdinalIgnoreCase)) + .ToArray(); + if (matches.Length > 1) + { + matches = matches.Where(pullRequest => pullRequest.State.Value == ItemState.Open).ToArray(); + } + + int[] numbers = matches.Select(pullRequest => pullRequest.Number).ToArray(); + if (numbers.Length != 1) + { + Log.Information("Found {Count} pull requests for the commit {Sha} of the workflow run", + numbers.Length, headSha); + return null; + } + + Log.Information("The workflow run was triggered by the pull request #{PullRequest}", numbers[0]); + return numbers[0]; + } + + /// + /// Whether the is the one that the pipeline wrote under the + /// . + /// + /// + /// Anyone can write a comment that contains the heading, so the author has to match too. + /// + public static bool IsPipelineComment(this IssueComment comment, string heading) + => comment.User?.Login == "github-actions[bot]" && + comment.Body?.StartsWith(heading, StringComparison.Ordinal) == true; + + /// + /// Prepares the of an artifact for a comment of the pipeline. + /// + /// + /// The artifact is untrusted, so the markdown is bound to what a comment can hold, and a zero-width space + /// after every @ keeps it from notifying users and teams. + /// + public static string AsUntrustedCommentContent(this string markdown) + { + const int maxLength = 60000; + markdown = markdown.Replace("@", "@​"); + if (markdown.Length > maxLength) + { + markdown = markdown.Substring(0, maxLength) + Environment.NewLine + Environment.NewLine + + "_The report was truncated._"; + } + + return markdown; + } + public static Task DownloadArtifactTo(this string artifactName, string artifactsDirectory, string githubToken) => DownloadArtifactsWhere(name => name.Equals(artifactName, StringComparison.OrdinalIgnoreCase), artifactsDirectory, githubToken); @@ -146,6 +239,14 @@ private static async Task DownloadArtifactsFromRun(string runId, Func ((entry.ExternalAttributes >> 16) & 0xF000) == 0xA000)) + { + throw new InvalidOperationException( + $"The artifact #{artifactId} contains a symbolic link"); + } + archive.ExtractToDirectory(artifactsDirectory, true); Log.Information( $"Extracted artifact '{name}' (#{artifactId}) with {archive.Entries.Count} entries to {artifactsDirectory}:\n - {string.Join("\n - ", archive.Entries.Select(entry => $"{entry.Name} ({entry.Length})"))}");