diff --git a/.fallout/build.schema.json b/.fallout/build.schema.json index 8d700115..5ef3e111 100644 --- a/.fallout/build.schema.json +++ b/.fallout/build.schema.json @@ -37,10 +37,9 @@ "CodeCoverage", "Compile", "DotNetUnitTests", - "MutationTestComment", - "MutationTestDashboard", - "MutationTestExecution", "MutationTests", + "MutationTestsComment", + "MutationTestsDashboard", "Pack", "PublishBenchmarkReport", "Restore", diff --git a/.gitattributes b/.gitattributes index 1f24147a..a8341d80 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,7 +7,7 @@ # ReSharper DotSettings files are in Unix Format *.DotSettings text eol=lf -*.sln text eol=crlf +*.slnx text eol=crlf # Allows checking out and developing in Windows # while mounting and running tests in Linux diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 39b08e4b..29277b1a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,3 +29,6 @@ updates: aweXpect: patterns: - "aweXpect*" + # The analyzers and generators only load in compilers at least as new as the Roslyn they reference. + ignore: + - dependency-name: "Microsoft.CodeAnalysis*" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df74b88f..358cedcb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,7 +149,7 @@ jobs: 8.0.x 10.0.x - name: Run mutation tests - run: ./build.sh MutationTests MutationTestDashboard + run: ./build.sh MutationTests MutationTestsDashboard static-code-analysis: name: "Static code analysis" diff --git a/.github/workflows/ci-analysis.yml b/.github/workflows/ci-analysis.yml index d1483f63..f8da99c7 100644 --- a/.github/workflows/ci-analysis.yml +++ b/.github/workflows/ci-analysis.yml @@ -12,39 +12,55 @@ jobs: name: "Mutation tests" runs-on: ubuntu-latest permissions: + actions: read + contents: read pull-requests: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 10.0.x - - name: Create mutation test comment - run: ./build.sh MutationTestComment + - name: Upload mutation dashboard and create comment + run: ./build.sh MutationTestsComment + # The artifacts of the workflow run are untrusted, because a pull request controls the code that + # produces them, so the build identifies the pull request by these values of the event alone. env: GithubToken: ${{ secrets.GITHUB_TOKEN }} + STRYKER_DASHBOARD_API_KEY: ${{ secrets.STRYKER_DASHBOARD_API_KEY }} WorkflowRunId: ${{ github.event.workflow_run.id }} - + WorkflowRunEvent: ${{ github.event.workflow_run.event }} + WorkflowRunHeadOwner: ${{ github.event.workflow_run.head_repository.owner.login }} + WorkflowRunHeadBranch: ${{ github.event.workflow_run.head_branch }} + WorkflowRunHeadSha: ${{ github.event.workflow_run.head_sha }} + benchmarks: name: "Benchmarks" runs-on: ubuntu-latest permissions: - contents: write + actions: read + contents: read pull-requests: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 10.0.x - name: Create benchmark comment run: ./build.sh BenchmarkComment + # The artifacts of the workflow run are untrusted, because a pull request controls the code that + # produces them, so the build identifies the pull request by these values of the event alone. env: GithubToken: ${{ secrets.GITHUB_TOKEN }} WorkflowRunId: ${{ github.event.workflow_run.id }} + WorkflowRunEvent: ${{ github.event.workflow_run.event }} + WorkflowRunHeadOwner: ${{ github.event.workflow_run.head_repository.owner.login }} + WorkflowRunHeadBranch: ${{ github.event.workflow_run.head_branch }} + WorkflowRunHeadSha: ${{ github.event.workflow_run.head_sha }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3611296c..5d610b6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,11 +14,11 @@ jobs: os: [ ubuntu-latest, windows-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -31,7 +31,7 @@ jobs: run: ./build.sh CodeCoverage - name: Upload artifacts if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ runner.os }}-artifacts path: | @@ -44,11 +44,11 @@ jobs: env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -57,7 +57,7 @@ jobs: run: ./build.sh ApiChecks - name: Upload artifacts if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: API-tests path: | @@ -70,11 +70,11 @@ jobs: env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -83,7 +83,7 @@ jobs: run: ./build.sh MutationTests - name: Upload artifacts if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: MutationTests path: | @@ -105,11 +105,11 @@ jobs: env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -118,7 +118,7 @@ jobs: run: ./build.sh Benchmarks --benchmark-filter "*${{ matrix.benchmark }}*" - name: Upload artifacts if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: Benchmarks-${{ matrix.benchmark }} path: | @@ -134,11 +134,11 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Setup .NET SDKs - uses: actions/setup-dotnet@v6 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: | 8.0.x @@ -156,11 +156,11 @@ jobs: if: always() steps: - name: Download Artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: path: artifacts - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 + uses: EnricoMi/publish-unit-test-result-action@7622c5bf3d112a72ab4fa3951c2aa5723d8ab92d # v2.25.0 with: comment_mode: always files: "artifacts/**/**/*.trx" diff --git a/.github/workflows/notify-docs-site.yml b/.github/workflows/notify-docs-site.yml index d9b37588..2b7a10ec 100644 --- a/.github/workflows/notify-docs-site.yml +++ b/.github/workflows/notify-docs-site.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Dispatch extension-documentation-updated-event to Testably/Testably.Site - uses: peter-evans/repository-dispatch@v4 + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: token: ${{ secrets.SITE_DISPATCH_TOKEN }} repository: Testably/Testably.Site diff --git a/Directory.Packages.props b/Directory.Packages.props index 4ff7250f..c141a6fa 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -35,7 +35,7 @@ - + diff --git a/Mockolate.slnx b/Mockolate.slnx index 36e94390..b7223a04 100644 --- a/Mockolate.slnx +++ b/Mockolate.slnx @@ -31,7 +31,7 @@ - + diff --git a/Mockolate.sln.DotSettings b/Mockolate.slnx.DotSettings similarity index 100% rename from Mockolate.sln.DotSettings rename to Mockolate.slnx.DotSettings diff --git a/Pipeline/Build.Benchmarks.cs b/Pipeline/Build.Benchmarks.cs index 641852ec..22d4597c 100644 --- a/Pipeline/Build.Benchmarks.cs +++ b/Pipeline/Build.Benchmarks.cs @@ -63,16 +63,20 @@ partial class Build string fileContent = await File.ReadAllTextAsync(file); Log.Information("Report ({FileName}):\n {FileContent}", Path.GetFileName(file), fileContent); } - - if (GitHubActions?.IsPullRequest == true) - { - File.WriteAllText(ArtifactsDirectory / "PR.txt", GitHubActions.PullRequestNumber.ToString()); - } }); Target BenchmarkComment => _ => _ .Executes(async () => { + // The artifacts are untrusted, because the pull request controls the code that produces them, so the pull + // request to comment on comes from the event of the workflow run. + int? prId = await BuildExtensions.ResolvePullRequestOfWorkflowRun(GithubToken); + if (prId == null) + { + Log.Information("Skip writing a comment, as the workflow run belongs to no pull request."); + return; + } + await "Benchmarks-".DownloadArtifactsStartingWith(ArtifactsDirectory, GithubToken); if (!Directory.Exists(ArtifactsDirectory / "Benchmarks" / "results")) { @@ -87,46 +91,35 @@ partial class Build return; } - if (!File.Exists(ArtifactsDirectory / "PR.txt")) - { - Log.Information("Skip writing a comment, as no PR number was specified."); - return; - } - AbsolutePath baselineDirectory = ArtifactsDirectory / "Baseline"; string baselineResultsDirectory = await DownloadBaselineBenchmarks(baselineDirectory); - string prNumber = File.ReadAllText(ArtifactsDirectory / "PR.txt"); - string body = CreateBenchmarkCommentBody(files, baselineResultsDirectory); - Log.Debug("Pull request number: {PullRequestId}", prNumber); - if (int.TryParse(prNumber, out int prId)) + string body = CreateBenchmarkCommentBody(files, baselineResultsDirectory).AsUntrustedCommentContent(); + GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")); + Credentials tokenAuth = new(GithubToken); + gitHubClient.Credentials = tokenAuth; + IReadOnlyList comments = + await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value); + long? commentId = null; + Log.Information($"Found {comments.Count} comments"); + foreach (IssueComment comment in comments) { - GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")); - Credentials tokenAuth = new(GithubToken); - gitHubClient.Credentials = tokenAuth; - IReadOnlyList comments = - await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId); - long? commentId = null; - Log.Information($"Found {comments.Count} comments"); - foreach (IssueComment comment in comments) + if (comment.IsPipelineComment("## :rocket: Benchmark Results")) { - if (comment.Body.Contains("## :rocket: Benchmark Results")) - { - Log.Information($"Found comment: {comment.Body}"); - commentId = comment.Id; - } + Log.Information($"Found comment: {comment.Body}"); + commentId = comment.Id; } + } - if (commentId == null) - { - Log.Information($"Create comment:\n{body}"); - await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId, body); - } - else - { - Log.Information($"Update comment:\n{body}"); - await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body); - } + if (commentId == null) + { + Log.Information($"Create comment:\n{body}"); + await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value, body); + } + else + { + Log.Information($"Update comment:\n{body}"); + await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body); } }); diff --git a/Pipeline/Build.CodeAnalysis.cs b/Pipeline/Build.CodeAnalysis.cs index ec5c7019..a947ed23 100644 --- a/Pipeline/Build.CodeAnalysis.cs +++ b/Pipeline/Build.CodeAnalysis.cs @@ -19,7 +19,7 @@ partial class Build .SetOrganization("testably") .SetProjectKey("Testably_Mockolate") .AddVSTestReports(TestResultsDirectory / "*.trx") - .AddOpenCoverPaths(TestResultsDirectory / "reports" / "OpenCover.xml") + .AddGenericCoveragePaths(TestResultsDirectory / "reports" / "SonarQube.xml") .SetPullRequestOrBranchName(GitHubActions, GitVersion) .SetVersion(GitVersion.SemVer) .SetToken(SonarToken)); diff --git a/Pipeline/Build.CodeCoverage.cs b/Pipeline/Build.CodeCoverage.cs index 7631d451..0efea962 100644 --- a/Pipeline/Build.CodeCoverage.cs +++ b/Pipeline/Build.CodeCoverage.cs @@ -15,10 +15,12 @@ partial class Build { ReportGenerator(s => s .SetProcessToolPath(NuGetToolPathResolver.GetPackageExecutable("ReportGenerator", "ReportGenerator.dll", - framework: "net8.0")) + framework: "net10.0")) .SetTargetDirectory(TestResultsDirectory / "reports") .AddReports(TestResultsDirectory / "**/coverage.cobertura.xml") - .AddReportTypes(ReportTypes.OpenCover) + // The generic format of Sonar lists the lines per file. A format that lists them per method leaves out the + // lines of a lambda in a generic class, because ReportGenerator keeps them in a class without methods. + .AddReportTypes(ReportTypes.SonarQube) .AddFileFilters("-*.g.cs") .SetAssemblyFilters("+Mockolate*")); }); diff --git a/Pipeline/Build.Compile.cs b/Pipeline/Build.Compile.cs index 1cd95957..973c502b 100644 --- a/Pipeline/Build.Compile.cs +++ b/Pipeline/Build.Compile.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Text.RegularExpressions; using Fallout.Common; using Fallout.Common.IO; using Fallout.Common.Tools.DotNet; @@ -30,6 +31,13 @@ partial class Build } else if (GitHubActions.Ref.StartsWith("refs/tags/", StringComparison.OrdinalIgnoreCase)) { + if (!Regex.IsMatch(GitHubActions.Ref, + @"\Arefs/tags/v\d+\.\d+\.\d+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?\z")) + { + Assert.Fail( + $"The tag '{GitHubActions.Ref["refs/tags/".Length..]}' is no valid release tag (expected 'v1.2.3', optionally with a pre-release suffix like '-pre.1')"); + } + int preReleaseIndex = GitHubActions.Ref.IndexOf('-'); preRelease = preReleaseIndex > 0 ? GitHubActions.Ref[preReleaseIndex..] : ""; } diff --git a/Pipeline/Build.MutationTests.cs b/Pipeline/Build.MutationTests.cs index dedfd0bb..f25b7613 100644 --- a/Pipeline/Build.MutationTests.cs +++ b/Pipeline/Build.MutationTests.cs @@ -5,6 +5,10 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Xml.Linq; using Fallout.Common; using Fallout.Common.IO; using Fallout.Common.Tooling; @@ -23,13 +27,19 @@ namespace Build; partial class Build { private static readonly bool DisableMutationTests = false; - private static string MutationCommentBody = ""; + private const long MaxMutationReportSize = 100 * 1024 * 1024; - Target MutationTests => _ => _ - .DependsOn(MutationTestExecution) - .DependsOn(MutationTestComment); + Dictionary MutationTestProjects => new() + { + { + Solution.Mockolate, [ + Solution.Tests.Mockolate_Tests, + Solution.Tests.Mockolate_Internal_Tests, + ] + }, + }; - Target MutationTestExecution => _ => _ + Target MutationTests => _ => _ .DependsOn(Compile) .OnlyWhenDynamic(() => !DisableMutationTests) .Executes(() => @@ -42,20 +52,10 @@ partial class Build DotNetToolInstall(_ => _ .SetPackageName("dotnet-stryker") - .SetVersion("4.10.0") + .SetVersion(GetStrykerVersion()) .SetToolInstallationPath(toolPath)); - Dictionary projects = new() - { - { - Solution.Mockolate, [ - Solution.Tests.Mockolate_Tests, - Solution.Tests.Mockolate_Internal_Tests, - ] - }, - }; - - foreach (KeyValuePair project in projects) + foreach (KeyValuePair project in MutationTestProjects) { string branchName = BranchName; if (GitHubActions?.Ref.StartsWith("refs/tags/", StringComparison.OrdinalIgnoreCase) == true) @@ -65,8 +65,6 @@ partial class Build Log.Information("Use release branch analysis for '{BranchName}'", branchName); } - File.WriteAllText(ArtifactsDirectory / "BranchName.txt", branchName); - string configText = $$""" { "stryker-config": { @@ -109,103 +107,173 @@ partial class Build $"Stryker did not execute successfully for {project.Key.Name}: (exit code {process.ExitCode})."); } - MutationCommentBody += Environment.NewLine + CreateMutationCommentBody(project.Key.Name); + File.WriteAllText(ArtifactsDirectory / $"MutationTest_{project.Key.Name}.md", + CreateMutationCommentBody(project.Key.Name)); } }); - Target MutationTestComment => _ => _ - .After(MutationTestExecution) + Target MutationTestsComment => _ => _ + .After(MutationTests) + .DependsOn(MutationTestsDashboard) .OnlyWhenDynamic(() => !DisableMutationTests) - .OnlyWhenDynamic(() => GitHubActions.IsPullRequest) - .Executes(() => + .Executes(async () => { - int? prId = GitHubActions.PullRequestNumber; - Log.Debug("Pull request number: {PullRequestId}", prId); - if (string.IsNullOrWhiteSpace(MutationCommentBody)) + // The artifacts are untrusted, because the pull request controls the code that produces them, so the pull + // request to comment on comes from the event of the workflow run. + int? prId = await BuildExtensions.ResolvePullRequestOfWorkflowRun(GithubToken); + if (prId == null) + { + Log.Information( + "The workflow run belongs to no pull request, so there is no mutation comment to write"); + return; + } + + List mutationCommentBodies = []; + foreach (AbsolutePath file in ArtifactsDirectory.GetFiles("MutationTest_*.md", 2)) + { + mutationCommentBodies.Add(await File.ReadAllTextAsync(file)); + } + + if (mutationCommentBodies.Count == 0) { + Log.Warning("No files matching \"MutationTest_*.md\" found"); return; } + GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")); + Credentials tokenAuth = new(GithubToken); + gitHubClient.Credentials = tokenAuth; + IReadOnlyList comments = + await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value); + long? commentId = null; + Log.Information($"Found {comments.Count} comments"); + foreach (IssueComment comment in comments) + { + if (comment.IsPipelineComment("## :alien: Mutation Results")) + { + Log.Information($"Found comment: {comment.Body}"); + commentId = comment.Id; + } + } + string body = "## :alien: Mutation Results" + Environment.NewLine + $"[![Mutation testing badge](https://img.shields.io/endpoint?style=flat&url=https%3A%2F%2Fbadge-api.stryker-mutator.io%2Fgithub.com%2F{BuildExtensions.Owner}%2F{BuildExtensions.Repo}%2Fpull/{prId}/merge)](https://dashboard.stryker-mutator.io/reports/github.com/{BuildExtensions.Owner}/{BuildExtensions.Repo}/pull/{prId}/merge)" + Environment.NewLine - + MutationCommentBody; - File.WriteAllText(ArtifactsDirectory / "PR_Comment.md", body); - - if (prId != null) + + string.Join(Environment.NewLine, mutationCommentBodies).AsUntrustedCommentContent(); + if (commentId == null) { - File.WriteAllText(ArtifactsDirectory / "PR.txt", prId.Value.ToString()); + Log.Information($"Create comment:\n{body}"); + await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value, body); + } + else + { + Log.Information($"Update comment:\n{body}"); + await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body); } }); - Target MutationTestDashboard => _ => _ - .After(MutationTestExecution) + Target MutationTestsDashboard => _ => _ + .After(MutationTests) .OnlyWhenDynamic(() => !DisableMutationTests) .Executes(async () => { + string version = await GetMutationDashboardVersion(); + if (version == null) + { + Log.Information("The run has no version on the mutation dashboard, so there is nothing to publish"); + return; + } + await "MutationTests".DownloadArtifactTo(ArtifactsDirectory, GithubToken); - Dictionary projects = new() - { - { - Solution.Mockolate, [ - Solution.Tests.Mockolate_Tests, - Solution.Tests.Mockolate_Internal_Tests, - ] - }, - }; string apiKey = Environment.GetEnvironmentVariable("STRYKER_DASHBOARD_API_KEY"); - string branchName = File.ReadAllText(ArtifactsDirectory / "BranchName.txt"); - foreach (KeyValuePair project in projects) + foreach (Project project in MutationTestProjects.Keys) { - string reportComment = - File.ReadAllText(ArtifactsDirectory / "Stryker" / "reports" / "mutation-report.json"); + AbsolutePath reportFile = ArtifactsDirectory / "Stryker" / "reports" / "mutation-report.json"; + if (!File.Exists(reportFile)) + { + Log.Information( + "Skip uploading the mutation report of {ProjectName}, as Stryker found nothing to mutate", + project.Name); + continue; + } + + if (new FileInfo(reportFile).Length > MaxMutationReportSize) + { + Assert.Fail($"The {project.Name} mutation report is larger than {MaxMutationReportSize} bytes"); + } + + string reportComment = File.ReadAllText(reportFile); + if (JsonNode.Parse(reportComment) is not JsonObject) + { + Assert.Fail($"The {project.Name} mutation report is no JSON object"); + } + using HttpClient client = new(); client.DefaultRequestHeaders.Add("X-Api-Key", apiKey); // https://stryker-mutator.io/docs/General/dashboard/#send-a-report-via-curl - await client.PutAsync( - $"https://dashboard.stryker-mutator.io/api/reports/github.com/{BuildExtensions.Owner}/{BuildExtensions.Repo}/{branchName}?module={project.Key.Name}", + HttpResponseMessage response = await client.PutAsync( + $"https://dashboard.stryker-mutator.io/api/reports/github.com/{BuildExtensions.Owner}/{BuildExtensions.Repo}/{version}?module={project.Name}", new StringContent(reportComment, new MediaTypeHeaderValue("application/json"))); - } - - if (File.Exists(ArtifactsDirectory / "PR.txt")) - { - string prNumber = File.ReadAllText(ArtifactsDirectory / "PR.txt"); - Log.Debug("Pull request number: {PullRequestId}", prNumber); - string body = File.ReadAllText(ArtifactsDirectory / "PR_Comment.md"); - if (int.TryParse(prNumber, out int prId)) + string responseContent = await response.Content.ReadAsStringAsync(); + if (response.IsSuccessStatusCode) { - GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")); - Credentials tokenAuth = new(GithubToken); - gitHubClient.Credentials = tokenAuth; - IReadOnlyList comments = - await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId); - long? commentId = null; - Log.Information($"Found {comments.Count} comments"); - foreach (IssueComment comment in comments) - { - if (comment.Body.Contains("## :alien: Mutation Results")) - { - Log.Information($"Found comment: {comment.Body}"); - commentId = comment.Id; - } - } - - if (commentId == null) - { - Log.Information($"Create comment:\n{body}"); - await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId, body); - } - else - { - Log.Information($"Update comment:\n{body}"); - await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body); - } + Log.Information("Uploaded the {Module} mutation report ({Size} bytes): {Response}", + project.Name, reportComment.Length, responseContent); + } + else + { + // Without this the job stays green while the dashboard keeps showing the previous score. + Assert.Fail( + $"Could not upload the {project.Name} mutation report ({reportComment.Length} bytes), " + + $"the dashboard answered {(int)response.StatusCode} {response.ReasonPhrase}: {responseContent}"); } } }); + /// + /// The version under which the dashboard stores the mutation reports of the analysed run, or + /// when it has none. + /// + /// + /// The artifacts are untrusted, because a pull request controls the code that produces them, so the version + /// comes from the event alone, and a run that a pull request triggered can only publish to the version of + /// that pull request. + /// + private async Task GetMutationDashboardVersion() + { + string version = null; + if (GitHubActions?.EventName != "workflow_run") + { + // The reports come from this very run, so its own ref identifies them. + string gitRef = GitHubActions?.Ref ?? ""; + if (gitRef.StartsWith("refs/tags/", StringComparison.Ordinal)) + { + version = "release/" + gitRef.Substring("refs/tags/".Length); + } + else if (gitRef.StartsWith("refs/heads/", StringComparison.Ordinal)) + { + version = gitRef.Substring("refs/heads/".Length); + } + } + else if (Environment.GetEnvironmentVariable("WorkflowRunEvent") == "pull_request") + { + int? prId = await BuildExtensions.ResolvePullRequestOfWorkflowRun(GithubToken); + return prId == null ? null : $"pull/{prId}/merge"; + } + else + { + version = Environment.GetEnvironmentVariable("WorkflowRunHeadBranch"); + } + + // The version becomes a part of the path that the report is sent to. + return version != null && + Regex.IsMatch(version, @"\A[A-Za-z0-9][A-Za-z0-9._-]*(/[A-Za-z0-9][A-Za-z0-9._-]*)*\z") + ? version + : null; + } + string CreateMutationCommentBody(string projectName) { string[] fileContent = File.ReadAllLines(ArtifactsDirectory / "Stryker" / "reports" / "mutation-report.md"); @@ -249,5 +317,15 @@ string CreateMutationCommentBody(string projectName) return body; } + /// + /// The version of Stryker to install, which is the one that the PackageDownload of the build project + /// pins, so that Dependabot keeps it up to date. + /// + static string GetStrykerVersion() + => XDocument.Load(RootDirectory / "Pipeline" / "Build.csproj") + .Descendants("PackageDownload") + .Single(package => package.Attribute("Include")?.Value == "dotnet-stryker") + .Attribute("Version")!.Value.Trim('[', ']'); + static string PathForJson(Project project) => $"\"{project.Path.ToString().Replace(@"\", @"\\")}\""; } diff --git a/Pipeline/Build.cs b/Pipeline/Build.cs index 8f0a9ad2..3d0c665a 100644 --- a/Pipeline/Build.cs +++ b/Pipeline/Build.cs @@ -18,7 +18,7 @@ partial class Build : FalloutBuild [Parameter("Github Token")] readonly string GithubToken; - [Required] [GitVersion(Framework = "net8.0", NoCache = true, NoFetch = true)] readonly GitVersion GitVersion; + [Required] [GitVersion(Framework = "net10.0", NoCache = true, NoFetch = true)] readonly GitVersion GitVersion; [Solution(GenerateProjects = true)] readonly Solution Solution; diff --git a/Pipeline/Build.csproj b/Pipeline/Build.csproj index cd7dc33f..52471879 100644 --- a/Pipeline/Build.csproj +++ b/Pipeline/Build.csproj @@ -29,7 +29,7 @@ - + diff --git a/Pipeline/BuildExtensions.cs b/Pipeline/BuildExtensions.cs index 2c1decfa..57ff01e3 100644 --- a/Pipeline/BuildExtensions.cs +++ b/Pipeline/BuildExtensions.cs @@ -10,7 +10,9 @@ using Fallout.Common.CI.GitHubActions; using Fallout.Common.Tools.GitVersion; using Fallout.Common.Tools.SonarScanner; +using Octokit; using Serilog; +using ProductHeaderValue = Octokit.ProductHeaderValue; namespace Build; @@ -46,6 +48,97 @@ public static SonarScannerBeginSettings SetPullRequestOrBranchName( return settings.SetBranchName(gitVersion.BranchName); } + /// + /// Resolves the pull request that triggered the workflow run under analysis, or + /// when there is none. + /// + /// + /// The artifacts of that run are untrusted, because the pull request controls the code that produces them, so + /// the pull request is identified by the workflow_run event alone. The event lists no pull requests for + /// a fork, which is why the number is looked up by the head branch and verified against the head commit. + /// + /// A pull request that was merged or closed while its run was analysed is still found. When the same commit + /// of the same branch belongs to several pull requests, the open one is taken. + /// + public static async Task ResolvePullRequestOfWorkflowRun(string githubToken) + { + string workflowRunEvent = Environment.GetEnvironmentVariable("WorkflowRunEvent"); + string headOwner = Environment.GetEnvironmentVariable("WorkflowRunHeadOwner"); + string headBranch = Environment.GetEnvironmentVariable("WorkflowRunHeadBranch"); + string headSha = Environment.GetEnvironmentVariable("WorkflowRunHeadSha"); + if (workflowRunEvent != "pull_request" || + string.IsNullOrEmpty(headOwner) || string.IsNullOrEmpty(headBranch) || string.IsNullOrEmpty(headSha)) + { + Log.Information("The workflow run was not triggered by a pull request"); + return null; + } + + GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout")) + { + Credentials = new Credentials(githubToken), + }; + IReadOnlyList pullRequests = await gitHubClient.PullRequest.GetAllForRepository( + Owner, Repo, + new PullRequestRequest + { + State = ItemStateFilter.All, + Head = $"{headOwner}:{headBranch}", + }); + // GitHub ignores a head filter it cannot interpret and then lists every pull request. + PullRequest[] matches = pullRequests + .Where(pullRequest => pullRequest.Head.Sha == headSha && + pullRequest.Head.Ref == headBranch && + string.Equals(pullRequest.Head.User?.Login, headOwner, + StringComparison.OrdinalIgnoreCase)) + .ToArray(); + if (matches.Length > 1) + { + matches = matches.Where(pullRequest => pullRequest.State.Value == ItemState.Open).ToArray(); + } + + int[] numbers = matches.Select(pullRequest => pullRequest.Number).ToArray(); + if (numbers.Length != 1) + { + Log.Information("Found {Count} pull requests for the commit {Sha} of the workflow run", + numbers.Length, headSha); + return null; + } + + Log.Information("The workflow run was triggered by the pull request #{PullRequest}", numbers[0]); + return numbers[0]; + } + + /// + /// Whether the is the one that the pipeline wrote under the + /// . + /// + /// + /// Anyone can write a comment that contains the heading, so the author has to match too. + /// + public static bool IsPipelineComment(this IssueComment comment, string heading) + => comment.User?.Login == "github-actions[bot]" && + comment.Body?.StartsWith(heading, StringComparison.Ordinal) == true; + + /// + /// Prepares the of an artifact for a comment of the pipeline. + /// + /// + /// The artifact is untrusted, so the markdown is bound to what a comment can hold, and a zero-width space + /// after every @ keeps it from notifying users and teams. + /// + public static string AsUntrustedCommentContent(this string markdown) + { + const int maxLength = 60000; + markdown = markdown.Replace("@", "@​"); + if (markdown.Length > maxLength) + { + markdown = markdown.Substring(0, maxLength) + Environment.NewLine + Environment.NewLine + + "_The report was truncated._"; + } + + return markdown; + } + public static Task DownloadArtifactTo(this string artifactName, string artifactsDirectory, string githubToken) => DownloadArtifactsWhere(name => name.Equals(artifactName, StringComparison.OrdinalIgnoreCase), artifactsDirectory, githubToken); @@ -146,6 +239,14 @@ private static async Task DownloadArtifactsFromRun(string runId, Func ((entry.ExternalAttributes >> 16) & 0xF000) == 0xA000)) + { + throw new InvalidOperationException( + $"The artifact #{artifactId} contains a symbolic link"); + } + archive.ExtractToDirectory(artifactsDirectory, true); Log.Information( $"Extracted artifact '{name}' (#{artifactId}) with {archive.Entries.Count} entries to {artifactsDirectory}:\n - {string.Join("\n - ", archive.Entries.Select(entry => $"{entry.Name} ({entry.Length})"))}");