Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions .fallout/build.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,9 @@
"CodeCoverage",
"Compile",
"DotNetUnitTests",
"MutationTestComment",
"MutationTestDashboard",
"MutationTestExecution",
"MutationTests",
"MutationTestsComment",
"MutationTestsDashboard",
"Pack",
"PublishBenchmarkReport",
"Restore",
Expand Down
3 changes: 3 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,6 @@ updates:
aweXpect:
patterns:
- "aweXpect*"
# The analyzers and generators only load in compilers at least as new as the Roslyn they reference.
ignore:
- dependency-name: "Microsoft.CodeAnalysis*"
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ jobs:
8.0.x
10.0.x
- name: Run mutation tests
run: ./build.sh MutationTests MutationTestDashboard
run: ./build.sh MutationTests MutationTestsDashboard

static-code-analysis:
name: "Static code analysis"
Expand Down
32 changes: 24 additions & 8 deletions .github/workflows/ci-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,39 +12,55 @@ jobs:
name: "Mutation tests"
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Setup .NET SDKs
uses: actions/setup-dotnet@v6
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
with:
dotnet-version: |
10.0.x
- name: Create mutation test comment
run: ./build.sh MutationTestComment
- name: Upload mutation dashboard and create comment
run: ./build.sh MutationTestsComment
# The artifacts of the workflow run are untrusted, because a pull request controls the code that
# produces them, so the build identifies the pull request by these values of the event alone.
env:
GithubToken: ${{ secrets.GITHUB_TOKEN }}
STRYKER_DASHBOARD_API_KEY: ${{ secrets.STRYKER_DASHBOARD_API_KEY }}
WorkflowRunId: ${{ github.event.workflow_run.id }}

WorkflowRunEvent: ${{ github.event.workflow_run.event }}
WorkflowRunHeadOwner: ${{ github.event.workflow_run.head_repository.owner.login }}
WorkflowRunHeadBranch: ${{ github.event.workflow_run.head_branch }}
WorkflowRunHeadSha: ${{ github.event.workflow_run.head_sha }}

benchmarks:
name: "Benchmarks"
runs-on: ubuntu-latest
permissions:
contents: write
actions: read
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Setup .NET SDKs
uses: actions/setup-dotnet@v6
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
with:
dotnet-version: |
10.0.x
- name: Create benchmark comment
run: ./build.sh BenchmarkComment
# The artifacts of the workflow run are untrusted, because a pull request controls the code that
# produces them, so the build identifies the pull request by these values of the event alone.
env:
GithubToken: ${{ secrets.GITHUB_TOKEN }}
WorkflowRunId: ${{ github.event.workflow_run.id }}
WorkflowRunEvent: ${{ github.event.workflow_run.event }}
WorkflowRunHeadOwner: ${{ github.event.workflow_run.head_repository.owner.login }}
WorkflowRunHeadBranch: ${{ github.event.workflow_run.head_branch }}
WorkflowRunHeadSha: ${{ github.event.workflow_run.head_sha }}
32 changes: 16 additions & 16 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ jobs:
os: [ ubuntu-latest, windows-latest, macos-latest ]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Setup .NET SDKs
uses: actions/setup-dotnet@v6
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
with:
dotnet-version: |
8.0.x
Expand All @@ -31,7 +31,7 @@ jobs:
run: ./build.sh CodeCoverage
- name: Upload artifacts
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: ${{ runner.os }}-artifacts
path: |
Expand All @@ -44,11 +44,11 @@ jobs:
env:
DOTNET_NOLOGO: true
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Setup .NET SDKs
uses: actions/setup-dotnet@v6
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
with:
dotnet-version: |
8.0.x
Expand All @@ -57,7 +57,7 @@ jobs:
run: ./build.sh ApiChecks
- name: Upload artifacts
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: API-tests
path: |
Expand All @@ -70,11 +70,11 @@ jobs:
env:
DOTNET_NOLOGO: true
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Setup .NET SDKs
uses: actions/setup-dotnet@v6
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
with:
dotnet-version: |
8.0.x
Expand All @@ -83,7 +83,7 @@ jobs:
run: ./build.sh MutationTests
- name: Upload artifacts
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: MutationTests
path: |
Expand All @@ -105,11 +105,11 @@ jobs:
env:
DOTNET_NOLOGO: true
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Setup .NET SDKs
uses: actions/setup-dotnet@v6
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
with:
dotnet-version: |
8.0.x
Expand All @@ -118,7 +118,7 @@ jobs:
run: ./build.sh Benchmarks --benchmark-filter "*${{ matrix.benchmark }}*"
- name: Upload artifacts
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: Benchmarks-${{ matrix.benchmark }}
path: |
Expand All @@ -134,11 +134,11 @@ jobs:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
DOTNET_NOLOGO: true
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- name: Setup .NET SDKs
uses: actions/setup-dotnet@v6
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
with:
dotnet-version: |
8.0.x
Expand All @@ -156,11 +156,11 @@ jobs:
if: always()
steps:
- name: Download Artifacts
uses: actions/download-artifact@v8
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
path: artifacts
- name: Publish Test Results
uses: EnricoMi/publish-unit-test-result-action@v2
uses: EnricoMi/publish-unit-test-result-action@7622c5bf3d112a72ab4fa3951c2aa5723d8ab92d # v2.25.0
with:
comment_mode: always
files: "artifacts/**/**/*.trx"
2 changes: 1 addition & 1 deletion .github/workflows/notify-docs-site.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Dispatch extension-documentation-updated-event to Testably/Testably.Site
uses: peter-evans/repository-dispatch@v4
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
with:
token: ${{ secrets.SITE_DISPATCH_TOKEN }}
repository: Testably/Testably.Site
Expand Down
69 changes: 31 additions & 38 deletions Pipeline/Build.Benchmarks.cs
Original file line number Diff line number Diff line change
Expand Up @@ -63,16 +63,20 @@ partial class Build
string fileContent = await File.ReadAllTextAsync(file);
Log.Information("Report ({FileName}):\n {FileContent}", Path.GetFileName(file), fileContent);
}

if (GitHubActions?.IsPullRequest == true)
{
File.WriteAllText(ArtifactsDirectory / "PR.txt", GitHubActions.PullRequestNumber.ToString());
}
});

Target BenchmarkComment => _ => _
.Executes(async () =>
{
// The artifacts are untrusted, because the pull request controls the code that produces them, so the pull
// request to comment on comes from the event of the workflow run.
int? prId = await BuildExtensions.ResolvePullRequestOfWorkflowRun(GithubToken);
if (prId == null)
{
Log.Information("Skip writing a comment, as the workflow run belongs to no pull request.");
return;
}

await "Benchmarks-".DownloadArtifactsStartingWith(ArtifactsDirectory, GithubToken);
if (!Directory.Exists(ArtifactsDirectory / "Benchmarks" / "results"))
{
Expand All @@ -87,46 +91,35 @@ partial class Build
return;
}

if (!File.Exists(ArtifactsDirectory / "PR.txt"))
{
Log.Information("Skip writing a comment, as no PR number was specified.");
return;
}

AbsolutePath baselineDirectory = ArtifactsDirectory / "Baseline";
string baselineResultsDirectory = await DownloadBaselineBenchmarks(baselineDirectory);

string prNumber = File.ReadAllText(ArtifactsDirectory / "PR.txt");
string body = CreateBenchmarkCommentBody(files, baselineResultsDirectory);
Log.Debug("Pull request number: {PullRequestId}", prNumber);
if (int.TryParse(prNumber, out int prId))
string body = CreateBenchmarkCommentBody(files, baselineResultsDirectory).AsUntrustedCommentContent();
GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout"));
Credentials tokenAuth = new(GithubToken);
gitHubClient.Credentials = tokenAuth;
IReadOnlyList<IssueComment> comments =
await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value);
long? commentId = null;
Log.Information($"Found {comments.Count} comments");
foreach (IssueComment comment in comments)
{
GitHubClient gitHubClient = new(new ProductHeaderValue("Fallout"));
Credentials tokenAuth = new(GithubToken);
gitHubClient.Credentials = tokenAuth;
IReadOnlyList<IssueComment> comments =
await gitHubClient.Issue.Comment.GetAllForIssue(BuildExtensions.Owner, BuildExtensions.Repo, prId);
long? commentId = null;
Log.Information($"Found {comments.Count} comments");
foreach (IssueComment comment in comments)
if (comment.IsPipelineComment("## :rocket: Benchmark Results"))
{
if (comment.Body.Contains("## :rocket: Benchmark Results"))
{
Log.Information($"Found comment: {comment.Body}");
commentId = comment.Id;
}
Log.Information($"Found comment: {comment.Body}");
commentId = comment.Id;
}
}

if (commentId == null)
{
Log.Information($"Create comment:\n{body}");
await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId, body);
}
else
{
Log.Information($"Update comment:\n{body}");
await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body);
}
if (commentId == null)
{
Log.Information($"Create comment:\n{body}");
await gitHubClient.Issue.Comment.Create(BuildExtensions.Owner, BuildExtensions.Repo, prId.Value, body);
}
else
{
Log.Information($"Update comment:\n{body}");
await gitHubClient.Issue.Comment.Update(BuildExtensions.Owner, BuildExtensions.Repo, commentId.Value, body);
}
});

Expand Down
2 changes: 1 addition & 1 deletion Pipeline/Build.CodeAnalysis.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ partial class Build
.SetOrganization("testably")
.SetProjectKey("Testably_Mockolate")
.AddVSTestReports(TestResultsDirectory / "*.trx")
.AddOpenCoverPaths(TestResultsDirectory / "reports" / "OpenCover.xml")
.AddGenericCoveragePaths(TestResultsDirectory / "reports" / "SonarQube.xml")
.SetPullRequestOrBranchName(GitHubActions, GitVersion)
.SetVersion(GitVersion.SemVer)
.SetToken(SonarToken));
Expand Down
6 changes: 4 additions & 2 deletions Pipeline/Build.CodeCoverage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ partial class Build
{
ReportGenerator(s => s
.SetProcessToolPath(NuGetToolPathResolver.GetPackageExecutable("ReportGenerator", "ReportGenerator.dll",
framework: "net8.0"))
framework: "net10.0"))
.SetTargetDirectory(TestResultsDirectory / "reports")
.AddReports(TestResultsDirectory / "**/coverage.cobertura.xml")
.AddReportTypes(ReportTypes.OpenCover)
// The generic format of Sonar lists the lines per file. A format that lists them per method leaves out the
// lines of a lambda in a generic class, because ReportGenerator keeps them in a class without methods.
.AddReportTypes(ReportTypes.SonarQube)
.AddFileFilters("-*.g.cs")
.SetAssemblyFilters("+Mockolate*"));
});
Expand Down
8 changes: 8 additions & 0 deletions Pipeline/Build.Compile.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Text.RegularExpressions;
using Fallout.Common;
using Fallout.Common.IO;
using Fallout.Common.Tools.DotNet;
Expand Down Expand Up @@ -30,6 +31,13 @@ partial class Build
}
else if (GitHubActions.Ref.StartsWith("refs/tags/", StringComparison.OrdinalIgnoreCase))
{
if (!Regex.IsMatch(GitHubActions.Ref,
@"\Arefs/tags/v\d+\.\d+\.\d+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?\z"))
{
Assert.Fail(
$"The tag '{GitHubActions.Ref["refs/tags/".Length..]}' is no valid release tag (expected 'v1.2.3', optionally with a pre-release suffix like '-pre.1')");
}

int preReleaseIndex = GitHubActions.Ref.IndexOf('-');
preRelease = preReleaseIndex > 0 ? GitHubActions.Ref[preReleaseIndex..] : "";
}
Expand Down
Loading
Loading