Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## Unreleased

- Isolate incoming payload hydration failures to the failed delivery, preserving the shared work-item stream and unrelated in-flight work ([#825](https://github.com/microsoft/durabletask-dotnet/issues/825)).
- Avoid allocating outcome-observation continuations for already-successful gRPC client responses while preserving transport-failure tracking and channel recreation.
- Validate all eligible externalized payload sizes before uploading any field of an orchestration response or entity batch, including nested carryover and rewind history. Deterministic size failures now precede hypothetical upload failures and cause no uploads or partial token substitutions ([#820](https://github.com/microsoft/durabletask-dotnet/issues/820)).

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
using System.Text;
using Grpc.Core;
using Grpc.Core.Interceptors;
using Microsoft.DurableTask.Worker.Grpc.Internal;
using P = Microsoft.DurableTask.Protobuf;

namespace Microsoft.DurableTask;

Expand Down Expand Up @@ -280,7 +282,20 @@ public async Task<bool> MoveNext(CancellationToken cancellationToken)
return false;
}

this.Current = await this.transform(this.inner.Current, cancellationToken);
try
{
this.Current = await this.transform(this.inner.Current, cancellationToken);
}
catch (Exception ex) when (this.inner.Current is P.WorkItem workItem
&& workItem.RequestCase is P.WorkItem.RequestOneofCase.ActivityRequest
or P.WorkItem.RequestOneofCase.OrchestratorRequest
or P.WorkItem.RequestOneofCase.EntityRequest
or P.WorkItem.RequestOneofCase.EntityRequestV2
&& !cancellationToken.IsCancellationRequested)
{
throw new WorkItemPayloadException(workItem, ex);
}

return true;
}
}
Expand Down
75 changes: 70 additions & 5 deletions src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Diagnostics;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using DurableTask.Core;
using DurableTask.Core.Entities;
Expand Down Expand Up @@ -387,12 +388,10 @@ async Task ProcessWorkItemsAsync(
// window. If that fires we treat the stream as silently disconnected and reconnect.
TimeSpan silentDisconnectTimeout = this.internalOptions.SilentDisconnectTimeout;

// NOTE: the consumer deliberately does NOT wrap its await foreach in an outer loop.
// The underlying IAsyncStreamReader is single-use — once the server terminates the stream
// (e.g. via a graceful HTTP/2 GOAWAY with OK trailers during a rolling upgrade), MoveNext
// returns false forever and re-entering await foreach would tight-spin with no yield.
// Catch delivery failures inside the iterator; a faulted ReadAllAsync cannot be resumed.
// A false transport read still ends this stream permanently.
WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync(
ct => stream.ResponseStream.ReadAllAsync(ct),
(ct, onMessageReceived) => this.ReadWorkItemsAsync(stream.ResponseStream, onMessageReceived, ct),
silentDisconnectTimeout,
workItem => this.DispatchWorkItem(workItem, cancellation),
onFirstMessage,
Expand Down Expand Up @@ -430,6 +429,72 @@ async Task ProcessWorkItemsAsync(
}
}

async IAsyncEnumerable<P.WorkItem> ReadWorkItemsAsync(
IAsyncStreamReader<P.WorkItem> reader,
Action onMessageReceived,
[EnumeratorCancellation] CancellationToken cancellation)
{
while (true)
{
bool hasNext;
try
{
hasNext = await reader.MoveNext(cancellation).ConfigureAwait(false);
}
catch (WorkItemPayloadException ex)
{
onMessageReceived();
await this.AbandonUnresolvedWorkItemAsync(ex, cancellation).ConfigureAwait(false);
continue;
Comment thread
YunchuWang marked this conversation as resolved.
Resolved
}

if (!hasNext)
{
yield break;
}

yield return reader.Current;
}
}

async Task AbandonUnresolvedWorkItemAsync(WorkItemPayloadException exception, CancellationToken cancellation)
{
cancellation.ThrowIfCancellationRequested();
P.WorkItem workItem = exception.WorkItem;
string instanceId = workItem.OrchestratorRequest?.InstanceId
?? workItem.ActivityRequest?.OrchestrationInstance?.InstanceId
?? workItem.EntityRequest?.InstanceId
?? workItem.EntityRequestV2?.InstanceId
?? string.Empty;
this.Logger.UnexpectedError(exception, instanceId);
try
{
await this.ExecuteWithRetryAsync(
() => workItem.RequestCase switch
{
P.WorkItem.RequestOneofCase.OrchestratorRequest =>
this.client.AbandonTaskOrchestratorWorkItemAsync(
new P.AbandonOrchestrationTaskRequest { CompletionToken = workItem.CompletionToken },
cancellationToken: cancellation).ResponseAsync,
P.WorkItem.RequestOneofCase.ActivityRequest =>
this.client.AbandonTaskActivityWorkItemAsync(
new P.AbandonActivityTaskRequest { CompletionToken = workItem.CompletionToken },
cancellationToken: cancellation).ResponseAsync,
P.WorkItem.RequestOneofCase.EntityRequest or P.WorkItem.RequestOneofCase.EntityRequestV2 =>
this.client.AbandonTaskEntityWorkItemAsync(
new P.AbandonEntityTaskRequest { CompletionToken = workItem.CompletionToken },
cancellationToken: cancellation).ResponseAsync,
_ => throw new InvalidOperationException($"Cannot abandon unresolved {workItem.RequestCase}."),
},
"AbandonUnresolvedWorkItem",
cancellation);
}
catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound && !cancellation.IsCancellationRequested)
Comment thread
YunchuWang marked this conversation as resolved.
Resolved
{
this.Logger.WorkItemOwnershipLost(workItem.RequestCase.ToString(), workItem.CompletionToken, ex);
}
}

void DispatchWorkItem(P.WorkItem workItem, CancellationToken cancellation)
{
if (workItem.RequestCase == P.WorkItem.RequestOneofCase.OrchestratorRequest)
Expand Down
23 changes: 23 additions & 0 deletions src/Worker/Grpc/Internal/WorkItemPayloadException.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using P = Microsoft.DurableTask.Protobuf;

namespace Microsoft.DurableTask.Worker.Grpc.Internal;

/// <summary>
/// A received delivery whose payloads could not be resolved. The item is only safe to abandon, not dispatch.
/// </summary>
/// <param name="workItem">The failed delivery, including its original completion token.</param>
/// <param name="innerException">The payload resolution failure.</param>
/// <remarks>
/// Infrastructure contract between worker extensions and the gRPC worker. This is not a supported application API.
/// </remarks>
public sealed class WorkItemPayloadException(P.WorkItem workItem, Exception innerException)
: Exception("Failed to resolve payloads for a work-item delivery.", innerException)
{
/// <summary>
/// Gets the failed delivery. Its payload fields may be partially resolved.
/// </summary>
public P.WorkItem WorkItem { get; } = workItem;
}
3 changes: 3 additions & 0 deletions src/Worker/Grpc/Logs.cs
Original file line number Diff line number Diff line change
Expand Up @@ -107,5 +107,8 @@ static partial class Logs

[LoggerMessage(EventId = 78, Level = LogLevel.Warning, Message = "Activity notification callback failed for phase '{Phase}'.")]
public static partial void ActivityNotificationFailed(this ILogger logger, ActivityNotificationPhase phase, Exception exception);

[LoggerMessage(EventId = 79, Level = LogLevel.Warning, Message = "Ownership of '{workItemType}' work item was lost before abandonment. Completion token = '{completionToken}'")]
Comment thread
YunchuWang marked this conversation as resolved.
Resolved
public static partial void WorkItemOwnershipLost(this ILogger logger, string workItemType, string completionToken, Exception exception);
}
}
28 changes: 16 additions & 12 deletions src/Worker/Grpc/WorkItemStreamConsumer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ internal static class WorkItemStreamConsumer
/// Consume a work-item stream until shutdown, silent disconnect, or graceful drain.
/// </summary>
/// <param name="openStream">
/// Factory that opens the stream with the supplied linked-cancellation token. Production passes
/// <c>ct => stream.ResponseStream.ReadAllAsync(ct)</c>; tests pass arbitrary fakes.
/// Factory that opens the stream with linked cancellation and a receipt callback. The callback
/// records real deliveries handled without yielding them; yielded items are observed automatically.
/// </param>
/// <param name="silentDisconnectTimeout">
/// How long to wait between successive items (or health pings) before treating the stream as
Expand All @@ -70,7 +70,7 @@ internal static class WorkItemStreamConsumer
/// </param>
/// <returns>The classified outcome plus whether any message was observed.</returns>
public static async Task<WorkItemStreamResult> ConsumeAsync(
Func<CancellationToken, IAsyncEnumerable<P.WorkItem>> openStream,
Func<CancellationToken, Action, IAsyncEnumerable<P.WorkItem>> openStream,
TimeSpan silentDisconnectTimeout,
Action<P.WorkItem> onItem,
Action? onFirstMessage,
Expand Down Expand Up @@ -100,18 +100,22 @@ void ArmSilentDisconnectTimer()

bool firstMessageObserved = false;

try
void ObserveMessage()
{
await foreach (P.WorkItem workItem in openStream(tokenSource.Token).ConfigureAwait(false))
{
ArmSilentDisconnectTimer();
ArmSilentDisconnectTimer();

if (!firstMessageObserved)
{
firstMessageObserved = true;
onFirstMessage?.Invoke();
}
if (!firstMessageObserved)
{
firstMessageObserved = true;
onFirstMessage?.Invoke();
}
}

try
{
await foreach (P.WorkItem workItem in openStream(tokenSource.Token, ObserveMessage).ConfigureAwait(false))
{
ObserveMessage();
onItem(workItem);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,8 @@ public async Task WorkItem_ResolvesHistoryUsingConfiguredScopeAsync(
}
else
{
await moveNext.Should().ThrowAsync<PayloadStorageException>();
(await moveNext.Should().ThrowAsync<Microsoft.DurableTask.Worker.Grpc.Internal.WorkItemPayloadException>())
.Which.InnerException.Should().BeOfType<PayloadStorageException>();
workItem.OrchestratorRequest.NewEvents[0].ExecutionStarted.Input.Should().Be(token);
handler.Requests.Should().BeEmpty();
}
Expand Down
5 changes: 4 additions & 1 deletion test/Grpc.IntegrationTests/GrpcSidecarFixture.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,12 @@ public sealed class GrpcSidecarFixture : IDisposable
readonly IHost host;

public GrpcSidecarFixture()
: this(new InMemoryOrchestrationService())
{
InMemoryOrchestrationService service = new();
}

internal GrpcSidecarFixture(InMemoryOrchestrationService service)
{
// Use a random port number to allow multiple instances to run in parallel
string address = $"http://{ListenHost}:{Random.Shared.Next(30000, 40000)}";
this.host = Host.CreateDefaultBuilder()
Expand Down
108 changes: 108 additions & 0 deletions test/Grpc.IntegrationTests/LargePayloadTests.cs
Original file line number Diff line number Diff line change
@@ -1,23 +1,109 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.IO;
using System.Text.Json;
using Azure;
using FluentAssertions;
using Grpc.Core;
using Grpc.Core.Interceptors;
using Microsoft.DurableTask.Client;
using Microsoft.DurableTask;
using Microsoft.DurableTask.Converters;
using Microsoft.DurableTask.Testing.Sidecar;
using Microsoft.DurableTask.Worker;
using Microsoft.DurableTask.Worker.Grpc;
using Microsoft.Extensions.DependencyInjection;
using Moq;
using Xunit.Abstractions;
using IOrchestrationService = global::DurableTask.Core.IOrchestrationService;
using P = Microsoft.DurableTask.Protobuf;

namespace Microsoft.DurableTask.Grpc.Tests;

public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) : IntegrationTestBase(output, sidecarFixture)
{
[Theory]
[InlineData(0)]
[InlineData(403)]
[InlineData(404)]
public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int storageStatus)
{
// Arrange
using GrpcSidecarFixture isolatedSidecar = new(new TwoActivitySlotService());
string token = $"test-blob://missing-{Guid.NewGuid():N}";
TaskName orchestration = nameof(HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync);
TaskName activity = "HydrationActivity";
TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously);
TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously);
TaskCompletionSource downloadFailed = new(TaskCreationOptions.RunContinuationsAsynchronously);
Mock<PayloadStore> store = new();
store.Setup(s => s.IsKnownPayloadToken(It.IsAny<string>())).Returns((string value) => value == token);
store.Setup(s => s.UploadAsync(It.IsAny<string>(), It.IsAny<CancellationToken>())).ReturnsAsync(token);
store.Setup(s => s.DownloadAsync(token, It.IsAny<CancellationToken>())).Returns(() =>
{
downloadFailed.TrySetResult();
Exception failure = storageStatus == 0
? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed.");
return Task.FromException<string>(failure);
});
await using HostTestLifetime server = await this.StartWorkerAsync(worker =>
{
worker.UseGrpc(isolatedSidecar.Channel);
worker.Services.Configure<DurableTaskWorkerOptions>(worker.Name,
options => options.Concurrency.MaximumConcurrentActivityWorkItems = 2);
worker.AddTasks(tasks => tasks
.AddOrchestratorFunc<string, string>(orchestration, (context, input) =>
context.CallActivityAsync<string>(activity, input == "bad" ? new string('X', 1024) : input))
.AddActivityFunc<string, string>(activity, async (_, input) =>
{
if (input == "running")
{
running.TrySetResult();
await release.Task.WaitAsync(this.TimeoutToken);
}

return input;
}));
worker.UseExternalizedPayloads();
worker.Services.AddSingleton(store.Object);
worker.Services.PostConfigure<GrpcDurableTaskWorkerOptions>(
options => options.Interceptors.Insert(0, new ForwardingStreamInterceptor()));
}, client => client.UseGrpc(isolatedSidecar.Channel), services => services.AddExternalizedPayloadStore(options =>
{
options.ThresholdBytes = 16;
options.ConnectionString = "UseDevelopmentStorage=true";
}));

try
{
// Act
string alreadyRunning = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "running");
await running.Task.WaitAsync(this.TimeoutToken);
await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "bad");
await downloadFailed.Task.WaitAsync(this.TimeoutToken);
Comment thread
YunchuWang marked this conversation as resolved.
Resolved

// The test sidecar does not settle abandoned activities; free the healthy slot before starting another.
release.TrySetResult();
OrchestrationMetadata runningResult = await server.Client.WaitForInstanceCompletionAsync(
alreadyRunning, getInputsAndOutputs: true, this.TimeoutToken);
string following = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "following");
OrchestrationMetadata followingResult = await server.Client.WaitForInstanceCompletionAsync(
following, getInputsAndOutputs: true, this.TimeoutToken);

// Assert
Assert.Equal(OrchestrationRuntimeStatus.Completed, followingResult.RuntimeStatus);
Assert.Equal("following", followingResult.ReadOutputAs<string>());
Assert.Equal(OrchestrationRuntimeStatus.Completed, runningResult.RuntimeStatus);
Assert.Equal("running", runningResult.ReadOutputAs<string>());
Assert.Single(this.GetLogs(), log => log.EventId.Name == "EstablishedWorkItemConnection");
}
finally
{
release.TrySetResult();
}
}

[Theory]
[InlineData(true)]
[InlineData(false)]
Expand Down Expand Up @@ -908,6 +994,28 @@ public async Task LargeActivityOutput_ExceedsGrpcLimit_ExternalizesSuccessfully(
Assert.Equal(largeOutput, output);
}

sealed class TwoActivitySlotService : InMemoryOrchestrationService, IOrchestrationService
{
int IOrchestrationService.MaxConcurrentTaskActivityWorkItems => 2;
}

sealed class ForwardingStreamInterceptor : Interceptor
{
public override AsyncServerStreamingCall<TResponse> AsyncServerStreamingCall<TRequest, TResponse>(
TRequest request,
ClientInterceptorContext<TRequest, TResponse> context,
AsyncServerStreamingCallContinuation<TRequest, TResponse> continuation)
{
AsyncServerStreamingCall<TResponse> call = continuation(request, context);
Mock<IAsyncStreamReader<TResponse>> reader = new();
reader.Setup(r => r.MoveNext(It.IsAny<CancellationToken>()))
.Returns((CancellationToken cancellation) => call.ResponseStream.MoveNext(cancellation));
reader.SetupGet(r => r.Current).Returns(() => call.ResponseStream.Current);
return new AsyncServerStreamingCall<TResponse>(
reader.Object, call.ResponseHeadersAsync, call.GetStatus, call.GetTrailers, call.Dispose);
}
}

sealed class RetryCompletionInterceptor(bool retryActivity) : Interceptor
{
public List<string> Payloads { get; } = [];
Expand Down
Loading
Loading