From 7b811e3faf9489302e4a5e393392478226f6ac88 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 30 Oct 2026 12:15:00 -0700 Subject: [PATCH 1/9] Isolate inbound payload hydration failures by delivery Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + .../Interceptors/PayloadInterceptor.cs | 16 +- src/Grpc/Grpc.csproj | 6 + src/Grpc/WorkItemPayloadException.cs | 20 + .../Grpc/GrpcDurableTaskWorker.Processor.cs | 170 +++--- src/Worker/Grpc/Logs.cs | 3 + src/Worker/Grpc/WorkItemStreamConsumer.cs | 47 +- src/Worker/Grpc/Worker.Grpc.csproj | 1 + .../PayloadHydrationFailureTests.cs | 520 ++++++++++++++++++ .../Grpc.Tests/WorkItemStreamConsumerTests.cs | 168 ++++-- 10 files changed, 809 insertions(+), 143 deletions(-) create mode 100644 src/Grpc/WorkItemPayloadException.cs create mode 100644 test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f2ffe8ff..c5dcc9956 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Isolate incoming payload hydration failures to the failed delivery, preserving the shared work-item stream and unrelated in-flight work ([#825](https://github.com/microsoft/durabletask-dotnet/issues/825)). - Externalize continue-as-new carryover event payloads using the configured UTF-8 size limits, preventing oversized completion responses ([#819](https://github.com/microsoft/durabletask-dotnet/issues/819)). ## v1.27.1 diff --git a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs index 3274229cb..b2e88a841 100644 --- a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs +++ b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs @@ -4,6 +4,7 @@ using System.Text; using Grpc.Core; using Grpc.Core.Interceptors; +using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask; @@ -143,7 +144,20 @@ public override AsyncServerStreamingCall AsyncServerStreamingCall wrapped = new TransformingStreamReader(call.ResponseStream, async (msg, ct) => { - await this.ResolveResponsePayloadsAsync(msg, ct); + try + { + await this.ResolveResponsePayloadsAsync(msg, ct); + } + catch (Exception ex) when (msg is P.WorkItem workItem + && workItem.RequestCase is P.WorkItem.RequestOneofCase.ActivityRequest + or P.WorkItem.RequestOneofCase.OrchestratorRequest + or P.WorkItem.RequestOneofCase.EntityRequest + or P.WorkItem.RequestOneofCase.EntityRequestV2 + && !ct.IsCancellationRequested) + { + throw new WorkItemPayloadException(workItem, ex); + } + return msg; }); diff --git a/src/Grpc/Grpc.csproj b/src/Grpc/Grpc.csproj index e433e01ef..4798b8bac 100644 --- a/src/Grpc/Grpc.csproj +++ b/src/Grpc/Grpc.csproj @@ -21,4 +21,10 @@ + + + + + + diff --git a/src/Grpc/WorkItemPayloadException.cs b/src/Grpc/WorkItemPayloadException.cs new file mode 100644 index 000000000..6386098a6 --- /dev/null +++ b/src/Grpc/WorkItemPayloadException.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask; + +/// +/// A received delivery whose payloads could not be resolved. The item is only safe to abandon, not dispatch. +/// +/// The failed delivery, including its original completion token. +/// The payload resolution failure. +internal sealed class WorkItemPayloadException(P.WorkItem workItem, Exception innerException) + : Exception("Failed to resolve payloads for a work-item delivery.", innerException) +{ + /// + /// Gets the failed delivery. Its payload fields may be partially resolved. + /// + public P.WorkItem WorkItem { get; } = workItem; +} diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index abfba18a4..96e063e35 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -279,6 +279,13 @@ static OrchestrationRuntimeState BuildRuntimeState( return runtimeState; } + static string GetWorkItemInstanceId(P.WorkItem? workItem) => + workItem?.OrchestratorRequest?.InstanceId ?? + workItem?.ActivityRequest?.OrchestrationInstance?.InstanceId ?? + workItem?.EntityRequest?.InstanceId ?? + workItem?.EntityRequestV2?.InstanceId ?? + string.Empty; + async ValueTask> GetPastEventsAsync( P.OrchestratorRequest orchestratorRequest, CancellationToken cancellation) @@ -385,16 +392,15 @@ async Task ProcessWorkItemsAsync( // window. If that fires we treat the stream as silently disconnected and reconnect. TimeSpan silentDisconnectTimeout = this.internalOptions.SilentDisconnectTimeout; - // NOTE: the consumer deliberately does NOT wrap its await foreach in an outer loop. - // The underlying IAsyncStreamReader is single-use — once the server terminates the stream - // (e.g. via a graceful HTTP/2 GOAWAY with OK trailers during a rolling upgrade), MoveNext - // returns false forever and re-entering await foreach would tight-spin with no yield. + // Read directly so a delivery-local hydration failure does not terminate a ReadAllAsync + // iterator. A false MoveNext still ends consumption permanently (no drain tight-spin). WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - ct => stream.ResponseStream.ReadAllAsync(ct), + stream.ResponseStream, silentDisconnectTimeout, workItem => this.DispatchWorkItem(workItem, cancellation), onFirstMessage, - cancellation); + cancellation, + onPayloadFailure: this.OnWorkItemPayloadFailureAsync); switch (result.Outcome) { @@ -534,90 +540,82 @@ void RunBackgroundTask(P.WorkItem? workItem, Func handler, CancellationTok } catch (Exception ex) { - string instanceId = - workItem?.OrchestratorRequest?.InstanceId ?? - workItem?.ActivityRequest?.OrchestrationInstance?.InstanceId ?? - workItem?.EntityRequest?.InstanceId ?? - workItem?.EntityRequestV2?.InstanceId ?? - string.Empty; - this.Logger.UnexpectedError(ex, instanceId); - - if (workItem?.OrchestratorRequest != null) - { - this.Logger.AbandoningOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskOrchestratorWorkItemAsync( - new P.AbandonOrchestrationTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskOrchestratorWorkItemAsync), - cancellation); - this.Logger.AbandonedOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.ActivityRequest != null) - { - this.Logger.AbandoningActivityWorkItem( - instanceId, - workItem.ActivityRequest.Name, - workItem.ActivityRequest.TaskId, - workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskActivityWorkItemAsync( - new P.AbandonActivityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskActivityWorkItemAsync), - cancellation); - this.Logger.AbandonedActivityWorkItem( - instanceId, - workItem.ActivityRequest.Name, - workItem.ActivityRequest.TaskId, - workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.EntityRequest != null) - { - this.Logger.AbandoningEntityWorkItem( - workItem.EntityRequest.InstanceId, - workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskEntityWorkItemAsync( - new P.AbandonEntityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskEntityWorkItemAsync), - cancellation); - this.Logger.AbandonedEntityWorkItem( - workItem.EntityRequest.InstanceId, - workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.EntityRequestV2 != null) - { - this.Logger.AbandoningEntityWorkItem( - workItem.EntityRequestV2.InstanceId, - workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskEntityWorkItemAsync( - new P.AbandonEntityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskEntityWorkItemAsync), - cancellation); - this.Logger.AbandonedEntityWorkItem( - workItem.EntityRequestV2.InstanceId, - workItem.CompletionToken ?? string.Empty); - } + this.Logger.UnexpectedError(ex, GetWorkItemInstanceId(workItem)); + await this.AbandonWorkItemAsync(workItem, cancellation); } }); } + async Task OnWorkItemPayloadFailureAsync(WorkItemPayloadException exception, CancellationToken cancellation) + { + P.WorkItem workItem = exception.WorkItem; + this.Logger.UnexpectedError(exception, GetWorkItemInstanceId(workItem)); + try + { + await this.AbandonWorkItemAsync(workItem, cancellation); + } + catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) + { + // Ownership loss is delivery-local; other abandon failures must reach the reconnect loop. + this.Logger.WorkItemOwnershipLost(workItem.RequestCase.ToString(), workItem.CompletionToken, ex); + } + } + + async Task AbandonWorkItemAsync(P.WorkItem? workItem, CancellationToken cancellation) + { + string instanceId = GetWorkItemInstanceId(workItem); + if (workItem?.OrchestratorRequest != null) + { + this.Logger.AbandoningOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskOrchestratorWorkItemAsync( + new P.AbandonOrchestrationTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskOrchestratorWorkItemAsync), + cancellation); + this.Logger.AbandonedOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + } + else if (workItem?.ActivityRequest != null) + { + this.Logger.AbandoningActivityWorkItem( + instanceId, + workItem.ActivityRequest.Name, + workItem.ActivityRequest.TaskId, + workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskActivityWorkItemAsync( + new P.AbandonActivityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskActivityWorkItemAsync), + cancellation); + this.Logger.AbandonedActivityWorkItem( + instanceId, + workItem.ActivityRequest.Name, + workItem.ActivityRequest.TaskId, + workItem.CompletionToken ?? string.Empty); + } + else if (workItem?.EntityRequest != null || workItem?.EntityRequestV2 != null) + { + this.Logger.AbandoningEntityWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskEntityWorkItemAsync), + cancellation); + this.Logger.AbandonedEntityWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + } + } + async Task OnRunOrchestratorAsync( P.OrchestratorRequest request, string completionToken, diff --git a/src/Worker/Grpc/Logs.cs b/src/Worker/Grpc/Logs.cs index f378a7fce..ea1efa348 100644 --- a/src/Worker/Grpc/Logs.cs +++ b/src/Worker/Grpc/Logs.cs @@ -107,5 +107,8 @@ static partial class Logs [LoggerMessage(EventId = 78, Level = LogLevel.Warning, Message = "Activity notification callback failed for phase '{Phase}'.")] public static partial void ActivityNotificationFailed(this ILogger logger, ActivityNotificationPhase phase, Exception exception); + + [LoggerMessage(EventId = 79, Level = LogLevel.Warning, Message = "Ownership of '{workItemType}' work item was lost before abandonment. Completion token = '{completionToken}'")] + public static partial void WorkItemOwnershipLost(this ILogger logger, string workItemType, string completionToken, Exception exception); } } diff --git a/src/Worker/Grpc/WorkItemStreamConsumer.cs b/src/Worker/Grpc/WorkItemStreamConsumer.cs index 9c39ef7bc..d814f2a3b 100644 --- a/src/Worker/Grpc/WorkItemStreamConsumer.cs +++ b/src/Worker/Grpc/WorkItemStreamConsumer.cs @@ -43,10 +43,7 @@ internal static class WorkItemStreamConsumer /// /// Consume a work-item stream until shutdown, silent disconnect, or graceful drain. /// - /// - /// Factory that opens the stream with the supplied linked-cancellation token. Production passes - /// ct => stream.ResponseStream.ReadAllAsync(ct); tests pass arbitrary fakes. - /// + /// The response reader. Each read receives the linked cancellation token. /// /// How long to wait between successive items (or health pings) before treating the stream as /// silently disconnected. Non-positive values disable detection entirely. @@ -68,14 +65,19 @@ internal static class WorkItemStreamConsumer /// real elapsed time. Never invoked when disables /// detection. /// + /// + /// Optional delivery-local recovery for a payload resolution failure. Exceptions from recovery + /// propagate to the caller for connection-level handling. + /// /// The classified outcome plus whether any message was observed. public static async Task ConsumeAsync( - Func> openStream, + IAsyncStreamReader stream, TimeSpan silentDisconnectTimeout, Action onItem, Action? onFirstMessage, CancellationToken cancellation, - Action? onSilentDisconnectTimerArmed = null) + Action? onSilentDisconnectTimerArmed = null, + Func? onPayloadFailure = null) { bool silentDisconnectEnabled = silentDisconnectTimeout > TimeSpan.Zero; @@ -100,19 +102,38 @@ void ArmSilentDisconnectTimer() bool firstMessageObserved = false; - try + void ObserveMessage() { - await foreach (P.WorkItem workItem in openStream(tokenSource.Token).ConfigureAwait(false)) + ArmSilentDisconnectTimer(); + + if (!firstMessageObserved) { - ArmSilentDisconnectTimer(); + firstMessageObserved = true; + onFirstMessage?.Invoke(); + } + } - if (!firstMessageObserved) + try + { + while (true) + { + try { - firstMessageObserved = true; - onFirstMessage?.Invoke(); + if (!await stream.MoveNext(tokenSource.Token).ConfigureAwait(false)) + { + break; + } } + catch (WorkItemPayloadException ex) when (onPayloadFailure is not null) + { + ObserveMessage(); + await onPayloadFailure(ex, tokenSource.Token).ConfigureAwait(false); + continue; + } + + ObserveMessage(); - onItem(workItem); + onItem(stream.Current); } } catch (OperationCanceledException) when (cancellation.IsCancellationRequested) diff --git a/src/Worker/Grpc/Worker.Grpc.csproj b/src/Worker/Grpc/Worker.Grpc.csproj index 2b5d41138..0fa7b856b 100644 --- a/src/Worker/Grpc/Worker.Grpc.csproj +++ b/src/Worker/Grpc/Worker.Grpc.csproj @@ -22,6 +22,7 @@ + diff --git a/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs b/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs new file mode 100644 index 000000000..e3682722f --- /dev/null +++ b/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs @@ -0,0 +1,520 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Concurrent; +using System.IO; +using System.Net; +using System.Text.Json; +using System.Threading.Channels; +using Azure; +using Google.Protobuf.WellKnownTypes; +using Grpc.Core; +using Grpc.Core.Interceptors; +using Grpc.Net.Client; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.Grpc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Moq; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Grpc.Tests; + +public class PayloadHydrationFailureTests +{ + const string TokenPrefix = "test-blob://"; + const string BadToken = TokenPrefix + "bad"; + const string FailedCompletionToken = "failed-delivery"; + static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(15); + + public static IEnumerable HydrationFailures() + { + P.WorkItem.RequestOneofCase[] kinds = + [ + P.WorkItem.RequestOneofCase.ActivityRequest, + P.WorkItem.RequestOneofCase.OrchestratorRequest, + P.WorkItem.RequestOneofCase.EntityRequest, + P.WorkItem.RequestOneofCase.EntityRequestV2, + ]; + + foreach (P.WorkItem.RequestOneofCase kind in kinds) + { + foreach (int storageStatus in new[] { 404, 403, 0 }) + { + foreach (bool ownershipLost in new[] { false, true }) + { + yield return [kind, storageStatus, ownershipLost]; + } + } + } + } + + [Theory] + [MemberData(nameof(HydrationFailures))] + public async Task HydrationFailure_AbandonsOnlyFailedDeliveryAndKeepsStreamAsync( + P.WorkItem.RequestOneofCase kind, int storageStatus, bool ownershipLost) + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + sidecar.AbandonStatus = ownershipLost ? StatusCode.NotFound : StatusCode.OK; + Exception failure = storageStatus == 0 + ? new IOException("Transient payload download failure.") + : new RequestFailedException(storageStatus, "Payload download failed."); + Mock store = CreatePayloadStore(failure); + TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + ConcurrentQueue inputs = new(); + await using ServiceProvider provider = CreateWorker(sidecar, store.Object, async input => + { + inputs.Enqueue(input); + if (input == "running") + { + running.TrySetResult(); + await release.Task.WaitAsync(timeout.Token); + } + + return input; + }); + IHostedService worker = Assert.Single(provider.GetServices()); + await worker.StartAsync(timeout.Token); + + try + { + // Act + sidecar.Send(CreateHealthyActivity("running")); + await running.Task.WaitAsync(timeout.Token); + sidecar.Send(CreateFailedWorkItem(kind)); + sidecar.Send(new P.WorkItem { HealthPing = new P.HealthPing() }); + sidecar.Send(CreateHealthyActivity("following")); + P.ActivityResponse following = await sidecar.Completions.Reader.ReadAsync(timeout.Token); + release.TrySetResult(); + P.ActivityResponse alreadyRunning = await sidecar.Completions.Reader.ReadAsync(timeout.Token); + + // Assert + Assert.Equal(1, sidecar.StreamCount); + Assert.Equal(0, sidecar.ClosedStreamCount); + Assert.Equal(1, sidecar.HelloCount); + P.WorkItem.RequestOneofCase abandonKind = kind == P.WorkItem.RequestOneofCase.EntityRequestV2 + ? P.WorkItem.RequestOneofCase.EntityRequest + : kind; + Assert.Equal((abandonKind, FailedCompletionToken), Assert.Single(sidecar.Abandons)); + Assert.Equal(new[] { "running", "following" }, inputs.ToArray()); + Assert.Equal("following", following.CompletionToken); + Assert.Equal("\"following\"", following.Result); + Assert.Null(following.FailureDetails); + Assert.Equal("running", alreadyRunning.CompletionToken); + Assert.Equal("\"running\"", alreadyRunning.Result); + Assert.Null(alreadyRunning.FailureDetails); + store.Verify(s => s.DownloadAsync(BadToken, It.IsAny()), Times.Once); + } + finally + { + release.TrySetResult(); + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + } + } + + [Theory] + [InlineData(StatusCode.Unknown, false)] + [InlineData(StatusCode.Unavailable, false)] + [InlineData(StatusCode.PermissionDenied, false)] + [InlineData(StatusCode.Internal, false)] + [InlineData(StatusCode.Unavailable, true)] + public async Task TransportOrAbandonFailure_ReconnectsAsync(StatusCode status, bool streamFailure) + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + sidecar.AbandonStatus = status; + Mock store = CreatePayloadStore(new IOException("Payload download failed.")); + await using ServiceProvider provider = CreateWorker(sidecar, store.Object); + IHostedService worker = Assert.Single(provider.GetServices()); + await worker.StartAsync(timeout.Token); + + try + { + Assert.Equal(1, await sidecar.Streams.Reader.ReadAsync(timeout.Token)); + + // Act + if (streamFailure) + { + sidecar.FailStream(status); + } + else + { + sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); + } + + Assert.Equal(2, await sidecar.Streams.Reader.ReadAsync(timeout.Token)); + sidecar.Send(CreateHealthyActivity("following")); + P.ActivityResponse response = await sidecar.Completions.Reader.ReadAsync(timeout.Token); + + // Assert + Assert.Equal(2, sidecar.StreamCount); + Assert.Equal(2, sidecar.HelloCount); + Assert.Equal("following", response.CompletionToken); + Assert.Null(response.FailureDetails); + Assert.Equal(streamFailure ? 0 : 1, sidecar.Abandons.Count); + } + finally + { + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + } + } + + [Fact] + public async Task HydrationFailure_RedeliveryCompletesAfterPayloadRecoversAsync() + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + Mock store = CreatePayloadStore(new IOException("Payload download failed.")); + store.SetupSequence(s => s.DownloadAsync(BadToken, It.IsAny())) + .ThrowsAsync(new RequestFailedException(404, "Payload not found.")) + .ReturnsAsync("\"recovered\""); + await using ServiceProvider provider = CreateWorker(sidecar, store.Object); + IHostedService worker = Assert.Single(provider.GetServices()); + await worker.StartAsync(timeout.Token); + + try + { + // Act + sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); + await sidecar.AbandonStarted.Task.WaitAsync(timeout.Token); + P.WorkItem redelivery = CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest); + redelivery.CompletionToken = "redelivery"; + sidecar.Send(redelivery); + P.ActivityResponse response = await sidecar.Completions.Reader.ReadAsync(timeout.Token); + + // Assert + Assert.Equal(1, sidecar.StreamCount); + Assert.Equal((P.WorkItem.RequestOneofCase.ActivityRequest, FailedCompletionToken), Assert.Single(sidecar.Abandons)); + Assert.Equal("redelivery", response.CompletionToken); + Assert.Equal("\"recovered\"", response.Result); + Assert.Null(response.FailureDetails); + } + finally + { + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ShutdownDuringHydrationOrAbandon_DoesNotReconnectAsync(bool duringAbandon) + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + sidecar.HoldAbandon = duringAbandon; + TaskCompletionSource downloading = new(TaskCreationOptions.RunContinuationsAsynchronously); + Mock store = CreatePayloadStore(new IOException("Payload download failed.")); + if (!duringAbandon) + { + store.Setup(s => s.DownloadAsync(BadToken, It.IsAny())) + .Returns(async (string _, CancellationToken cancellation) => + { + downloading.TrySetResult(cancellation); + await Task.Delay(Timeout.Infinite, cancellation); + return string.Empty; + }); + } + + await using ServiceProvider provider = CreateWorker(sidecar, store.Object); + IHostedService worker = Assert.Single(provider.GetServices()); + await worker.StartAsync(timeout.Token); + + try + { + sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); + CancellationToken downloadCancellation = default; + if (duringAbandon) + { + await sidecar.AbandonStarted.Task.WaitAsync(timeout.Token); + } + else + { + downloadCancellation = await downloading.Task.WaitAsync(timeout.Token); + } + + // Act + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + + // Assert + Assert.Equal(1, sidecar.StreamCount); + Assert.Equal(duringAbandon ? 1 : 0, sidecar.Abandons.Count); + Assert.False(sidecar.Completions.Reader.TryRead(out _)); + if (!duringAbandon) + { + Assert.True(downloadCancellation.IsCancellationRequested); + } + } + finally + { + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NonWorkItemHydrationFailure_PropagatesOriginalExceptionAsync(bool historyStream) + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + RequestFailedException failure = new(403, "Payload download forbidden."); + Mock store = CreatePayloadStore(failure); + using GrpcChannel channel = GrpcChannel.ForAddress(sidecar.Address); + P.TaskHubSidecarService.TaskHubSidecarServiceClient client = new(channel.Intercept( + new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions()))); + + // Act + Exception? observed; + if (historyStream) + { + using AsyncServerStreamingCall call = client.StreamInstanceHistory( + new P.StreamInstanceHistoryRequest { InstanceId = "instance" }, cancellationToken: timeout.Token); + observed = await Record.ExceptionAsync(() => call.ResponseStream.MoveNext(timeout.Token)); + } + else + { + using AsyncUnaryCall call = client.GetInstanceAsync( + new P.GetInstanceRequest { InstanceId = "instance" }, cancellationToken: timeout.Token); + observed = await Record.ExceptionAsync(async () => await call.ResponseAsync); + } + + // Assert + Assert.Same(failure, observed); + Assert.Empty(sidecar.Abandons); + } + + static Mock CreatePayloadStore(Exception failure) + { + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())) + .Returns((string value) => value.StartsWith(TokenPrefix, StringComparison.Ordinal)); + store.Setup(s => s.DownloadAsync(It.IsAny(), It.IsAny())) + .Returns((string token, CancellationToken _) => token == BadToken + ? Task.FromException(failure) + : Task.FromResult(JsonSerializer.Serialize(token[TokenPrefix.Length..]))); + return store; + } + + static ServiceProvider CreateWorker(ScriptedSidecar sidecar, PayloadStore store, Func>? runActivity = null) + { + ServiceCollection services = new(); + services.AddLogging(logging => logging.ClearProviders()); + services.AddSingleton(store); + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(options => + { + options.Address = sidecar.Address; + options.Internal.TransientRetryMaxAttempts = 1; + options.Internal.ReconnectBackoffBase = TimeSpan.Zero; + options.Internal.ReconnectBackoffCap = TimeSpan.Zero; + }); + builder.UseExternalizedPayloads(); + builder.AddTasks(tasks => tasks.AddActivityFunc( + "Healthy", (_, input) => runActivity?.Invoke(input!) ?? Task.FromResult(input!))); + }); + return services.BuildServiceProvider(); + } + + static P.WorkItem CreateHealthyActivity(string name) => new() + { + CompletionToken = name, + ActivityRequest = new P.ActivityRequest + { + Name = "Healthy", + TaskId = 1, + Input = TokenPrefix + name, + OrchestrationInstance = new P.OrchestrationInstance { InstanceId = name, ExecutionId = "execution" }, + }, + }; + + static P.WorkItem CreateFailedWorkItem(P.WorkItem.RequestOneofCase kind) + { + P.WorkItem item = new() { CompletionToken = FailedCompletionToken }; + switch (kind) + { + case P.WorkItem.RequestOneofCase.ActivityRequest: + item.ActivityRequest = CreateHealthyActivity("failed").ActivityRequest; + item.ActivityRequest.Input = BadToken; + break; + case P.WorkItem.RequestOneofCase.OrchestratorRequest: + item.OrchestratorRequest = new P.OrchestratorRequest + { + InstanceId = "failed", + PastEvents = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = TokenPrefix + "state" } } }, + NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Name = "event", Input = BadToken } } }, + }; + break; + case P.WorkItem.RequestOneofCase.EntityRequest: + item.EntityRequest = new P.EntityBatchRequest + { + InstanceId = "@entity@failed", + EntityState = TokenPrefix + "state", + Operations = { new P.OperationRequest { Operation = "operation", Input = BadToken } }, + }; + break; + case P.WorkItem.RequestOneofCase.EntityRequestV2: + item.EntityRequestV2 = new P.EntityRequest + { + InstanceId = "@entity@failed", + EntityState = TokenPrefix + "state", + OperationRequests = { new P.HistoryEvent { EntityOperationSignaled = new P.EntityOperationSignaledEvent { Input = BadToken } } }, + }; + break; + default: + throw new ArgumentOutOfRangeException(nameof(kind), kind, "Unsupported work item kind."); + } + + return item; + } + + sealed class ScriptedSidecar : P.TaskHubSidecarService.TaskHubSidecarServiceBase, IAsyncDisposable + { + readonly Channel, Task>> deliveries = + Channel.CreateUnbounded, Task>>(); + IHost host = null!; + int streamCount; + int closedStreamCount; + int helloCount; + + public string Address { get; private set; } = string.Empty; + + public StatusCode AbandonStatus { get; set; } + + public bool HoldAbandon { get; set; } + + public int StreamCount => Volatile.Read(ref this.streamCount); + + public int ClosedStreamCount => Volatile.Read(ref this.closedStreamCount); + + public int HelloCount => Volatile.Read(ref this.helloCount); + + public Channel Streams { get; } = Channel.CreateUnbounded(); + + public Channel Completions { get; } = Channel.CreateUnbounded(); + + public ConcurrentQueue<(P.WorkItem.RequestOneofCase Kind, string Token)> Abandons { get; } = new(); + + public TaskCompletionSource AbandonStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public static async Task StartAsync() + { + ScriptedSidecar sidecar = new(); + sidecar.host = Host.CreateDefaultBuilder() + .ConfigureLogging(logging => logging.ClearProviders()) + .ConfigureWebHostDefaults(web => web + .UseKestrel(options => options.Listen(IPAddress.Loopback, 0, endpoint => endpoint.Protocols = HttpProtocols.Http2)) + .ConfigureServices(services => + { + services.AddGrpc(); + services.AddSingleton(sidecar); + }) + .Configure(app => + { + app.UseRouting(); + app.UseEndpoints(endpoints => endpoints.MapGrpcService()); + })) + .Build(); + await sidecar.host.StartAsync().WaitAsync(TestTimeout); + sidecar.Address = Assert.Single(sidecar.host.Services.GetRequiredService() + .Features.Get()!.Addresses); + return sidecar; + } + + public void Send(P.WorkItem item) => Assert.True(this.deliveries.Writer.TryWrite(stream => stream.WriteAsync(item))); + + public void FailStream(StatusCode status) => Assert.True(this.deliveries.Writer.TryWrite( + _ => Task.FromException(new RpcException(new Status(status, "Stream transport failed."))))); + + public override Task Hello(Empty request, ServerCallContext context) + { + Interlocked.Increment(ref this.helloCount); + return Task.FromResult(new Empty()); + } + + public override async Task GetWorkItems(P.GetWorkItemsRequest request, IServerStreamWriter responseStream, ServerCallContext context) + { + await context.WriteResponseHeadersAsync(new Metadata()); + this.Streams.Writer.TryWrite(Interlocked.Increment(ref this.streamCount)); + try + { + await foreach (Func, Task> deliver in this.deliveries.Reader.ReadAllAsync(context.CancellationToken)) + { + await deliver(responseStream); + } + } + finally + { + Interlocked.Increment(ref this.closedStreamCount); + } + } + + public override Task CompleteActivityTask(P.ActivityResponse request, ServerCallContext context) + { + this.Completions.Writer.TryWrite(request); + return Task.FromResult(new P.CompleteTaskResponse()); + } + + public override async Task AbandonTaskActivityWorkItem(P.AbandonActivityTaskRequest request, ServerCallContext context) + { + await this.AbandonAsync(P.WorkItem.RequestOneofCase.ActivityRequest, request.CompletionToken, context); + return new P.AbandonActivityTaskResponse(); + } + + public override async Task AbandonTaskOrchestratorWorkItem(P.AbandonOrchestrationTaskRequest request, ServerCallContext context) + { + await this.AbandonAsync(P.WorkItem.RequestOneofCase.OrchestratorRequest, request.CompletionToken, context); + return new P.AbandonOrchestrationTaskResponse(); + } + + public override async Task AbandonTaskEntityWorkItem(P.AbandonEntityTaskRequest request, ServerCallContext context) + { + await this.AbandonAsync(P.WorkItem.RequestOneofCase.EntityRequest, request.CompletionToken, context); + return new P.AbandonEntityTaskResponse(); + } + + public override Task GetInstance(P.GetInstanceRequest request, ServerCallContext context) + => Task.FromResult(new P.GetInstanceResponse { OrchestrationState = new P.OrchestrationState { Input = BadToken } }); + + public override Task StreamInstanceHistory(P.StreamInstanceHistoryRequest request, IServerStreamWriter responseStream, ServerCallContext context) + => responseStream.WriteAsync(new P.HistoryChunk + { + Events = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = BadToken } } }, + }); + + public async ValueTask DisposeAsync() + { + using IHost host = this.host; + await host.StopAsync().WaitAsync(TestTimeout); + } + + async Task AbandonAsync(P.WorkItem.RequestOneofCase kind, string token, ServerCallContext context) + { + this.Abandons.Enqueue((kind, token)); + this.AbandonStarted.TrySetResult(); + if (this.HoldAbandon) + { + await Task.Delay(Timeout.Infinite, context.CancellationToken); + } + + if (this.AbandonStatus != StatusCode.OK) + { + throw new RpcException(new Status(this.AbandonStatus, "Abandon failed.")); + } + } + } +} diff --git a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs index ba9104c11..330af9131 100644 --- a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs +++ b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Runtime.CompilerServices; using Grpc.Core; using Microsoft.DurableTask.Worker.Grpc; using P = Microsoft.DurableTask.Protobuf; @@ -16,7 +15,7 @@ public class WorkItemStreamConsumerTests public async Task EmptyStream_ReturnsGracefulDrain() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => EmptyStream(), + stream: EmptyStream(), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: _ => throw new InvalidOperationException("onItem should not be invoked"), onFirstMessage: () => throw new InvalidOperationException("onFirstMessage should not be invoked"), @@ -35,7 +34,7 @@ public async Task StreamWithItems_ReturnsGracefulDrain_AndFiresCallbacks() int firstMessageCount = 0; WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => StreamOf(item1, item2), + stream: StreamOf(item1, item2), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: received.Add, onFirstMessage: () => firstMessageCount++, @@ -51,7 +50,7 @@ public async Task StreamWithItems_ReturnsGracefulDrain_AndFiresCallbacks() public async Task VeryLargeSilentDisconnectTimeout_IsClamped_AndStreamCanStillComplete() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => EmptyStream(), + stream: EmptyStream(), silentDisconnectTimeout: TimeSpan.FromDays(365), onItem: _ => throw new InvalidOperationException("onItem should not be invoked"), onFirstMessage: null, @@ -65,7 +64,7 @@ public async Task VeryLargeSilentDisconnectTimeout_IsClamped_AndStreamCanStillCo public async Task HangingStream_SurfacingOce_ReturnsSilentDisconnect() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + stream: new HangingStreamReader(throwAsRpc: false), silentDisconnectTimeout: ShortTimeout, onItem: _ => { }, onFirstMessage: null, @@ -85,7 +84,7 @@ public async Task HangingStream_SurfacingOce_ReturnsSilentDisconnect() public async Task HangingStream_SurfacingRpcCancelled_ReturnsSilentDisconnect() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: true), + stream: new HangingStreamReader(throwAsRpc: true), silentDisconnectTimeout: ShortTimeout, onItem: _ => { }, onFirstMessage: null, @@ -104,7 +103,7 @@ public async Task OuterCancellation_WithOceFromStream_ReturnsShutdown() outer.CancelAfter(ShortTimeout); WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + stream: new HangingStreamReader(throwAsRpc: false), silentDisconnectTimeout: TimeSpan.FromSeconds(30), onItem: _ => { }, onFirstMessage: null, @@ -126,7 +125,7 @@ public async Task OuterCancellation_WithRpcCancelledFromStream_PropagatesExcepti outer.CancelAfter(ShortTimeout); Func act = () => WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: true), + stream: new HangingStreamReader(throwAsRpc: true), silentDisconnectTimeout: TimeSpan.FromSeconds(30), onItem: _ => { }, onFirstMessage: null, @@ -165,7 +164,7 @@ public async Task PerItem_HeartbeatReset_KeepsTimerAlive() } WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => StreamOf(items), + stream: StreamOf(items), silentDisconnectTimeout: TimeSpan.FromMilliseconds(500), onItem: _ => events.Add($"item{itemIndex++}"), onFirstMessage: null, @@ -193,7 +192,7 @@ public async Task PerItem_HeartbeatReset_KeepsTimerAlive() public async Task UnrelatedRpcException_Propagates() { Func act = () => WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => ThrowingStream(new RpcException(new Status(StatusCode.Unavailable, "boom"))), + stream: ThrowingStream(new RpcException(new Status(StatusCode.Unavailable, "boom"))), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: _ => { }, onFirstMessage: null, @@ -213,7 +212,7 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti // Act WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + stream: new HangingStreamReader(throwAsRpc: false), silentDisconnectTimeout: TimeSpan.FromMilliseconds(timeoutMilliseconds), onItem: _ => { }, onFirstMessage: null, @@ -223,55 +222,138 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti result.Outcome.Should().Be(WorkItemStreamOutcome.Shutdown); } -#pragma warning disable CS1998 // Async method lacks 'await' operators - static async IAsyncEnumerable EmptyStream() + [Fact] + public async Task HydrationFailure_RearmsTimerAndRecoversBeforeReadingNextItemAsync() { - yield break; + // Arrange + P.WorkItem failed = new() + { + CompletionToken = "failed-token", + OrchestratorRequest = new P.OrchestratorRequest + { + NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Input = "partially resolved" } } }, + }, + }; + WorkItemPayloadException failure = new(failed, new InvalidOperationException("Download failed.")); + P.WorkItem healthy = new() { HealthPing = new P.HealthPing() }; + Mock> reader = new(MockBehavior.Strict); + reader.SetupSequence(r => r.MoveNext(It.IsAny())) + .ThrowsAsync(failure) + .ReturnsAsync(true) + .ReturnsAsync(false); + reader.SetupGet(r => r.Current).Returns(healthy); + List events = new(); + List dispatched = new(); + + // Act + WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( + stream: reader.Object, + silentDisconnectTimeout: TimeSpan.FromSeconds(5), + onItem: item => + { + events.Add("dispatch"); + dispatched.Add(item); + }, + onFirstMessage: () => events.Add("first"), + cancellation: CancellationToken.None, + onSilentDisconnectTimerArmed: () => events.Add("armed"), + onPayloadFailure: (exception, cancellation) => + { + Assert.Same(failure, exception); + Assert.Same(failed, exception.WorkItem); + Assert.True(cancellation.CanBeCanceled); + events.Add("abandon"); + return Task.CompletedTask; + }); + + // Assert + Assert.Equal(WorkItemStreamOutcome.GracefulDrain, result.Outcome); + Assert.True(result.FirstMessageObserved); + Assert.Equal(new[] { "armed", "armed", "first", "abandon", "armed", "dispatch" }, events); + Assert.Same(healthy, Assert.Single(dispatched)); + reader.VerifyGet(r => r.Current, Times.Once); + reader.Verify(r => r.MoveNext(It.IsAny()), Times.Exactly(3)); } - static async IAsyncEnumerable StreamOf(params P.WorkItem[] items) + [Theory] + [InlineData(StatusCode.Unavailable)] + [InlineData(StatusCode.PermissionDenied)] + public async Task HydrationRecoveryFailure_PropagatesForConnectionRecoveryAsync(StatusCode status) { - foreach (P.WorkItem item in items) - { - yield return item; - } + // Arrange + WorkItemPayloadException failure = new(new P.WorkItem(), new InvalidOperationException("Download failed.")); + RpcException abandonFailure = new(new Status(status, "Abandon failed.")); + + // Act + Func act = () => WorkItemStreamConsumer.ConsumeAsync( + stream: ThrowingStream(failure), + silentDisconnectTimeout: TimeSpan.FromSeconds(5), + onItem: _ => throw new InvalidOperationException("Failed delivery must not be dispatched."), + onFirstMessage: null, + cancellation: CancellationToken.None, + onPayloadFailure: (_, _) => Task.FromException(abandonFailure)); + + // Assert + (await act.Should().ThrowAsync()).Which.Should().BeSameAs(abandonFailure); } - static IAsyncEnumerable ThrowingStream(Exception ex) => new ThrowingAsyncEnumerable(ex); -#pragma warning restore CS1998 + [Fact] + public async Task HydrationFailure_WithoutRecoveryHandler_PropagatesAsync() + { + // Arrange + WorkItemPayloadException failure = new(new P.WorkItem(), new InvalidOperationException("Download failed.")); + + // Act + Func act = () => WorkItemStreamConsumer.ConsumeAsync( + stream: ThrowingStream(failure), + silentDisconnectTimeout: TimeSpan.FromSeconds(5), + onItem: _ => throw new InvalidOperationException("Failed delivery must not be dispatched."), + onFirstMessage: null, + cancellation: CancellationToken.None); + + // Assert + (await act.Should().ThrowAsync()).Which.Should().BeSameAs(failure); + } + + static IAsyncStreamReader EmptyStream() => StreamOf(); - static async IAsyncEnumerable HangingStream( - [EnumeratorCancellation] CancellationToken ct, - bool throwAsRpc) + static IAsyncStreamReader StreamOf(params P.WorkItem[] items) => new SequenceStreamReader(items); + + static IAsyncStreamReader ThrowingStream(Exception ex) => new ThrowingStreamReader(ex); + + sealed class SequenceStreamReader(P.WorkItem[] items) : IAsyncStreamReader { - try - { - await Task.Delay(Timeout.Infinite, ct); - } - catch (OperationCanceledException) when (throwAsRpc) - { - // Mimic grpc-dotnet's default surface shape for linked-token cancellation. - throw new RpcException(new Status(StatusCode.Cancelled, "stream cancelled")); - } + int index = -1; - yield break; + public P.WorkItem Current => items[this.index]; + + public Task MoveNext(CancellationToken cancellationToken) => Task.FromResult(++this.index < items.Length); } - sealed class ThrowingAsyncEnumerable : IAsyncEnumerable, IAsyncEnumerator + sealed class HangingStreamReader(bool throwAsRpc) : IAsyncStreamReader { - readonly Exception exception; + public P.WorkItem Current => throw new InvalidOperationException("No item is available for a hanging stream."); - public ThrowingAsyncEnumerable(Exception exception) + public async Task MoveNext(CancellationToken cancellationToken) { - this.exception = exception; + try + { + await Task.Delay(Timeout.Infinite, cancellationToken); + } + catch (OperationCanceledException) when (throwAsRpc) + { + // Mimic grpc-dotnet's default surface shape for linked-token cancellation. + throw new RpcException(new Status(StatusCode.Cancelled, "stream cancelled")); + } + + return false; } + } + sealed class ThrowingStreamReader(Exception exception) : IAsyncStreamReader + { public P.WorkItem Current => throw new InvalidOperationException("No current item is available for a throwing stream."); - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) => this; - - public ValueTask DisposeAsync() => default; - - public ValueTask MoveNextAsync() => ValueTask.FromException(this.exception); + public Task MoveNext(CancellationToken cancellationToken) => Task.FromException(exception); } } From a0f5320589170f8298411a31507ae38f8b12d331 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 10 Nov 2026 15:33:01 -0800 Subject: [PATCH 2/9] Fail confirmed-missing payload deliveries without disconnecting healthy work Replace the standalone test server with focused coverage in existing harnesses. Preserve ordinary background error handling, validate execution metadata before terminal settlement, and release entity locks for inline and streamed history. Recoverable download failures remain delivery-local and retryable. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- .../Interceptors/PayloadInterceptor.cs | 35 +- src/Extensions/AzureBlobPayloads/README.md | 14 - src/Grpc/Grpc.csproj | 1 - src/Grpc/WorkItemPayloadException.cs | 27 +- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 320 +++++++++-- src/Worker/Grpc/Worker.Grpc.csproj | 1 - .../OrchestratorPayloadFailureTests.cs | 64 ++- .../LargePayloadTests.cs | 96 ++++ .../PayloadHydrationFailureTests.cs | 536 ------------------ .../Grpc.Tests/GrpcDurableTaskWorkerTests.cs | 163 ++++++ .../Grpc.Tests/WorkItemStreamConsumerTests.cs | 53 -- 12 files changed, 637 insertions(+), 675 deletions(-) delete mode 100644 test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 050e32e52..9bca6c72c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Isolate incoming payload hydration failures to the failed delivery, preserving the shared work-item stream and unrelated in-flight work ([#825](https://github.com/microsoft/durabletask-dotnet/issues/825)). +- Isolate incoming payload download failures to their deliveries. Fail activity/orchestration deliveries with confirmed missing blobs while preserving entity releases and the shared stream; retry recoverable failures without disrupting unrelated work ([#825](https://github.com/microsoft/durabletask-dotnet/issues/825)). - Validate all eligible externalized payload sizes before uploading any field of an orchestration response or entity batch, including nested carryover and rewind history. Deterministic size failures now precede hypothetical upload failures and cause no uploads or partial token substitutions ([#820](https://github.com/microsoft/durabletask-dotnet/issues/820)). ## v1.28.0 diff --git a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs index 9207456a3..11e12a3ea 100644 --- a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs +++ b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System.Text; +using Azure; using Grpc.Core; using Grpc.Core.Interceptors; using P = Microsoft.DurableTask.Protobuf; @@ -148,14 +149,30 @@ public override AsyncServerStreamingCall AsyncServerStreamingCall( IAsyncStreamReader inner, - Func> transform) : IAsyncStreamReader + Func> transform) : IAsyncStreamReader, IPayloadStreamReader { readonly IAsyncStreamReader inner = inner; readonly Func> transform = transform; public T Current { get; private set; } = default!; + public IAsyncStreamReader Inner => this.inner; + public async Task MoveNext(CancellationToken cancellationToken) { bool hasNext = await this.inner.MoveNext(cancellationToken); diff --git a/src/Extensions/AzureBlobPayloads/README.md b/src/Extensions/AzureBlobPayloads/README.md index e6fd9f6d1..146700caf 100644 --- a/src/Extensions/AzureBlobPayloads/README.md +++ b/src/Extensions/AzureBlobPayloads/README.md @@ -170,20 +170,6 @@ orphan blobs: real I/O failures, cancellation during uploads, and uncertain backend submission can still leave unrecorded uploads. No catch-and-delete rollback is performed. -## Incoming work-item hydration failures - -If downloading an incoming work item's payload fails, the worker abandons only -that delivery's completion token and continues reading the same work-item stream. -This includes missing blobs, authorization failures, and transient I/O errors. -The failed delivery is not dispatched, even if some fields were already hydrated, -and remains retryable; this does not introduce terminal poison-payload settlement. - -An abandonment response of `NotFound` means ownership was already lost and does -not disconnect the stream. Other abandonment failures retain connection-level -recovery after the worker's existing retry policy. Worker shutdown cancellation -is unchanged. Unary responses and streamed history still propagate hydration -errors to their existing callers. - ## SDK and dashboard boundaries The client and worker gRPC interceptors resolve orchestration inputs, external diff --git a/src/Grpc/Grpc.csproj b/src/Grpc/Grpc.csproj index 4798b8bac..f2e1ef3e2 100644 --- a/src/Grpc/Grpc.csproj +++ b/src/Grpc/Grpc.csproj @@ -24,7 +24,6 @@ - diff --git a/src/Grpc/WorkItemPayloadException.cs b/src/Grpc/WorkItemPayloadException.cs index 6386098a6..301099e2e 100644 --- a/src/Grpc/WorkItemPayloadException.cs +++ b/src/Grpc/WorkItemPayloadException.cs @@ -1,20 +1,41 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Grpc.Core; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask; /// -/// A received delivery whose payloads could not be resolved. The item is only safe to abandon, not dispatch. +/// A payload resolution failure after a successful transport read. /// +/// The payload resolution failure. +/// Safe failure details when the payload is confirmed missing. +internal class PayloadResolutionException(Exception innerException, P.TaskFailureDetails? permanentFailure) + : Exception("Failed to resolve external payloads.", innerException) +{ + /// Gets failure details for a confirmed permanent failure, or null for a retryable failure. + public P.TaskFailureDetails? PermanentFailure { get; } = permanentFailure; +} + +/// Identifies a failed delivery that must not be dispatched. /// The failed delivery, including its original completion token. /// The payload resolution failure. -internal sealed class WorkItemPayloadException(P.WorkItem workItem, Exception innerException) - : Exception("Failed to resolve payloads for a work-item delivery.", innerException) +/// Safe failure details when the payload is confirmed missing. +internal sealed class WorkItemPayloadException( + P.WorkItem workItem, Exception innerException, P.TaskFailureDetails? permanentFailure) + : PayloadResolutionException(innerException, permanentFailure) { /// /// Gets the failed delivery. Its payload fields may be partially resolved. /// public P.WorkItem WorkItem { get; } = workItem; } + +/// Exposes transport-only reads for recovery that needs history metadata, not payload values. +/// The response type. +internal interface IPayloadStreamReader +{ + /// Gets the underlying reader without payload resolution. + IAsyncStreamReader Inner { get; } +} diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 0052fd920..5aa77fe4a 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -536,14 +536,97 @@ void RunBackgroundTask(P.WorkItem? workItem, Func handler, CancellationTok { await handler(); } + catch (PayloadResolutionException ex) when (workItem is not null) + { + await this.OnWorkItemPayloadFailureAsync( + new WorkItemPayloadException(workItem, ex.InnerException!, ex.PermanentFailure), cancellation); + } catch (OperationCanceledException) { // Shutting down - ignore } catch (Exception ex) { - this.Logger.UnexpectedError(ex, GetWorkItemInstanceId(workItem)); - await this.AbandonWorkItemAsync(workItem, cancellation); + string instanceId = + workItem?.OrchestratorRequest?.InstanceId ?? + workItem?.ActivityRequest?.OrchestrationInstance?.InstanceId ?? + workItem?.EntityRequest?.InstanceId ?? + workItem?.EntityRequestV2?.InstanceId ?? + string.Empty; + this.Logger.UnexpectedError(ex, instanceId); + + if (workItem?.OrchestratorRequest != null) + { + this.Logger.AbandoningOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskOrchestratorWorkItemAsync( + new P.AbandonOrchestrationTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskOrchestratorWorkItemAsync), + cancellation); + this.Logger.AbandonedOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + } + else if (workItem?.ActivityRequest != null) + { + this.Logger.AbandoningActivityWorkItem( + instanceId, + workItem.ActivityRequest.Name, + workItem.ActivityRequest.TaskId, + workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskActivityWorkItemAsync( + new P.AbandonActivityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskActivityWorkItemAsync), + cancellation); + this.Logger.AbandonedActivityWorkItem( + instanceId, + workItem.ActivityRequest.Name, + workItem.ActivityRequest.TaskId, + workItem.CompletionToken ?? string.Empty); + } + else if (workItem?.EntityRequest != null) + { + this.Logger.AbandoningEntityWorkItem( + workItem.EntityRequest.InstanceId, + workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskEntityWorkItemAsync), + cancellation); + this.Logger.AbandonedEntityWorkItem( + workItem.EntityRequest.InstanceId, + workItem.CompletionToken ?? string.Empty); + } + else if (workItem?.EntityRequestV2 != null) + { + this.Logger.AbandoningEntityWorkItem( + workItem.EntityRequestV2.InstanceId, + workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskEntityWorkItemAsync), + cancellation); + this.Logger.AbandonedEntityWorkItem( + workItem.EntityRequestV2.InstanceId, + workItem.CompletionToken ?? string.Empty); + } } }); } @@ -552,9 +635,33 @@ async Task OnWorkItemPayloadFailureAsync(WorkItemPayloadException exception, Can { P.WorkItem workItem = exception.WorkItem; this.Logger.UnexpectedError(exception, GetWorkItemInstanceId(workItem)); + if (exception.PermanentFailure is { } failure + && await this.TryCompletePayloadFailureAsync(workItem, failure, cancellation)) + { + return; + } + try { - await this.AbandonWorkItemAsync(workItem, cancellation); + await this.ExecuteWithRetryAsync( + () => workItem.RequestCase switch + { + P.WorkItem.RequestOneofCase.OrchestratorRequest => + this.client.AbandonTaskOrchestratorWorkItemAsync( + new P.AbandonOrchestrationTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + P.WorkItem.RequestOneofCase.ActivityRequest => + this.client.AbandonTaskActivityWorkItemAsync( + new P.AbandonActivityTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + P.WorkItem.RequestOneofCase.EntityRequest or P.WorkItem.RequestOneofCase.EntityRequestV2 => + this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + _ => throw new InvalidOperationException($"Cannot abandon unresolved {workItem.RequestCase}."), + }, + "AbandonUnresolvedWorkItem", + cancellation); } catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) { @@ -563,58 +670,179 @@ async Task OnWorkItemPayloadFailureAsync(WorkItemPayloadException exception, Can } } - async Task AbandonWorkItemAsync(P.WorkItem? workItem, CancellationToken cancellation) + async Task TryCompletePayloadFailureAsync( + P.WorkItem workItem, P.TaskFailureDetails failure, CancellationToken cancellation) { - string instanceId = GetWorkItemInstanceId(workItem); - if (workItem?.OrchestratorRequest != null) + bool completing = false; + try { - this.Logger.AbandoningOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskOrchestratorWorkItemAsync( - new P.AbandonOrchestrationTaskRequest + if (workItem.ActivityRequest is { } activity) + { + if (activity.OrchestrationInstance is not { InstanceId.Length: > 0 } instance) + { + throw new InvalidOperationException("The activity has no owning orchestration instance."); + } + + completing = true; + await this.ExecuteWithRetryAsync( + async () => await this.client.CompleteActivityTaskAsync( + new P.ActivityResponse + { + InstanceId = instance.InstanceId, + TaskId = activity.TaskId, + CompletionToken = workItem.CompletionToken, + FailureDetails = failure, + }, + cancellationToken: cancellation), + nameof(this.client.CompleteActivityTaskAsync), + cancellation); + return true; + } + + if (workItem.OrchestratorRequest is not { } request) + { + // A missing entity state cannot be replaced without losing state or operation ownership. + return false; + } + + IReadOnlyList pastEvents = request.PastEvents; + if (request.RequiresHistoryStreaming) + { + using AsyncServerStreamingCall history = this.client.StreamInstanceHistory( + new P.StreamInstanceHistoryRequest { - CompletionToken = workItem.CompletionToken, + InstanceId = request.InstanceId, + ExecutionId = request.ExecutionId, + ForWorkItemProcessing = true, }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskOrchestratorWorkItemAsync), - cancellation); - this.Logger.AbandonedOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.ActivityRequest != null) - { - this.Logger.AbandoningActivityWorkItem( - instanceId, - workItem.ActivityRequest.Name, - workItem.ActivityRequest.TaskId, - workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskActivityWorkItemAsync( - new P.AbandonActivityTaskRequest + cancellationToken: cancellation); + if (history.ResponseStream is not IPayloadStreamReader payloadReader) + { + throw new InvalidOperationException("Unresolved history is unavailable for payload failure recovery."); + } + + List events = []; + await foreach (P.HistoryChunk chunk in payloadReader.Inner.ReadAllAsync(cancellation)) + { + events.AddRange(chunk.Events); + } + + pastEvents = events; + } + + ProtoUtils.EntityConversionState entities = new(insertMissingEntityUnlocks: true); + entities.SetOrchestrationInstance(new DTCore.OrchestrationInstance + { + InstanceId = request.InstanceId, ExecutionId = request.ExecutionId, + }); + bool foundStart = false; + foreach (P.HistoryEvent historyEvent in pastEvents.Concat(request.NewEvents)) + { + if (historyEvent.ExecutionStarted is { } start) + { + if (foundStart || string.IsNullOrEmpty(request.InstanceId) || string.IsNullOrEmpty(request.ExecutionId) + || start.OrchestrationInstance?.InstanceId != request.InstanceId + || start.OrchestrationInstance.ExecutionId != request.ExecutionId) { - CompletionToken = workItem.CompletionToken, + throw new InvalidOperationException("History does not identify the delivered execution."); + } + + foundStart = true; + } + else if (historyEvent.ExecutionCompleted is not null || historyEvent.ContinueAsNew is not null) + { + throw new InvalidOperationException("History already contains a terminal execution event."); + } + + P.SendEntityMessageAction? entityMessage = null; + if (historyEvent.EventSent is { } sent && DTCore.Common.Entities.IsEntityInstance(sent.InstanceId)) + { + entityMessage = new(); + EntityConversions.DecodeEntityMessageAction(sent.Name, sent.Input, sent.InstanceId, entityMessage, out _); + } + + if ((historyEvent.EntityLockRequested ?? entityMessage?.EntityLockRequested) is { } acquired) + { + if (string.IsNullOrEmpty(acquired.CriticalSectionId) || acquired.LockSet.Any(string.IsNullOrEmpty)) + { + throw new InvalidOperationException("History contains incomplete entity lock metadata."); + } + + entities.AddUnlockObligations(acquired); + } + else if ((historyEvent.EntityUnlockSent ?? entityMessage?.EntityUnlockSent) is { } released) + { + entities.RemoveUnlockObligation(released.TargetInstanceId); + } + } + + if (!foundStart) + { + throw new InvalidOperationException("History has no execution start for payload failure recovery."); + } + + P.OrchestratorResponse response = new() + { + InstanceId = request.InstanceId, + CompletionToken = workItem.CompletionToken, + NumEventsProcessed = request.NewEvents.Count, + OrchestrationTraceContext = request.OrchestrationTraceContext, + Actions = + { + new P.OrchestratorAction + { + CompleteOrchestration = new P.CompleteOrchestrationAction + { + OrchestrationStatus = P.OrchestrationStatus.Failed, + FailureDetails = failure, + }, }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskActivityWorkItemAsync), + }, + }; + await this.PrepareOrchestratorResponseAsync(response, entities, cancellation); + if (!this.internalOptions.ConvertOrchestrationEntityEvents) + { + foreach (P.OrchestratorAction action in response.Actions) + { + if (action.SendEntityMessage?.EntityUnlockSent is { } unlock) + { + if (EntityConversions.EncodeUnlockSent( + new P.HistoryEvent { EntityUnlockSent = unlock }, entities.CurrentInstance) is not EventSentEvent sent) + { + throw new InvalidOperationException("Entity release has no target."); + } + + action.SendEvent = new P.SendEventAction + { + Instance = new P.OrchestrationInstance { InstanceId = sent.InstanceId }, + Name = sent.Name, Data = sent.Input, + }; + } + } + } + + if (response.CalculateSize() > MaxInlineOrchestratorResponseSizeInBytes) + { + throw new InvalidOperationException("Payload failure response exceeds the inline gRPC limit."); + } + + completing = true; + await this.ExecuteWithRetryAsync( + async () => await this.client.CompleteOrchestratorTaskAsync(response, cancellationToken: cancellation), + nameof(this.client.CompleteOrchestratorTaskAsync), cancellation); - this.Logger.AbandonedActivityWorkItem( - instanceId, - workItem.ActivityRequest.Name, - workItem.ActivityRequest.TaskId, - workItem.CompletionToken ?? string.Empty); + return true; } - else if (workItem?.EntityRequest != null || workItem?.EntityRequestV2 != null) + catch (RpcException ex) when (completing && ex.StatusCode == StatusCode.NotFound) { - this.Logger.AbandoningEntityWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskEntityWorkItemAsync( - new P.AbandonEntityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskEntityWorkItemAsync), - cancellation); - this.Logger.AbandonedEntityWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + this.Logger.WorkItemOwnershipLost(workItem.RequestCase.ToString(), workItem.CompletionToken, ex); + return true; + } + catch (Exception ex) when (!completing && ex is InvalidOperationException or Newtonsoft.Json.JsonException) + { + // Do not terminally settle an execution when its routing/lock metadata cannot be reconstructed. + this.Logger.UnexpectedError(ex, GetWorkItemInstanceId(workItem)); + return false; } } diff --git a/src/Worker/Grpc/Worker.Grpc.csproj b/src/Worker/Grpc/Worker.Grpc.csproj index 0fa7b856b..2b5d41138 100644 --- a/src/Worker/Grpc/Worker.Grpc.csproj +++ b/src/Worker/Grpc/Worker.Grpc.csproj @@ -22,7 +22,6 @@ - diff --git a/test/Extensions/AzureBlobPayloads.Tests/OrchestratorPayloadFailureTests.cs b/test/Extensions/AzureBlobPayloads.Tests/OrchestratorPayloadFailureTests.cs index 94b86ab34..7ece5d4ae 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/OrchestratorPayloadFailureTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/OrchestratorPayloadFailureTests.cs @@ -27,6 +27,7 @@ enum PayloadFailure SizeLimit, ForbiddenUpload, InlineSizeLimit, + MissingDownload, } [Theory] @@ -62,6 +63,18 @@ public Task SuccessfulCompletion_ReleasesEntitiesExactlyOnceBeforeCompletion( useExternalizedPayloads: useExternalizedPayloads, explicitlyReleaseLocks: explicitlyReleaseLocks); + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public Task MissingPayloadDownload_ReleasesEntitiesBeforeFailingAsync(bool streamHistory, bool historyPayload) + => RunLockedOrchestrationAsync( + completeOrchestration: true, + streamHistory, + PayloadFailure.MissingDownload, + historyPayload: historyPayload); + [Theory] [InlineData(false, false, false)] [InlineData(false, false, true)] @@ -87,7 +100,8 @@ static async Task RunLockedOrchestrationAsync( bool streamHistory, PayloadFailure failure, bool useExternalizedPayloads = true, - bool explicitlyReleaseLocks = true) + bool explicitlyReleaseLocks = true, + bool historyPayload = false) { // Arrange const string InstanceId = "locked-payload-instance"; @@ -102,7 +116,7 @@ static async Task RunLockedOrchestrationAsync( }; string serializedPayload = JsonSerializer.Serialize(payload); Channel workItems = Channel.CreateUnbounded(); - Channel history = Channel.CreateUnbounded(); + P.HistoryChunk? streamedHistory = null; Channel completions = Channel.CreateUnbounded(); Mock invoker = new(MockBehavior.Strict); invoker.Setup(value => value.AsyncUnaryCall( @@ -127,12 +141,18 @@ static async Task RunLockedOrchestrationAsync( It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(() => new AsyncServerStreamingCall( - new ChannelStreamReader(history.Reader), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { })); + .Returns(() => + { + Channel replay = Channel.CreateUnbounded(); + replay.Writer.TryWrite(Assert.IsType(streamedHistory).Clone()); + replay.Writer.Complete(); + return new AsyncServerStreamingCall( + new ChannelStreamReader(replay.Reader), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { }); + }); invoker.Setup(value => value.AsyncUnaryCall( It.IsAny>(), It.IsAny(), @@ -142,7 +162,10 @@ static async Task RunLockedOrchestrationAsync( (_, _, _, response) => completions.Writer.TryWrite(response.Clone())) .Returns(() => CreateUnaryCall(new P.CompleteTaskResponse())); Mock store = new(MockBehavior.Strict); - store.Setup(value => value.IsKnownPayloadToken(It.IsAny())).Returns(false); + store.Setup(value => value.IsKnownPayloadToken(It.IsAny())) + .Returns((string value) => value == "blob:v1:missing"); + store.Setup(value => value.DownloadAsync("blob:v1:missing", It.IsAny())) + .ThrowsAsync(new PayloadStorageException("Payload missing.", new RequestFailedException(404, "Blob not found."))); if (failure == PayloadFailure.ForbiddenUpload) { store.Setup(value => value.UploadAsync(serializedPayload, It.IsAny())) @@ -280,11 +303,24 @@ await workItems.Writer.WriteAsync(new P.WorkItem }, }, }; + if (failure == PayloadFailure.MissingDownload) + { + if (historyPayload) + { + resumedRequest.PastEvents.Single(e => e.ExecutionStarted is not null).ExecutionStarted.Input = "blob:v1:missing"; + } + else + { + resumedRequest.NewEvents.Add(new P.HistoryEvent + { + EventRaised = new P.EventRaisedEvent { Name = "missing", Input = "blob:v1:missing" }, + }); + } + } + if (streamHistory) { - await history.Writer.WriteAsync( - new P.HistoryChunk { Events = { resumedRequest.PastEvents } }, timeout.Token); - history.Writer.Complete(); + streamedHistory = new P.HistoryChunk { Events = { resumedRequest.PastEvents } }; resumedRequest.PastEvents.Clear(); resumedRequest.RequiresHistoryStreaming = true; } @@ -345,6 +381,10 @@ await workItems.Writer.WriteAsync(new P.WorkItem store.Verify( value => value.UploadAsync(It.IsAny(), It.IsAny()), failure == PayloadFailure.ForbiddenUpload ? Times.Once() : Times.Never()); + if (failure == PayloadFailure.MissingDownload) + { + store.Verify(value => value.DownloadAsync("blob:v1:missing", It.IsAny()), Times.Once); + } } finally { diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index f29b71062..4edf73b44 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.IO; using System.Text.Json; +using Azure; using FluentAssertions; using Grpc.Core; using Grpc.Core.Interceptors; @@ -11,6 +13,7 @@ using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; +using Moq; using Xunit.Abstractions; using P = Microsoft.DurableTask.Protobuf; @@ -18,6 +21,99 @@ namespace Microsoft.DurableTask.Grpc.Tests; public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) : IntegrationTestBase(output, sidecarFixture) { + [Theory] + [InlineData(0, "activity")] + [InlineData(403, "activity")] + [InlineData(404, "activity")] + [InlineData(0, "orchestration")] + [InlineData(403, "orchestration")] + [InlineData(404, "orchestration")] + [InlineData(404, "suborchestration")] + public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int storageStatus, string target) + { + // Arrange + const string Token = "blob:v1:missing"; + TaskName orchestration = nameof(HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync); + TaskName activity = "HydrationActivity"; + TaskName child = "HydrationChild"; + TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource downloadFailed = new(TaskCreationOptions.RunContinuationsAsynchronously); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value == Token); + store.Setup(s => s.UploadAsync(It.IsAny(), It.IsAny())).ReturnsAsync(Token); + store.Setup(s => s.DownloadAsync(Token, It.IsAny())).Returns(() => + { + downloadFailed.TrySetResult(); + Exception failure = storageStatus == 0 + ? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed."); + return Task.FromException(failure); + }); + await using HostTestLifetime server = await this.StartWorkerAsync(worker => + { + worker.AddTasks(tasks => tasks + .AddOrchestratorFunc(orchestration, (context, input) => + input == "bad-child" + ? context.CallSubOrchestratorAsync(child, new string('X', 1024)) + : context.CallActivityAsync(activity, input == "bad" ? new string('X', 1024) : input)) + .AddOrchestratorFunc(child, (_, input) => Task.FromResult(input)) + .AddActivityFunc(activity, async (_, input) => + { + if (input == "running") + { + running.TrySetResult(); + await release.Task.WaitAsync(this.TimeoutToken); + } + + return input; + })); + worker.UseExternalizedPayloads(); + worker.Services.AddSingleton(store.Object); + }, clientConfigure: client => client.UseExternalizedPayloads(), + servicesConfigure: services => services.AddExternalizedPayloadStore(options => + { + options.ThresholdBytes = 16; + options.ConnectionString = "UseDevelopmentStorage=true"; + })); + + try + { + // Act + string alreadyRunning = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "running"); + await running.Task.WaitAsync(this.TimeoutToken); + string failed = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, target switch + { + "orchestration" => new string('X', 1024), + "suborchestration" => "bad-child", + _ => "bad", + }); + await downloadFailed.Task.WaitAsync(this.TimeoutToken); + string following = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "following"); + OrchestrationMetadata followingResult = await server.Client.WaitForInstanceCompletionAsync( + following, getInputsAndOutputs: true, this.TimeoutToken); + release.TrySetResult(); + OrchestrationMetadata runningResult = await server.Client.WaitForInstanceCompletionAsync( + alreadyRunning, getInputsAndOutputs: true, this.TimeoutToken); + + // Assert + Assert.Equal(OrchestrationRuntimeStatus.Completed, followingResult.RuntimeStatus); + Assert.Equal("following", followingResult.ReadOutputAs()); + Assert.Equal(OrchestrationRuntimeStatus.Completed, runningResult.RuntimeStatus); + Assert.Equal("running", runningResult.ReadOutputAs()); + Assert.Single(this.GetLogs(), log => log.EventId.Name == "EstablishedWorkItemConnection"); + if (storageStatus == 404) + { + OrchestrationMetadata failure = await server.Client.WaitForInstanceCompletionAsync( + failed, getInputsAndOutputs: false, this.TimeoutToken); + Assert.Equal(OrchestrationRuntimeStatus.Failed, failure.RuntimeStatus); + } + } + finally + { + release.TrySetResult(); + } + } + [Theory] [InlineData(true)] [InlineData(false)] diff --git a/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs b/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs deleted file mode 100644 index 42a65e63b..000000000 --- a/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs +++ /dev/null @@ -1,536 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Concurrent; -using System.IO; -using System.Net; -using System.Text.Json; -using System.Threading.Channels; -using Azure; -using Google.Protobuf.WellKnownTypes; -using Grpc.Core; -using Grpc.Core.Interceptors; -using Grpc.Net.Client; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Hosting.Server; -using Microsoft.AspNetCore.Hosting.Server.Features; -using Microsoft.AspNetCore.Server.Kestrel.Core; -using Microsoft.DurableTask.Worker; -using Microsoft.DurableTask.Worker.AzureManaged; -using Microsoft.DurableTask.Worker.Grpc; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Moq; -using P = Microsoft.DurableTask.Protobuf; - -namespace Microsoft.DurableTask.Grpc.Tests; - -public class PayloadHydrationFailureTests -{ - const string TokenPrefix = "test-blob://"; - const string BadToken = TokenPrefix + "bad"; - const string FailedCompletionToken = "failed-delivery"; - static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(15); - - public static IEnumerable HydrationFailures() - { - P.WorkItem.RequestOneofCase[] kinds = - [ - P.WorkItem.RequestOneofCase.ActivityRequest, - P.WorkItem.RequestOneofCase.OrchestratorRequest, - P.WorkItem.RequestOneofCase.EntityRequest, - P.WorkItem.RequestOneofCase.EntityRequestV2, - ]; - - foreach (P.WorkItem.RequestOneofCase kind in kinds) - { - foreach (int storageStatus in new[] { 404, 403, 0 }) - { - foreach (bool ownershipLost in new[] { false, true }) - { - foreach (bool useAzureManaged in new[] { false, true }) - { - yield return [kind, storageStatus, ownershipLost, useAzureManaged]; - } - } - } - } - } - - [Theory] - [MemberData(nameof(HydrationFailures))] - public async Task HydrationFailure_AbandonsOnlyFailedDeliveryAndKeepsStreamAsync( - P.WorkItem.RequestOneofCase kind, int storageStatus, bool ownershipLost, bool useAzureManaged) - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - sidecar.AbandonStatus = ownershipLost ? StatusCode.NotFound : StatusCode.OK; - Exception failure = storageStatus == 0 - ? new IOException("Transient payload download failure.") - : new RequestFailedException(storageStatus, "Payload download failed."); - Mock store = CreatePayloadStore(failure); - TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously); - TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); - ConcurrentQueue inputs = new(); - await using ServiceProvider provider = CreateWorker(sidecar, store.Object, async input => - { - inputs.Enqueue(input); - if (input == "running") - { - running.TrySetResult(); - await release.Task.WaitAsync(timeout.Token); - } - - return input; - }, useAzureManaged); - IHostedService worker = Assert.Single(provider.GetServices()); - await worker.StartAsync(timeout.Token); - - try - { - // Act - sidecar.Send(CreateHealthyActivity("running")); - await running.Task.WaitAsync(timeout.Token); - sidecar.Send(CreateFailedWorkItem(kind)); - sidecar.Send(new P.WorkItem { HealthPing = new P.HealthPing() }); - sidecar.Send(CreateHealthyActivity("following")); - P.ActivityResponse following = await sidecar.Completions.Reader.ReadAsync(timeout.Token); - release.TrySetResult(); - P.ActivityResponse alreadyRunning = await sidecar.Completions.Reader.ReadAsync(timeout.Token); - - // Assert - Assert.Equal(1, sidecar.StreamCount); - Assert.Equal(0, sidecar.ClosedStreamCount); - Assert.Equal(1, sidecar.HelloCount); - P.WorkItem.RequestOneofCase abandonKind = kind == P.WorkItem.RequestOneofCase.EntityRequestV2 - ? P.WorkItem.RequestOneofCase.EntityRequest - : kind; - Assert.Equal((abandonKind, FailedCompletionToken), Assert.Single(sidecar.Abandons)); - Assert.Equal(new[] { "running", "following" }, inputs.ToArray()); - Assert.Equal("following", following.CompletionToken); - Assert.Equal("\"following\"", following.Result); - Assert.Null(following.FailureDetails); - Assert.Equal("running", alreadyRunning.CompletionToken); - Assert.Equal("\"running\"", alreadyRunning.Result); - Assert.Null(alreadyRunning.FailureDetails); - store.Verify(s => s.DownloadAsync(BadToken, It.IsAny()), Times.Once); - } - finally - { - release.TrySetResult(); - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - } - } - - [Theory] - [InlineData(StatusCode.Unknown, false)] - [InlineData(StatusCode.Unavailable, false)] - [InlineData(StatusCode.PermissionDenied, false)] - [InlineData(StatusCode.Internal, false)] - [InlineData(StatusCode.Unavailable, true)] - public async Task TransportOrAbandonFailure_ReconnectsAsync(StatusCode status, bool streamFailure) - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - sidecar.AbandonStatus = status; - Mock store = CreatePayloadStore(new IOException("Payload download failed.")); - await using ServiceProvider provider = CreateWorker(sidecar, store.Object); - IHostedService worker = Assert.Single(provider.GetServices()); - await worker.StartAsync(timeout.Token); - - try - { - Assert.Equal(1, await sidecar.Streams.Reader.ReadAsync(timeout.Token)); - - // Act - if (streamFailure) - { - sidecar.FailStream(status); - } - else - { - sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); - } - - Assert.Equal(2, await sidecar.Streams.Reader.ReadAsync(timeout.Token)); - sidecar.Send(CreateHealthyActivity("following")); - P.ActivityResponse response = await sidecar.Completions.Reader.ReadAsync(timeout.Token); - - // Assert - Assert.Equal(2, sidecar.StreamCount); - Assert.Equal(2, sidecar.HelloCount); - Assert.Equal("following", response.CompletionToken); - Assert.Null(response.FailureDetails); - Assert.Equal(streamFailure ? 0 : 1, sidecar.Abandons.Count); - } - finally - { - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - } - } - - [Fact] - public async Task HydrationFailure_RedeliveryCompletesAfterPayloadRecoversAsync() - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - Mock store = CreatePayloadStore(new IOException("Payload download failed.")); - store.SetupSequence(s => s.DownloadAsync(BadToken, It.IsAny())) - .ThrowsAsync(new RequestFailedException(404, "Payload not found.")) - .ReturnsAsync("\"recovered\""); - await using ServiceProvider provider = CreateWorker(sidecar, store.Object); - IHostedService worker = Assert.Single(provider.GetServices()); - await worker.StartAsync(timeout.Token); - - try - { - // Act - sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); - await sidecar.AbandonStarted.Task.WaitAsync(timeout.Token); - P.WorkItem redelivery = CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest); - redelivery.CompletionToken = "redelivery"; - sidecar.Send(redelivery); - P.ActivityResponse response = await sidecar.Completions.Reader.ReadAsync(timeout.Token); - - // Assert - Assert.Equal(1, sidecar.StreamCount); - Assert.Equal((P.WorkItem.RequestOneofCase.ActivityRequest, FailedCompletionToken), Assert.Single(sidecar.Abandons)); - Assert.Equal("redelivery", response.CompletionToken); - Assert.Equal("\"recovered\"", response.Result); - Assert.Null(response.FailureDetails); - } - finally - { - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ShutdownDuringHydrationOrAbandon_DoesNotReconnectAsync(bool duringAbandon) - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - sidecar.HoldAbandon = duringAbandon; - TaskCompletionSource downloading = new(TaskCreationOptions.RunContinuationsAsynchronously); - Mock store = CreatePayloadStore(new IOException("Payload download failed.")); - if (!duringAbandon) - { - store.Setup(s => s.DownloadAsync(BadToken, It.IsAny())) - .Returns(async (string _, CancellationToken cancellation) => - { - downloading.TrySetResult(cancellation); - await Task.Delay(Timeout.Infinite, cancellation); - return string.Empty; - }); - } - - await using ServiceProvider provider = CreateWorker(sidecar, store.Object); - IHostedService worker = Assert.Single(provider.GetServices()); - await worker.StartAsync(timeout.Token); - - try - { - sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); - CancellationToken downloadCancellation = default; - if (duringAbandon) - { - await sidecar.AbandonStarted.Task.WaitAsync(timeout.Token); - } - else - { - downloadCancellation = await downloading.Task.WaitAsync(timeout.Token); - } - - // Act - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - - // Assert - Assert.Equal(1, sidecar.StreamCount); - Assert.Equal(duringAbandon ? 1 : 0, sidecar.Abandons.Count); - Assert.False(sidecar.Completions.Reader.TryRead(out _)); - if (!duringAbandon) - { - Assert.True(downloadCancellation.IsCancellationRequested); - } - } - finally - { - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task NonWorkItemHydrationFailure_PropagatesOriginalExceptionAsync(bool historyStream) - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - RequestFailedException failure = new(403, "Payload download forbidden."); - Mock store = CreatePayloadStore(failure); - using GrpcChannel channel = GrpcChannel.ForAddress(sidecar.Address); - P.TaskHubSidecarService.TaskHubSidecarServiceClient client = new(channel.Intercept( - new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions()))); - - // Act - Exception? observed; - if (historyStream) - { - using AsyncServerStreamingCall call = client.StreamInstanceHistory( - new P.StreamInstanceHistoryRequest { InstanceId = "instance" }, cancellationToken: timeout.Token); - observed = await Record.ExceptionAsync(() => call.ResponseStream.MoveNext(timeout.Token)); - } - else - { - using AsyncUnaryCall call = client.GetInstanceAsync( - new P.GetInstanceRequest { InstanceId = "instance" }, cancellationToken: timeout.Token); - observed = await Record.ExceptionAsync(async () => await call.ResponseAsync); - } - - // Assert - Assert.Same(failure, observed); - Assert.Empty(sidecar.Abandons); - } - - static Mock CreatePayloadStore(Exception failure) - { - Mock store = new(); - store.Setup(s => s.IsKnownPayloadToken(It.IsAny())) - .Returns((string value) => value.StartsWith(TokenPrefix, StringComparison.Ordinal)); - store.Setup(s => s.DownloadAsync(It.IsAny(), It.IsAny())) - .Returns((string token, CancellationToken _) => token == BadToken - ? Task.FromException(failure) - : Task.FromResult(JsonSerializer.Serialize(token[TokenPrefix.Length..]))); - return store; - } - - static ServiceProvider CreateWorker( - ScriptedSidecar sidecar, - PayloadStore store, - Func>? runActivity = null, - bool useAzureManaged = false) - { - ServiceCollection services = new(); - services.AddLogging(logging => logging.ClearProviders()); - services.AddSingleton(store); - services.AddDurableTaskWorker(builder => - { - if (useAzureManaged) - { - builder.UseDurableTaskScheduler($"Endpoint={sidecar.Address};TaskHub=test;Authentication=None"); - } - else - { - builder.UseGrpc(sidecar.Address); - } - - builder.Services.Configure(builder.Name, options => - { - options.Internal.TransientRetryMaxAttempts = 1; - options.Internal.ReconnectBackoffBase = TimeSpan.Zero; - options.Internal.ReconnectBackoffCap = TimeSpan.Zero; - }); - builder.UseExternalizedPayloads(); - builder.AddTasks(tasks => tasks.AddActivityFunc( - "Healthy", (_, input) => runActivity?.Invoke(input!) ?? Task.FromResult(input!))); - }); - return services.BuildServiceProvider(); - } - - static P.WorkItem CreateHealthyActivity(string name) => new() - { - CompletionToken = name, - ActivityRequest = new P.ActivityRequest - { - Name = "Healthy", - TaskId = 1, - Input = TokenPrefix + name, - OrchestrationInstance = new P.OrchestrationInstance { InstanceId = name, ExecutionId = "execution" }, - }, - }; - - static P.WorkItem CreateFailedWorkItem(P.WorkItem.RequestOneofCase kind) - { - P.WorkItem item = new() { CompletionToken = FailedCompletionToken }; - switch (kind) - { - case P.WorkItem.RequestOneofCase.ActivityRequest: - item.ActivityRequest = CreateHealthyActivity("failed").ActivityRequest; - item.ActivityRequest.Input = BadToken; - break; - case P.WorkItem.RequestOneofCase.OrchestratorRequest: - item.OrchestratorRequest = new P.OrchestratorRequest - { - InstanceId = "failed", - PastEvents = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = TokenPrefix + "state" } } }, - NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Name = "event", Input = BadToken } } }, - }; - break; - case P.WorkItem.RequestOneofCase.EntityRequest: - item.EntityRequest = new P.EntityBatchRequest - { - InstanceId = "@entity@failed", - EntityState = TokenPrefix + "state", - Operations = { new P.OperationRequest { Operation = "operation", Input = BadToken } }, - }; - break; - case P.WorkItem.RequestOneofCase.EntityRequestV2: - item.EntityRequestV2 = new P.EntityRequest - { - InstanceId = "@entity@failed", - EntityState = TokenPrefix + "state", - OperationRequests = { new P.HistoryEvent { EntityOperationSignaled = new P.EntityOperationSignaledEvent { Input = BadToken } } }, - }; - break; - default: - throw new ArgumentOutOfRangeException(nameof(kind), kind, "Unsupported work item kind."); - } - - return item; - } - - sealed class ScriptedSidecar : P.TaskHubSidecarService.TaskHubSidecarServiceBase, IAsyncDisposable - { - readonly Channel, Task>> deliveries = - Channel.CreateUnbounded, Task>>(); - IHost host = null!; - int streamCount; - int closedStreamCount; - int helloCount; - - public string Address { get; private set; } = string.Empty; - - public StatusCode AbandonStatus { get; set; } - - public bool HoldAbandon { get; set; } - - public int StreamCount => Volatile.Read(ref this.streamCount); - - public int ClosedStreamCount => Volatile.Read(ref this.closedStreamCount); - - public int HelloCount => Volatile.Read(ref this.helloCount); - - public Channel Streams { get; } = Channel.CreateUnbounded(); - - public Channel Completions { get; } = Channel.CreateUnbounded(); - - public ConcurrentQueue<(P.WorkItem.RequestOneofCase Kind, string Token)> Abandons { get; } = new(); - - public TaskCompletionSource AbandonStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - - public static async Task StartAsync() - { - ScriptedSidecar sidecar = new(); - sidecar.host = Host.CreateDefaultBuilder() - .ConfigureLogging(logging => logging.ClearProviders()) - .ConfigureWebHostDefaults(web => web - .UseKestrel(options => options.Listen(IPAddress.Loopback, 0, endpoint => endpoint.Protocols = HttpProtocols.Http2)) - .ConfigureServices(services => - { - services.AddGrpc(); - services.AddSingleton(sidecar); - }) - .Configure(app => - { - app.UseRouting(); - app.UseEndpoints(endpoints => endpoints.MapGrpcService()); - })) - .Build(); - await sidecar.host.StartAsync().WaitAsync(TestTimeout); - sidecar.Address = Assert.Single(sidecar.host.Services.GetRequiredService() - .Features.Get()!.Addresses); - return sidecar; - } - - public void Send(P.WorkItem item) => Assert.True(this.deliveries.Writer.TryWrite(stream => stream.WriteAsync(item))); - - public void FailStream(StatusCode status) => Assert.True(this.deliveries.Writer.TryWrite( - _ => Task.FromException(new RpcException(new Status(status, "Stream transport failed."))))); - - public override Task Hello(Empty request, ServerCallContext context) - { - Interlocked.Increment(ref this.helloCount); - return Task.FromResult(new Empty()); - } - - public override async Task GetWorkItems(P.GetWorkItemsRequest request, IServerStreamWriter responseStream, ServerCallContext context) - { - await context.WriteResponseHeadersAsync(new Metadata()); - this.Streams.Writer.TryWrite(Interlocked.Increment(ref this.streamCount)); - try - { - await foreach (Func, Task> deliver in this.deliveries.Reader.ReadAllAsync(context.CancellationToken)) - { - await deliver(responseStream); - } - } - finally - { - Interlocked.Increment(ref this.closedStreamCount); - } - } - - public override Task CompleteActivityTask(P.ActivityResponse request, ServerCallContext context) - { - this.Completions.Writer.TryWrite(request); - return Task.FromResult(new P.CompleteTaskResponse()); - } - - public override async Task AbandonTaskActivityWorkItem(P.AbandonActivityTaskRequest request, ServerCallContext context) - { - await this.AbandonAsync(P.WorkItem.RequestOneofCase.ActivityRequest, request.CompletionToken, context); - return new P.AbandonActivityTaskResponse(); - } - - public override async Task AbandonTaskOrchestratorWorkItem(P.AbandonOrchestrationTaskRequest request, ServerCallContext context) - { - await this.AbandonAsync(P.WorkItem.RequestOneofCase.OrchestratorRequest, request.CompletionToken, context); - return new P.AbandonOrchestrationTaskResponse(); - } - - public override async Task AbandonTaskEntityWorkItem(P.AbandonEntityTaskRequest request, ServerCallContext context) - { - await this.AbandonAsync(P.WorkItem.RequestOneofCase.EntityRequest, request.CompletionToken, context); - return new P.AbandonEntityTaskResponse(); - } - - public override Task GetInstance(P.GetInstanceRequest request, ServerCallContext context) - => Task.FromResult(new P.GetInstanceResponse { OrchestrationState = new P.OrchestrationState { Input = BadToken } }); - - public override Task StreamInstanceHistory(P.StreamInstanceHistoryRequest request, IServerStreamWriter responseStream, ServerCallContext context) - => responseStream.WriteAsync(new P.HistoryChunk - { - Events = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = BadToken } } }, - }); - - public async ValueTask DisposeAsync() - { - using IHost host = this.host; - await host.StopAsync().WaitAsync(TestTimeout); - } - - async Task AbandonAsync(P.WorkItem.RequestOneofCase kind, string token, ServerCallContext context) - { - this.Abandons.Enqueue((kind, token)); - this.AbandonStarted.TrySetResult(); - if (this.HoldAbandon) - { - await Task.Delay(Timeout.Infinite, context.CancellationToken); - } - - if (this.AbandonStatus != StatusCode.OK) - { - throw new RpcException(new Status(this.AbandonStatus, "Abandon failed.")); - } - } - } -} diff --git a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs index 4b3505c16..69d642140 100644 --- a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs +++ b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs @@ -5,10 +5,12 @@ using System.Diagnostics; using System.IO; using System.Reflection; +using Azure; using DurableTask.Core; using DurableTask.Core.History; using Google.Protobuf.WellKnownTypes; using Grpc.Core; +using Grpc.Core.Interceptors; using Microsoft.DurableTask; using Microsoft.DurableTask.Tests.Logging; using Microsoft.DurableTask.Tracing; @@ -49,6 +51,160 @@ public class GrpcDurableTaskWorkerTests .GetNestedType("Processor", BindingFlags.NonPublic)! .GetMethod("OnRunOrchestratorAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; + [Theory] + [CombinatorialData] + public async Task ProcessorExecuteAsync_HydrationFailure_SettlesOnlyFailedDeliveryAsync( + [CombinatorialValues(P.WorkItem.RequestOneofCase.ActivityRequest, P.WorkItem.RequestOneofCase.OrchestratorRequest, + P.WorkItem.RequestOneofCase.EntityRequest, P.WorkItem.RequestOneofCase.EntityRequestV2)] P.WorkItem.RequestOneofCase kind, + [CombinatorialValues(0, 403, 404)] int storageStatus, + [CombinatorialValues(StatusCode.OK, StatusCode.NotFound, StatusCode.Unavailable, StatusCode.PermissionDenied)] StatusCode settlementStatus) + { + // Arrange + const string PayloadToken = "blob:v1:bad"; + P.WorkItem failed = new() { CompletionToken = "failed" }; + switch (kind) + { + case P.WorkItem.RequestOneofCase.ActivityRequest: + failed.ActivityRequest = new P.ActivityRequest + { + Input = PayloadToken, OrchestrationInstance = new P.OrchestrationInstance { InstanceId = "failed-instance" }, + }; + break; + case P.WorkItem.RequestOneofCase.OrchestratorRequest: + failed.OrchestratorRequest = new P.OrchestratorRequest + { + InstanceId = "failed-instance", ExecutionId = "failed-execution", + PastEvents = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent + { + OrchestrationInstance = new P.OrchestrationInstance { InstanceId = "failed-instance", ExecutionId = "failed-execution" }, + } } }, + NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Name = "input", Input = PayloadToken } } }, + }; + break; + case P.WorkItem.RequestOneofCase.EntityRequest: + failed.EntityRequest = new P.EntityBatchRequest { EntityState = PayloadToken }; + break; + case P.WorkItem.RequestOneofCase.EntityRequestV2: + failed.EntityRequestV2 = new P.EntityRequest { EntityState = PayloadToken }; + break; + } + + if (failed.OrchestratorRequest is { } orchestration) + { + EventSentEvent acquire = Assert.IsType(EntityConversions.EncodeLockRequested( + new P.HistoryEvent + { + EntityLockRequested = new P.EntityLockRequestedEvent + { + CriticalSectionId = "00000000-0000-0000-0000-000000000001", + LockSet = { "@counter@one" }, + ParentInstanceId = orchestration.InstanceId, + }, + }, + new OrchestrationInstance { InstanceId = orchestration.InstanceId, ExecutionId = orchestration.ExecutionId })); + orchestration.PastEvents.Add(new P.HistoryEvent + { + EventSent = new P.EventSentEvent { InstanceId = acquire.InstanceId, Name = acquire.Name, Input = acquire.Input }, + }); + } + + Queue deliveries = new([failed, new P.WorkItem { HealthPing = new P.HealthPing() }]); + TaskCompletionSource pingDispatched = new(TaskCreationOptions.RunContinuationsAsynchronously); + P.WorkItem current = null!; + Mock> reader = new(); + reader.SetupGet(r => r.Current).Returns(() => current); + reader.Setup(r => r.MoveNext(It.IsAny())).Returns(async (CancellationToken cancellation) => + { + if (deliveries.TryDequeue(out P.WorkItem? item)) + { + current = item; + return true; + } + + pingDispatched.TrySetResult(); + await Task.Delay(Timeout.Infinite, cancellation); + return false; + }); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value == PayloadToken); + Exception failure = storageStatus == 0 ? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed."); + store.Setup(s => s.DownloadAsync(PayloadToken, It.IsAny())).ThrowsAsync(failure); + bool terminal = storageStatus == 404 && kind is P.WorkItem.RequestOneofCase.ActivityRequest or P.WorkItem.RequestOneofCase.OrchestratorRequest; + string expectedOperation = terminal + ? kind == P.WorkItem.RequestOneofCase.ActivityRequest ? "CompleteActivityTask" : "CompleteOrchestratorTask" + : kind switch + { + P.WorkItem.RequestOneofCase.ActivityRequest => "AbandonTaskActivityWorkItem", + P.WorkItem.RequestOneofCase.OrchestratorRequest => "AbandonTaskOrchestratorWorkItem", + _ => "AbandonTaskEntityWorkItem", + }; + List settlements = []; + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (_, _) => CreateServerStreamingCall(reader.Object)) + { + UnaryResponse = (operation, request) => + { + Assert.Equal(expectedOperation, operation); + settlements.Add(request); + (string Token, object Reply) result = request switch + { + P.AbandonActivityTaskRequest r => (r.CompletionToken, new P.AbandonActivityTaskResponse()), + P.AbandonOrchestrationTaskRequest r => (r.CompletionToken, new P.AbandonOrchestrationTaskResponse()), + P.AbandonEntityTaskRequest r => (r.CompletionToken, new P.AbandonEntityTaskResponse()), + P.ActivityResponse r => (r.CompletionToken, new P.CompleteTaskResponse()), + P.OrchestratorResponse r => (r.CompletionToken, new P.CompleteTaskResponse()), + _ => throw new InvalidOperationException($"Unexpected request {request.GetType()}."), + }; + Assert.Equal("failed", result.Token); + if (settlementStatus != StatusCode.OK) + { + throw new RpcException(new Status(settlementStatus, "Settlement failed.")); + } + + return result.Reply; + }, + }; + GrpcDurableTaskWorkerOptions options = new(); + options.Internal.TransientRetryMaxAttempts = 1; + options.Internal.ReconnectBackoffBase = options.Internal.ReconnectBackoffCap = TimeSpan.Zero; + object processor = CreateProcessor(CreateWorker(options), new P.TaskHubSidecarService.TaskHubSidecarServiceClient( + invoker.Intercept(new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions())))); + using CancellationTokenSource shutdown = new(); + + // Act + Task processing = InvokeProcessorExecuteAsync(processor, shutdown.Token); + try + { + await pingDispatched.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally + { + shutdown.Cancel(); + await processing; + } + + // Assert + object settlement = Assert.Single(settlements); + if (settlement is P.ActivityResponse activityFailure) + { + Assert.True(activityFailure.FailureDetails.IsNonRetriable); + } + else if (settlement is P.OrchestratorResponse orchestrationFailure) + { + P.OrchestratorAction release = Assert.Single(orchestrationFailure.Actions, a => a.SendEvent is not null); + Assert.Equal("@counter@one", release.SendEvent.Instance.InstanceId); + Assert.Equal("release", release.SendEvent.Name); + P.CompleteOrchestrationAction action = Assert.Single(orchestrationFailure.Actions, a => a.CompleteOrchestration is not null).CompleteOrchestration; + Assert.Equal(P.OrchestrationStatus.Failed, action.OrchestrationStatus); + Assert.True(action.FailureDetails.IsNonRetriable); + } + + Assert.Equal(settlementStatus is StatusCode.OK or StatusCode.NotFound ? 1 : 2, invoker.GetWorkItemsCallCount); + store.Verify(s => s.DownloadAsync(PayloadToken, It.IsAny()), Times.Once); + + } + [Fact] public async Task ExecuteAsync_ConnectFailureThreshold_RecreatesConfiguredChannel() { @@ -2199,6 +2355,8 @@ public ScriptedWorkerCallInvoker( public int GetWorkItemsCallCount => Volatile.Read(ref this.getWorkItemsCallCount); + public Func? UnaryResponse { get; init; } + public override TResponse BlockingUnaryCall(Method method, string? host, CallOptions options, TRequest request) { throw new NotSupportedException(); @@ -2212,6 +2370,11 @@ public override AsyncUnaryCall AsyncUnaryCall(Me return (AsyncUnaryCall)(object)call; } + if (this.UnaryResponse is { } respond) + { + return CreateUnaryCall(Task.FromResult((TResponse)respond(method.Name, request))); + } + throw new NotSupportedException($"Unexpected unary method {method.FullName}."); } diff --git a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs index 4bc5bb9d4..f3008385d 100644 --- a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs +++ b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs @@ -222,59 +222,6 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti result.Outcome.Should().Be(WorkItemStreamOutcome.Shutdown); } - [Fact] - public async Task HydrationFailure_RearmsTimerAndRecoversBeforeReadingNextItemAsync() - { - // Arrange - P.WorkItem failed = new() - { - CompletionToken = "failed-token", - OrchestratorRequest = new P.OrchestratorRequest - { - NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Input = "partially resolved" } } }, - }, - }; - WorkItemPayloadException failure = new(failed, new InvalidOperationException("Download failed.")); - P.WorkItem healthy = new() { HealthPing = new P.HealthPing() }; - Mock> reader = new(MockBehavior.Strict); - reader.SetupSequence(r => r.MoveNext(It.IsAny())) - .ThrowsAsync(failure) - .ReturnsAsync(true) - .ReturnsAsync(false); - reader.SetupGet(r => r.Current).Returns(healthy); - List events = new(); - List dispatched = new(); - - // Act - WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - stream: reader.Object, - silentDisconnectTimeout: TimeSpan.FromSeconds(5), - onItem: item => - { - events.Add("dispatch"); - dispatched.Add(item); - }, - onFirstMessage: () => events.Add("first"), - cancellation: CancellationToken.None, - onSilentDisconnectTimerArmed: () => events.Add("armed"), - onPayloadFailure: (exception, cancellation) => - { - Assert.Same(failure, exception); - Assert.Same(failed, exception.WorkItem); - Assert.True(cancellation.CanBeCanceled); - events.Add("abandon"); - return Task.CompletedTask; - }); - - // Assert - Assert.Equal(WorkItemStreamOutcome.GracefulDrain, result.Outcome); - Assert.True(result.FirstMessageObserved); - Assert.Equal(new[] { "armed", "armed", "first", "abandon", "armed", "dispatch" }, events); - Assert.Same(healthy, Assert.Single(dispatched)); - reader.VerifyGet(r => r.Current, Times.Once); - reader.Verify(r => r.MoveNext(It.IsAny()), Times.Exactly(3)); - } - static IAsyncStreamReader EmptyStream() => StreamOf(); static IAsyncStreamReader StreamOf(params P.WorkItem[] items) => new SequenceStreamReader(items); From 3837371f333e4d5cf110fa88f2d7e140e9d0a38a Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 10 Nov 2026 15:45:17 -0800 Subject: [PATCH 3/9] Revert "Fail confirmed-missing payload deliveries without disconnecting healthy work" This reverts commit a0f5320589170f8298411a31507ae38f8b12d331 at the user request. Restore retryable delivery-local abandonment for all incoming hydration failures. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- .../Interceptors/PayloadInterceptor.cs | 35 +- src/Extensions/AzureBlobPayloads/README.md | 14 + src/Grpc/Grpc.csproj | 1 + src/Grpc/WorkItemPayloadException.cs | 27 +- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 320 ++--------- src/Worker/Grpc/Worker.Grpc.csproj | 1 + .../OrchestratorPayloadFailureTests.cs | 64 +-- .../LargePayloadTests.cs | 96 ---- .../PayloadHydrationFailureTests.cs | 536 ++++++++++++++++++ .../Grpc.Tests/GrpcDurableTaskWorkerTests.cs | 163 ------ .../Grpc.Tests/WorkItemStreamConsumerTests.cs | 53 ++ 12 files changed, 675 insertions(+), 637 deletions(-) create mode 100644 test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bca6c72c..050e32e52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Isolate incoming payload download failures to their deliveries. Fail activity/orchestration deliveries with confirmed missing blobs while preserving entity releases and the shared stream; retry recoverable failures without disrupting unrelated work ([#825](https://github.com/microsoft/durabletask-dotnet/issues/825)). +- Isolate incoming payload hydration failures to the failed delivery, preserving the shared work-item stream and unrelated in-flight work ([#825](https://github.com/microsoft/durabletask-dotnet/issues/825)). - Validate all eligible externalized payload sizes before uploading any field of an orchestration response or entity batch, including nested carryover and rewind history. Deterministic size failures now precede hypothetical upload failures and cause no uploads or partial token substitutions ([#820](https://github.com/microsoft/durabletask-dotnet/issues/820)). ## v1.28.0 diff --git a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs index 11e12a3ea..9207456a3 100644 --- a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs +++ b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System.Text; -using Azure; using Grpc.Core; using Grpc.Core.Interceptors; using P = Microsoft.DurableTask.Protobuf; @@ -149,30 +148,14 @@ public override AsyncServerStreamingCall AsyncServerStreamingCall( IAsyncStreamReader inner, - Func> transform) : IAsyncStreamReader, IPayloadStreamReader + Func> transform) : IAsyncStreamReader { readonly IAsyncStreamReader inner = inner; readonly Func> transform = transform; public T Current { get; private set; } = default!; - public IAsyncStreamReader Inner => this.inner; - public async Task MoveNext(CancellationToken cancellationToken) { bool hasNext = await this.inner.MoveNext(cancellationToken); diff --git a/src/Extensions/AzureBlobPayloads/README.md b/src/Extensions/AzureBlobPayloads/README.md index 146700caf..e6fd9f6d1 100644 --- a/src/Extensions/AzureBlobPayloads/README.md +++ b/src/Extensions/AzureBlobPayloads/README.md @@ -170,6 +170,20 @@ orphan blobs: real I/O failures, cancellation during uploads, and uncertain backend submission can still leave unrecorded uploads. No catch-and-delete rollback is performed. +## Incoming work-item hydration failures + +If downloading an incoming work item's payload fails, the worker abandons only +that delivery's completion token and continues reading the same work-item stream. +This includes missing blobs, authorization failures, and transient I/O errors. +The failed delivery is not dispatched, even if some fields were already hydrated, +and remains retryable; this does not introduce terminal poison-payload settlement. + +An abandonment response of `NotFound` means ownership was already lost and does +not disconnect the stream. Other abandonment failures retain connection-level +recovery after the worker's existing retry policy. Worker shutdown cancellation +is unchanged. Unary responses and streamed history still propagate hydration +errors to their existing callers. + ## SDK and dashboard boundaries The client and worker gRPC interceptors resolve orchestration inputs, external diff --git a/src/Grpc/Grpc.csproj b/src/Grpc/Grpc.csproj index f2e1ef3e2..4798b8bac 100644 --- a/src/Grpc/Grpc.csproj +++ b/src/Grpc/Grpc.csproj @@ -24,6 +24,7 @@ + diff --git a/src/Grpc/WorkItemPayloadException.cs b/src/Grpc/WorkItemPayloadException.cs index 301099e2e..6386098a6 100644 --- a/src/Grpc/WorkItemPayloadException.cs +++ b/src/Grpc/WorkItemPayloadException.cs @@ -1,41 +1,20 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Grpc.Core; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask; /// -/// A payload resolution failure after a successful transport read. +/// A received delivery whose payloads could not be resolved. The item is only safe to abandon, not dispatch. /// -/// The payload resolution failure. -/// Safe failure details when the payload is confirmed missing. -internal class PayloadResolutionException(Exception innerException, P.TaskFailureDetails? permanentFailure) - : Exception("Failed to resolve external payloads.", innerException) -{ - /// Gets failure details for a confirmed permanent failure, or null for a retryable failure. - public P.TaskFailureDetails? PermanentFailure { get; } = permanentFailure; -} - -/// Identifies a failed delivery that must not be dispatched. /// The failed delivery, including its original completion token. /// The payload resolution failure. -/// Safe failure details when the payload is confirmed missing. -internal sealed class WorkItemPayloadException( - P.WorkItem workItem, Exception innerException, P.TaskFailureDetails? permanentFailure) - : PayloadResolutionException(innerException, permanentFailure) +internal sealed class WorkItemPayloadException(P.WorkItem workItem, Exception innerException) + : Exception("Failed to resolve payloads for a work-item delivery.", innerException) { /// /// Gets the failed delivery. Its payload fields may be partially resolved. /// public P.WorkItem WorkItem { get; } = workItem; } - -/// Exposes transport-only reads for recovery that needs history metadata, not payload values. -/// The response type. -internal interface IPayloadStreamReader -{ - /// Gets the underlying reader without payload resolution. - IAsyncStreamReader Inner { get; } -} diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 5aa77fe4a..0052fd920 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -536,97 +536,14 @@ void RunBackgroundTask(P.WorkItem? workItem, Func handler, CancellationTok { await handler(); } - catch (PayloadResolutionException ex) when (workItem is not null) - { - await this.OnWorkItemPayloadFailureAsync( - new WorkItemPayloadException(workItem, ex.InnerException!, ex.PermanentFailure), cancellation); - } catch (OperationCanceledException) { // Shutting down - ignore } catch (Exception ex) { - string instanceId = - workItem?.OrchestratorRequest?.InstanceId ?? - workItem?.ActivityRequest?.OrchestrationInstance?.InstanceId ?? - workItem?.EntityRequest?.InstanceId ?? - workItem?.EntityRequestV2?.InstanceId ?? - string.Empty; - this.Logger.UnexpectedError(ex, instanceId); - - if (workItem?.OrchestratorRequest != null) - { - this.Logger.AbandoningOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskOrchestratorWorkItemAsync( - new P.AbandonOrchestrationTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskOrchestratorWorkItemAsync), - cancellation); - this.Logger.AbandonedOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.ActivityRequest != null) - { - this.Logger.AbandoningActivityWorkItem( - instanceId, - workItem.ActivityRequest.Name, - workItem.ActivityRequest.TaskId, - workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskActivityWorkItemAsync( - new P.AbandonActivityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskActivityWorkItemAsync), - cancellation); - this.Logger.AbandonedActivityWorkItem( - instanceId, - workItem.ActivityRequest.Name, - workItem.ActivityRequest.TaskId, - workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.EntityRequest != null) - { - this.Logger.AbandoningEntityWorkItem( - workItem.EntityRequest.InstanceId, - workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskEntityWorkItemAsync( - new P.AbandonEntityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskEntityWorkItemAsync), - cancellation); - this.Logger.AbandonedEntityWorkItem( - workItem.EntityRequest.InstanceId, - workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.EntityRequestV2 != null) - { - this.Logger.AbandoningEntityWorkItem( - workItem.EntityRequestV2.InstanceId, - workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskEntityWorkItemAsync( - new P.AbandonEntityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskEntityWorkItemAsync), - cancellation); - this.Logger.AbandonedEntityWorkItem( - workItem.EntityRequestV2.InstanceId, - workItem.CompletionToken ?? string.Empty); - } + this.Logger.UnexpectedError(ex, GetWorkItemInstanceId(workItem)); + await this.AbandonWorkItemAsync(workItem, cancellation); } }); } @@ -635,33 +552,9 @@ async Task OnWorkItemPayloadFailureAsync(WorkItemPayloadException exception, Can { P.WorkItem workItem = exception.WorkItem; this.Logger.UnexpectedError(exception, GetWorkItemInstanceId(workItem)); - if (exception.PermanentFailure is { } failure - && await this.TryCompletePayloadFailureAsync(workItem, failure, cancellation)) - { - return; - } - try { - await this.ExecuteWithRetryAsync( - () => workItem.RequestCase switch - { - P.WorkItem.RequestOneofCase.OrchestratorRequest => - this.client.AbandonTaskOrchestratorWorkItemAsync( - new P.AbandonOrchestrationTaskRequest { CompletionToken = workItem.CompletionToken }, - cancellationToken: cancellation).ResponseAsync, - P.WorkItem.RequestOneofCase.ActivityRequest => - this.client.AbandonTaskActivityWorkItemAsync( - new P.AbandonActivityTaskRequest { CompletionToken = workItem.CompletionToken }, - cancellationToken: cancellation).ResponseAsync, - P.WorkItem.RequestOneofCase.EntityRequest or P.WorkItem.RequestOneofCase.EntityRequestV2 => - this.client.AbandonTaskEntityWorkItemAsync( - new P.AbandonEntityTaskRequest { CompletionToken = workItem.CompletionToken }, - cancellationToken: cancellation).ResponseAsync, - _ => throw new InvalidOperationException($"Cannot abandon unresolved {workItem.RequestCase}."), - }, - "AbandonUnresolvedWorkItem", - cancellation); + await this.AbandonWorkItemAsync(workItem, cancellation); } catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) { @@ -670,179 +563,58 @@ await this.ExecuteWithRetryAsync( } } - async Task TryCompletePayloadFailureAsync( - P.WorkItem workItem, P.TaskFailureDetails failure, CancellationToken cancellation) + async Task AbandonWorkItemAsync(P.WorkItem? workItem, CancellationToken cancellation) { - bool completing = false; - try + string instanceId = GetWorkItemInstanceId(workItem); + if (workItem?.OrchestratorRequest != null) { - if (workItem.ActivityRequest is { } activity) - { - if (activity.OrchestrationInstance is not { InstanceId.Length: > 0 } instance) - { - throw new InvalidOperationException("The activity has no owning orchestration instance."); - } - - completing = true; - await this.ExecuteWithRetryAsync( - async () => await this.client.CompleteActivityTaskAsync( - new P.ActivityResponse - { - InstanceId = instance.InstanceId, - TaskId = activity.TaskId, - CompletionToken = workItem.CompletionToken, - FailureDetails = failure, - }, - cancellationToken: cancellation), - nameof(this.client.CompleteActivityTaskAsync), - cancellation); - return true; - } - - if (workItem.OrchestratorRequest is not { } request) - { - // A missing entity state cannot be replaced without losing state or operation ownership. - return false; - } - - IReadOnlyList pastEvents = request.PastEvents; - if (request.RequiresHistoryStreaming) - { - using AsyncServerStreamingCall history = this.client.StreamInstanceHistory( - new P.StreamInstanceHistoryRequest - { - InstanceId = request.InstanceId, - ExecutionId = request.ExecutionId, - ForWorkItemProcessing = true, - }, - cancellationToken: cancellation); - if (history.ResponseStream is not IPayloadStreamReader payloadReader) - { - throw new InvalidOperationException("Unresolved history is unavailable for payload failure recovery."); - } - - List events = []; - await foreach (P.HistoryChunk chunk in payloadReader.Inner.ReadAllAsync(cancellation)) - { - events.AddRange(chunk.Events); - } - - pastEvents = events; - } - - ProtoUtils.EntityConversionState entities = new(insertMissingEntityUnlocks: true); - entities.SetOrchestrationInstance(new DTCore.OrchestrationInstance - { - InstanceId = request.InstanceId, ExecutionId = request.ExecutionId, - }); - bool foundStart = false; - foreach (P.HistoryEvent historyEvent in pastEvents.Concat(request.NewEvents)) - { - if (historyEvent.ExecutionStarted is { } start) - { - if (foundStart || string.IsNullOrEmpty(request.InstanceId) || string.IsNullOrEmpty(request.ExecutionId) - || start.OrchestrationInstance?.InstanceId != request.InstanceId - || start.OrchestrationInstance.ExecutionId != request.ExecutionId) - { - throw new InvalidOperationException("History does not identify the delivered execution."); - } - - foundStart = true; - } - else if (historyEvent.ExecutionCompleted is not null || historyEvent.ContinueAsNew is not null) - { - throw new InvalidOperationException("History already contains a terminal execution event."); - } - - P.SendEntityMessageAction? entityMessage = null; - if (historyEvent.EventSent is { } sent && DTCore.Common.Entities.IsEntityInstance(sent.InstanceId)) - { - entityMessage = new(); - EntityConversions.DecodeEntityMessageAction(sent.Name, sent.Input, sent.InstanceId, entityMessage, out _); - } - - if ((historyEvent.EntityLockRequested ?? entityMessage?.EntityLockRequested) is { } acquired) - { - if (string.IsNullOrEmpty(acquired.CriticalSectionId) || acquired.LockSet.Any(string.IsNullOrEmpty)) - { - throw new InvalidOperationException("History contains incomplete entity lock metadata."); - } - - entities.AddUnlockObligations(acquired); - } - else if ((historyEvent.EntityUnlockSent ?? entityMessage?.EntityUnlockSent) is { } released) - { - entities.RemoveUnlockObligation(released.TargetInstanceId); - } - } - - if (!foundStart) - { - throw new InvalidOperationException("History has no execution start for payload failure recovery."); - } - - P.OrchestratorResponse response = new() - { - InstanceId = request.InstanceId, - CompletionToken = workItem.CompletionToken, - NumEventsProcessed = request.NewEvents.Count, - OrchestrationTraceContext = request.OrchestrationTraceContext, - Actions = - { - new P.OrchestratorAction + this.Logger.AbandoningOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskOrchestratorWorkItemAsync( + new P.AbandonOrchestrationTaskRequest { - CompleteOrchestration = new P.CompleteOrchestrationAction - { - OrchestrationStatus = P.OrchestrationStatus.Failed, - FailureDetails = failure, - }, + CompletionToken = workItem.CompletionToken, }, - }, - }; - await this.PrepareOrchestratorResponseAsync(response, entities, cancellation); - if (!this.internalOptions.ConvertOrchestrationEntityEvents) - { - foreach (P.OrchestratorAction action in response.Actions) - { - if (action.SendEntityMessage?.EntityUnlockSent is { } unlock) - { - if (EntityConversions.EncodeUnlockSent( - new P.HistoryEvent { EntityUnlockSent = unlock }, entities.CurrentInstance) is not EventSentEvent sent) - { - throw new InvalidOperationException("Entity release has no target."); - } - - action.SendEvent = new P.SendEventAction - { - Instance = new P.OrchestrationInstance { InstanceId = sent.InstanceId }, - Name = sent.Name, Data = sent.Input, - }; - } - } - } - - if (response.CalculateSize() > MaxInlineOrchestratorResponseSizeInBytes) - { - throw new InvalidOperationException("Payload failure response exceeds the inline gRPC limit."); - } - - completing = true; - await this.ExecuteWithRetryAsync( - async () => await this.client.CompleteOrchestratorTaskAsync(response, cancellationToken: cancellation), - nameof(this.client.CompleteOrchestratorTaskAsync), + cancellationToken: cancellation), + nameof(this.client.AbandonTaskOrchestratorWorkItemAsync), cancellation); - return true; + this.Logger.AbandonedOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); } - catch (RpcException ex) when (completing && ex.StatusCode == StatusCode.NotFound) + else if (workItem?.ActivityRequest != null) { - this.Logger.WorkItemOwnershipLost(workItem.RequestCase.ToString(), workItem.CompletionToken, ex); - return true; + this.Logger.AbandoningActivityWorkItem( + instanceId, + workItem.ActivityRequest.Name, + workItem.ActivityRequest.TaskId, + workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskActivityWorkItemAsync( + new P.AbandonActivityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskActivityWorkItemAsync), + cancellation); + this.Logger.AbandonedActivityWorkItem( + instanceId, + workItem.ActivityRequest.Name, + workItem.ActivityRequest.TaskId, + workItem.CompletionToken ?? string.Empty); } - catch (Exception ex) when (!completing && ex is InvalidOperationException or Newtonsoft.Json.JsonException) + else if (workItem?.EntityRequest != null || workItem?.EntityRequestV2 != null) { - // Do not terminally settle an execution when its routing/lock metadata cannot be reconstructed. - this.Logger.UnexpectedError(ex, GetWorkItemInstanceId(workItem)); - return false; + this.Logger.AbandoningEntityWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskEntityWorkItemAsync), + cancellation); + this.Logger.AbandonedEntityWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); } } diff --git a/src/Worker/Grpc/Worker.Grpc.csproj b/src/Worker/Grpc/Worker.Grpc.csproj index 2b5d41138..0fa7b856b 100644 --- a/src/Worker/Grpc/Worker.Grpc.csproj +++ b/src/Worker/Grpc/Worker.Grpc.csproj @@ -22,6 +22,7 @@ + diff --git a/test/Extensions/AzureBlobPayloads.Tests/OrchestratorPayloadFailureTests.cs b/test/Extensions/AzureBlobPayloads.Tests/OrchestratorPayloadFailureTests.cs index 7ece5d4ae..94b86ab34 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/OrchestratorPayloadFailureTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/OrchestratorPayloadFailureTests.cs @@ -27,7 +27,6 @@ enum PayloadFailure SizeLimit, ForbiddenUpload, InlineSizeLimit, - MissingDownload, } [Theory] @@ -63,18 +62,6 @@ public Task SuccessfulCompletion_ReleasesEntitiesExactlyOnceBeforeCompletion( useExternalizedPayloads: useExternalizedPayloads, explicitlyReleaseLocks: explicitlyReleaseLocks); - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public Task MissingPayloadDownload_ReleasesEntitiesBeforeFailingAsync(bool streamHistory, bool historyPayload) - => RunLockedOrchestrationAsync( - completeOrchestration: true, - streamHistory, - PayloadFailure.MissingDownload, - historyPayload: historyPayload); - [Theory] [InlineData(false, false, false)] [InlineData(false, false, true)] @@ -100,8 +87,7 @@ static async Task RunLockedOrchestrationAsync( bool streamHistory, PayloadFailure failure, bool useExternalizedPayloads = true, - bool explicitlyReleaseLocks = true, - bool historyPayload = false) + bool explicitlyReleaseLocks = true) { // Arrange const string InstanceId = "locked-payload-instance"; @@ -116,7 +102,7 @@ static async Task RunLockedOrchestrationAsync( }; string serializedPayload = JsonSerializer.Serialize(payload); Channel workItems = Channel.CreateUnbounded(); - P.HistoryChunk? streamedHistory = null; + Channel history = Channel.CreateUnbounded(); Channel completions = Channel.CreateUnbounded(); Mock invoker = new(MockBehavior.Strict); invoker.Setup(value => value.AsyncUnaryCall( @@ -141,18 +127,12 @@ static async Task RunLockedOrchestrationAsync( It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(() => - { - Channel replay = Channel.CreateUnbounded(); - replay.Writer.TryWrite(Assert.IsType(streamedHistory).Clone()); - replay.Writer.Complete(); - return new AsyncServerStreamingCall( - new ChannelStreamReader(replay.Reader), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { }); - }); + .Returns(() => new AsyncServerStreamingCall( + new ChannelStreamReader(history.Reader), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { })); invoker.Setup(value => value.AsyncUnaryCall( It.IsAny>(), It.IsAny(), @@ -162,10 +142,7 @@ static async Task RunLockedOrchestrationAsync( (_, _, _, response) => completions.Writer.TryWrite(response.Clone())) .Returns(() => CreateUnaryCall(new P.CompleteTaskResponse())); Mock store = new(MockBehavior.Strict); - store.Setup(value => value.IsKnownPayloadToken(It.IsAny())) - .Returns((string value) => value == "blob:v1:missing"); - store.Setup(value => value.DownloadAsync("blob:v1:missing", It.IsAny())) - .ThrowsAsync(new PayloadStorageException("Payload missing.", new RequestFailedException(404, "Blob not found."))); + store.Setup(value => value.IsKnownPayloadToken(It.IsAny())).Returns(false); if (failure == PayloadFailure.ForbiddenUpload) { store.Setup(value => value.UploadAsync(serializedPayload, It.IsAny())) @@ -303,24 +280,11 @@ await workItems.Writer.WriteAsync(new P.WorkItem }, }, }; - if (failure == PayloadFailure.MissingDownload) - { - if (historyPayload) - { - resumedRequest.PastEvents.Single(e => e.ExecutionStarted is not null).ExecutionStarted.Input = "blob:v1:missing"; - } - else - { - resumedRequest.NewEvents.Add(new P.HistoryEvent - { - EventRaised = new P.EventRaisedEvent { Name = "missing", Input = "blob:v1:missing" }, - }); - } - } - if (streamHistory) { - streamedHistory = new P.HistoryChunk { Events = { resumedRequest.PastEvents } }; + await history.Writer.WriteAsync( + new P.HistoryChunk { Events = { resumedRequest.PastEvents } }, timeout.Token); + history.Writer.Complete(); resumedRequest.PastEvents.Clear(); resumedRequest.RequiresHistoryStreaming = true; } @@ -381,10 +345,6 @@ await workItems.Writer.WriteAsync(new P.WorkItem store.Verify( value => value.UploadAsync(It.IsAny(), It.IsAny()), failure == PayloadFailure.ForbiddenUpload ? Times.Once() : Times.Never()); - if (failure == PayloadFailure.MissingDownload) - { - store.Verify(value => value.DownloadAsync("blob:v1:missing", It.IsAny()), Times.Once); - } } finally { diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index 4edf73b44..f29b71062 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.IO; using System.Text.Json; -using Azure; using FluentAssertions; using Grpc.Core; using Grpc.Core.Interceptors; @@ -13,7 +11,6 @@ using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; -using Moq; using Xunit.Abstractions; using P = Microsoft.DurableTask.Protobuf; @@ -21,99 +18,6 @@ namespace Microsoft.DurableTask.Grpc.Tests; public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) : IntegrationTestBase(output, sidecarFixture) { - [Theory] - [InlineData(0, "activity")] - [InlineData(403, "activity")] - [InlineData(404, "activity")] - [InlineData(0, "orchestration")] - [InlineData(403, "orchestration")] - [InlineData(404, "orchestration")] - [InlineData(404, "suborchestration")] - public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int storageStatus, string target) - { - // Arrange - const string Token = "blob:v1:missing"; - TaskName orchestration = nameof(HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync); - TaskName activity = "HydrationActivity"; - TaskName child = "HydrationChild"; - TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously); - TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); - TaskCompletionSource downloadFailed = new(TaskCreationOptions.RunContinuationsAsynchronously); - Mock store = new(); - store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value == Token); - store.Setup(s => s.UploadAsync(It.IsAny(), It.IsAny())).ReturnsAsync(Token); - store.Setup(s => s.DownloadAsync(Token, It.IsAny())).Returns(() => - { - downloadFailed.TrySetResult(); - Exception failure = storageStatus == 0 - ? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed."); - return Task.FromException(failure); - }); - await using HostTestLifetime server = await this.StartWorkerAsync(worker => - { - worker.AddTasks(tasks => tasks - .AddOrchestratorFunc(orchestration, (context, input) => - input == "bad-child" - ? context.CallSubOrchestratorAsync(child, new string('X', 1024)) - : context.CallActivityAsync(activity, input == "bad" ? new string('X', 1024) : input)) - .AddOrchestratorFunc(child, (_, input) => Task.FromResult(input)) - .AddActivityFunc(activity, async (_, input) => - { - if (input == "running") - { - running.TrySetResult(); - await release.Task.WaitAsync(this.TimeoutToken); - } - - return input; - })); - worker.UseExternalizedPayloads(); - worker.Services.AddSingleton(store.Object); - }, clientConfigure: client => client.UseExternalizedPayloads(), - servicesConfigure: services => services.AddExternalizedPayloadStore(options => - { - options.ThresholdBytes = 16; - options.ConnectionString = "UseDevelopmentStorage=true"; - })); - - try - { - // Act - string alreadyRunning = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "running"); - await running.Task.WaitAsync(this.TimeoutToken); - string failed = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, target switch - { - "orchestration" => new string('X', 1024), - "suborchestration" => "bad-child", - _ => "bad", - }); - await downloadFailed.Task.WaitAsync(this.TimeoutToken); - string following = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "following"); - OrchestrationMetadata followingResult = await server.Client.WaitForInstanceCompletionAsync( - following, getInputsAndOutputs: true, this.TimeoutToken); - release.TrySetResult(); - OrchestrationMetadata runningResult = await server.Client.WaitForInstanceCompletionAsync( - alreadyRunning, getInputsAndOutputs: true, this.TimeoutToken); - - // Assert - Assert.Equal(OrchestrationRuntimeStatus.Completed, followingResult.RuntimeStatus); - Assert.Equal("following", followingResult.ReadOutputAs()); - Assert.Equal(OrchestrationRuntimeStatus.Completed, runningResult.RuntimeStatus); - Assert.Equal("running", runningResult.ReadOutputAs()); - Assert.Single(this.GetLogs(), log => log.EventId.Name == "EstablishedWorkItemConnection"); - if (storageStatus == 404) - { - OrchestrationMetadata failure = await server.Client.WaitForInstanceCompletionAsync( - failed, getInputsAndOutputs: false, this.TimeoutToken); - Assert.Equal(OrchestrationRuntimeStatus.Failed, failure.RuntimeStatus); - } - } - finally - { - release.TrySetResult(); - } - } - [Theory] [InlineData(true)] [InlineData(false)] diff --git a/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs b/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs new file mode 100644 index 000000000..42a65e63b --- /dev/null +++ b/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs @@ -0,0 +1,536 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Concurrent; +using System.IO; +using System.Net; +using System.Text.Json; +using System.Threading.Channels; +using Azure; +using Google.Protobuf.WellKnownTypes; +using Grpc.Core; +using Grpc.Core.Interceptors; +using Grpc.Net.Client; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.AzureManaged; +using Microsoft.DurableTask.Worker.Grpc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Moq; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Grpc.Tests; + +public class PayloadHydrationFailureTests +{ + const string TokenPrefix = "test-blob://"; + const string BadToken = TokenPrefix + "bad"; + const string FailedCompletionToken = "failed-delivery"; + static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(15); + + public static IEnumerable HydrationFailures() + { + P.WorkItem.RequestOneofCase[] kinds = + [ + P.WorkItem.RequestOneofCase.ActivityRequest, + P.WorkItem.RequestOneofCase.OrchestratorRequest, + P.WorkItem.RequestOneofCase.EntityRequest, + P.WorkItem.RequestOneofCase.EntityRequestV2, + ]; + + foreach (P.WorkItem.RequestOneofCase kind in kinds) + { + foreach (int storageStatus in new[] { 404, 403, 0 }) + { + foreach (bool ownershipLost in new[] { false, true }) + { + foreach (bool useAzureManaged in new[] { false, true }) + { + yield return [kind, storageStatus, ownershipLost, useAzureManaged]; + } + } + } + } + } + + [Theory] + [MemberData(nameof(HydrationFailures))] + public async Task HydrationFailure_AbandonsOnlyFailedDeliveryAndKeepsStreamAsync( + P.WorkItem.RequestOneofCase kind, int storageStatus, bool ownershipLost, bool useAzureManaged) + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + sidecar.AbandonStatus = ownershipLost ? StatusCode.NotFound : StatusCode.OK; + Exception failure = storageStatus == 0 + ? new IOException("Transient payload download failure.") + : new RequestFailedException(storageStatus, "Payload download failed."); + Mock store = CreatePayloadStore(failure); + TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + ConcurrentQueue inputs = new(); + await using ServiceProvider provider = CreateWorker(sidecar, store.Object, async input => + { + inputs.Enqueue(input); + if (input == "running") + { + running.TrySetResult(); + await release.Task.WaitAsync(timeout.Token); + } + + return input; + }, useAzureManaged); + IHostedService worker = Assert.Single(provider.GetServices()); + await worker.StartAsync(timeout.Token); + + try + { + // Act + sidecar.Send(CreateHealthyActivity("running")); + await running.Task.WaitAsync(timeout.Token); + sidecar.Send(CreateFailedWorkItem(kind)); + sidecar.Send(new P.WorkItem { HealthPing = new P.HealthPing() }); + sidecar.Send(CreateHealthyActivity("following")); + P.ActivityResponse following = await sidecar.Completions.Reader.ReadAsync(timeout.Token); + release.TrySetResult(); + P.ActivityResponse alreadyRunning = await sidecar.Completions.Reader.ReadAsync(timeout.Token); + + // Assert + Assert.Equal(1, sidecar.StreamCount); + Assert.Equal(0, sidecar.ClosedStreamCount); + Assert.Equal(1, sidecar.HelloCount); + P.WorkItem.RequestOneofCase abandonKind = kind == P.WorkItem.RequestOneofCase.EntityRequestV2 + ? P.WorkItem.RequestOneofCase.EntityRequest + : kind; + Assert.Equal((abandonKind, FailedCompletionToken), Assert.Single(sidecar.Abandons)); + Assert.Equal(new[] { "running", "following" }, inputs.ToArray()); + Assert.Equal("following", following.CompletionToken); + Assert.Equal("\"following\"", following.Result); + Assert.Null(following.FailureDetails); + Assert.Equal("running", alreadyRunning.CompletionToken); + Assert.Equal("\"running\"", alreadyRunning.Result); + Assert.Null(alreadyRunning.FailureDetails); + store.Verify(s => s.DownloadAsync(BadToken, It.IsAny()), Times.Once); + } + finally + { + release.TrySetResult(); + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + } + } + + [Theory] + [InlineData(StatusCode.Unknown, false)] + [InlineData(StatusCode.Unavailable, false)] + [InlineData(StatusCode.PermissionDenied, false)] + [InlineData(StatusCode.Internal, false)] + [InlineData(StatusCode.Unavailable, true)] + public async Task TransportOrAbandonFailure_ReconnectsAsync(StatusCode status, bool streamFailure) + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + sidecar.AbandonStatus = status; + Mock store = CreatePayloadStore(new IOException("Payload download failed.")); + await using ServiceProvider provider = CreateWorker(sidecar, store.Object); + IHostedService worker = Assert.Single(provider.GetServices()); + await worker.StartAsync(timeout.Token); + + try + { + Assert.Equal(1, await sidecar.Streams.Reader.ReadAsync(timeout.Token)); + + // Act + if (streamFailure) + { + sidecar.FailStream(status); + } + else + { + sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); + } + + Assert.Equal(2, await sidecar.Streams.Reader.ReadAsync(timeout.Token)); + sidecar.Send(CreateHealthyActivity("following")); + P.ActivityResponse response = await sidecar.Completions.Reader.ReadAsync(timeout.Token); + + // Assert + Assert.Equal(2, sidecar.StreamCount); + Assert.Equal(2, sidecar.HelloCount); + Assert.Equal("following", response.CompletionToken); + Assert.Null(response.FailureDetails); + Assert.Equal(streamFailure ? 0 : 1, sidecar.Abandons.Count); + } + finally + { + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + } + } + + [Fact] + public async Task HydrationFailure_RedeliveryCompletesAfterPayloadRecoversAsync() + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + Mock store = CreatePayloadStore(new IOException("Payload download failed.")); + store.SetupSequence(s => s.DownloadAsync(BadToken, It.IsAny())) + .ThrowsAsync(new RequestFailedException(404, "Payload not found.")) + .ReturnsAsync("\"recovered\""); + await using ServiceProvider provider = CreateWorker(sidecar, store.Object); + IHostedService worker = Assert.Single(provider.GetServices()); + await worker.StartAsync(timeout.Token); + + try + { + // Act + sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); + await sidecar.AbandonStarted.Task.WaitAsync(timeout.Token); + P.WorkItem redelivery = CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest); + redelivery.CompletionToken = "redelivery"; + sidecar.Send(redelivery); + P.ActivityResponse response = await sidecar.Completions.Reader.ReadAsync(timeout.Token); + + // Assert + Assert.Equal(1, sidecar.StreamCount); + Assert.Equal((P.WorkItem.RequestOneofCase.ActivityRequest, FailedCompletionToken), Assert.Single(sidecar.Abandons)); + Assert.Equal("redelivery", response.CompletionToken); + Assert.Equal("\"recovered\"", response.Result); + Assert.Null(response.FailureDetails); + } + finally + { + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ShutdownDuringHydrationOrAbandon_DoesNotReconnectAsync(bool duringAbandon) + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + sidecar.HoldAbandon = duringAbandon; + TaskCompletionSource downloading = new(TaskCreationOptions.RunContinuationsAsynchronously); + Mock store = CreatePayloadStore(new IOException("Payload download failed.")); + if (!duringAbandon) + { + store.Setup(s => s.DownloadAsync(BadToken, It.IsAny())) + .Returns(async (string _, CancellationToken cancellation) => + { + downloading.TrySetResult(cancellation); + await Task.Delay(Timeout.Infinite, cancellation); + return string.Empty; + }); + } + + await using ServiceProvider provider = CreateWorker(sidecar, store.Object); + IHostedService worker = Assert.Single(provider.GetServices()); + await worker.StartAsync(timeout.Token); + + try + { + sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); + CancellationToken downloadCancellation = default; + if (duringAbandon) + { + await sidecar.AbandonStarted.Task.WaitAsync(timeout.Token); + } + else + { + downloadCancellation = await downloading.Task.WaitAsync(timeout.Token); + } + + // Act + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + + // Assert + Assert.Equal(1, sidecar.StreamCount); + Assert.Equal(duringAbandon ? 1 : 0, sidecar.Abandons.Count); + Assert.False(sidecar.Completions.Reader.TryRead(out _)); + if (!duringAbandon) + { + Assert.True(downloadCancellation.IsCancellationRequested); + } + } + finally + { + await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NonWorkItemHydrationFailure_PropagatesOriginalExceptionAsync(bool historyStream) + { + // Arrange + using CancellationTokenSource timeout = new(TestTimeout); + await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); + RequestFailedException failure = new(403, "Payload download forbidden."); + Mock store = CreatePayloadStore(failure); + using GrpcChannel channel = GrpcChannel.ForAddress(sidecar.Address); + P.TaskHubSidecarService.TaskHubSidecarServiceClient client = new(channel.Intercept( + new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions()))); + + // Act + Exception? observed; + if (historyStream) + { + using AsyncServerStreamingCall call = client.StreamInstanceHistory( + new P.StreamInstanceHistoryRequest { InstanceId = "instance" }, cancellationToken: timeout.Token); + observed = await Record.ExceptionAsync(() => call.ResponseStream.MoveNext(timeout.Token)); + } + else + { + using AsyncUnaryCall call = client.GetInstanceAsync( + new P.GetInstanceRequest { InstanceId = "instance" }, cancellationToken: timeout.Token); + observed = await Record.ExceptionAsync(async () => await call.ResponseAsync); + } + + // Assert + Assert.Same(failure, observed); + Assert.Empty(sidecar.Abandons); + } + + static Mock CreatePayloadStore(Exception failure) + { + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())) + .Returns((string value) => value.StartsWith(TokenPrefix, StringComparison.Ordinal)); + store.Setup(s => s.DownloadAsync(It.IsAny(), It.IsAny())) + .Returns((string token, CancellationToken _) => token == BadToken + ? Task.FromException(failure) + : Task.FromResult(JsonSerializer.Serialize(token[TokenPrefix.Length..]))); + return store; + } + + static ServiceProvider CreateWorker( + ScriptedSidecar sidecar, + PayloadStore store, + Func>? runActivity = null, + bool useAzureManaged = false) + { + ServiceCollection services = new(); + services.AddLogging(logging => logging.ClearProviders()); + services.AddSingleton(store); + services.AddDurableTaskWorker(builder => + { + if (useAzureManaged) + { + builder.UseDurableTaskScheduler($"Endpoint={sidecar.Address};TaskHub=test;Authentication=None"); + } + else + { + builder.UseGrpc(sidecar.Address); + } + + builder.Services.Configure(builder.Name, options => + { + options.Internal.TransientRetryMaxAttempts = 1; + options.Internal.ReconnectBackoffBase = TimeSpan.Zero; + options.Internal.ReconnectBackoffCap = TimeSpan.Zero; + }); + builder.UseExternalizedPayloads(); + builder.AddTasks(tasks => tasks.AddActivityFunc( + "Healthy", (_, input) => runActivity?.Invoke(input!) ?? Task.FromResult(input!))); + }); + return services.BuildServiceProvider(); + } + + static P.WorkItem CreateHealthyActivity(string name) => new() + { + CompletionToken = name, + ActivityRequest = new P.ActivityRequest + { + Name = "Healthy", + TaskId = 1, + Input = TokenPrefix + name, + OrchestrationInstance = new P.OrchestrationInstance { InstanceId = name, ExecutionId = "execution" }, + }, + }; + + static P.WorkItem CreateFailedWorkItem(P.WorkItem.RequestOneofCase kind) + { + P.WorkItem item = new() { CompletionToken = FailedCompletionToken }; + switch (kind) + { + case P.WorkItem.RequestOneofCase.ActivityRequest: + item.ActivityRequest = CreateHealthyActivity("failed").ActivityRequest; + item.ActivityRequest.Input = BadToken; + break; + case P.WorkItem.RequestOneofCase.OrchestratorRequest: + item.OrchestratorRequest = new P.OrchestratorRequest + { + InstanceId = "failed", + PastEvents = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = TokenPrefix + "state" } } }, + NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Name = "event", Input = BadToken } } }, + }; + break; + case P.WorkItem.RequestOneofCase.EntityRequest: + item.EntityRequest = new P.EntityBatchRequest + { + InstanceId = "@entity@failed", + EntityState = TokenPrefix + "state", + Operations = { new P.OperationRequest { Operation = "operation", Input = BadToken } }, + }; + break; + case P.WorkItem.RequestOneofCase.EntityRequestV2: + item.EntityRequestV2 = new P.EntityRequest + { + InstanceId = "@entity@failed", + EntityState = TokenPrefix + "state", + OperationRequests = { new P.HistoryEvent { EntityOperationSignaled = new P.EntityOperationSignaledEvent { Input = BadToken } } }, + }; + break; + default: + throw new ArgumentOutOfRangeException(nameof(kind), kind, "Unsupported work item kind."); + } + + return item; + } + + sealed class ScriptedSidecar : P.TaskHubSidecarService.TaskHubSidecarServiceBase, IAsyncDisposable + { + readonly Channel, Task>> deliveries = + Channel.CreateUnbounded, Task>>(); + IHost host = null!; + int streamCount; + int closedStreamCount; + int helloCount; + + public string Address { get; private set; } = string.Empty; + + public StatusCode AbandonStatus { get; set; } + + public bool HoldAbandon { get; set; } + + public int StreamCount => Volatile.Read(ref this.streamCount); + + public int ClosedStreamCount => Volatile.Read(ref this.closedStreamCount); + + public int HelloCount => Volatile.Read(ref this.helloCount); + + public Channel Streams { get; } = Channel.CreateUnbounded(); + + public Channel Completions { get; } = Channel.CreateUnbounded(); + + public ConcurrentQueue<(P.WorkItem.RequestOneofCase Kind, string Token)> Abandons { get; } = new(); + + public TaskCompletionSource AbandonStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public static async Task StartAsync() + { + ScriptedSidecar sidecar = new(); + sidecar.host = Host.CreateDefaultBuilder() + .ConfigureLogging(logging => logging.ClearProviders()) + .ConfigureWebHostDefaults(web => web + .UseKestrel(options => options.Listen(IPAddress.Loopback, 0, endpoint => endpoint.Protocols = HttpProtocols.Http2)) + .ConfigureServices(services => + { + services.AddGrpc(); + services.AddSingleton(sidecar); + }) + .Configure(app => + { + app.UseRouting(); + app.UseEndpoints(endpoints => endpoints.MapGrpcService()); + })) + .Build(); + await sidecar.host.StartAsync().WaitAsync(TestTimeout); + sidecar.Address = Assert.Single(sidecar.host.Services.GetRequiredService() + .Features.Get()!.Addresses); + return sidecar; + } + + public void Send(P.WorkItem item) => Assert.True(this.deliveries.Writer.TryWrite(stream => stream.WriteAsync(item))); + + public void FailStream(StatusCode status) => Assert.True(this.deliveries.Writer.TryWrite( + _ => Task.FromException(new RpcException(new Status(status, "Stream transport failed."))))); + + public override Task Hello(Empty request, ServerCallContext context) + { + Interlocked.Increment(ref this.helloCount); + return Task.FromResult(new Empty()); + } + + public override async Task GetWorkItems(P.GetWorkItemsRequest request, IServerStreamWriter responseStream, ServerCallContext context) + { + await context.WriteResponseHeadersAsync(new Metadata()); + this.Streams.Writer.TryWrite(Interlocked.Increment(ref this.streamCount)); + try + { + await foreach (Func, Task> deliver in this.deliveries.Reader.ReadAllAsync(context.CancellationToken)) + { + await deliver(responseStream); + } + } + finally + { + Interlocked.Increment(ref this.closedStreamCount); + } + } + + public override Task CompleteActivityTask(P.ActivityResponse request, ServerCallContext context) + { + this.Completions.Writer.TryWrite(request); + return Task.FromResult(new P.CompleteTaskResponse()); + } + + public override async Task AbandonTaskActivityWorkItem(P.AbandonActivityTaskRequest request, ServerCallContext context) + { + await this.AbandonAsync(P.WorkItem.RequestOneofCase.ActivityRequest, request.CompletionToken, context); + return new P.AbandonActivityTaskResponse(); + } + + public override async Task AbandonTaskOrchestratorWorkItem(P.AbandonOrchestrationTaskRequest request, ServerCallContext context) + { + await this.AbandonAsync(P.WorkItem.RequestOneofCase.OrchestratorRequest, request.CompletionToken, context); + return new P.AbandonOrchestrationTaskResponse(); + } + + public override async Task AbandonTaskEntityWorkItem(P.AbandonEntityTaskRequest request, ServerCallContext context) + { + await this.AbandonAsync(P.WorkItem.RequestOneofCase.EntityRequest, request.CompletionToken, context); + return new P.AbandonEntityTaskResponse(); + } + + public override Task GetInstance(P.GetInstanceRequest request, ServerCallContext context) + => Task.FromResult(new P.GetInstanceResponse { OrchestrationState = new P.OrchestrationState { Input = BadToken } }); + + public override Task StreamInstanceHistory(P.StreamInstanceHistoryRequest request, IServerStreamWriter responseStream, ServerCallContext context) + => responseStream.WriteAsync(new P.HistoryChunk + { + Events = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = BadToken } } }, + }); + + public async ValueTask DisposeAsync() + { + using IHost host = this.host; + await host.StopAsync().WaitAsync(TestTimeout); + } + + async Task AbandonAsync(P.WorkItem.RequestOneofCase kind, string token, ServerCallContext context) + { + this.Abandons.Enqueue((kind, token)); + this.AbandonStarted.TrySetResult(); + if (this.HoldAbandon) + { + await Task.Delay(Timeout.Infinite, context.CancellationToken); + } + + if (this.AbandonStatus != StatusCode.OK) + { + throw new RpcException(new Status(this.AbandonStatus, "Abandon failed.")); + } + } + } +} diff --git a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs index 69d642140..4b3505c16 100644 --- a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs +++ b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs @@ -5,12 +5,10 @@ using System.Diagnostics; using System.IO; using System.Reflection; -using Azure; using DurableTask.Core; using DurableTask.Core.History; using Google.Protobuf.WellKnownTypes; using Grpc.Core; -using Grpc.Core.Interceptors; using Microsoft.DurableTask; using Microsoft.DurableTask.Tests.Logging; using Microsoft.DurableTask.Tracing; @@ -51,160 +49,6 @@ public class GrpcDurableTaskWorkerTests .GetNestedType("Processor", BindingFlags.NonPublic)! .GetMethod("OnRunOrchestratorAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; - [Theory] - [CombinatorialData] - public async Task ProcessorExecuteAsync_HydrationFailure_SettlesOnlyFailedDeliveryAsync( - [CombinatorialValues(P.WorkItem.RequestOneofCase.ActivityRequest, P.WorkItem.RequestOneofCase.OrchestratorRequest, - P.WorkItem.RequestOneofCase.EntityRequest, P.WorkItem.RequestOneofCase.EntityRequestV2)] P.WorkItem.RequestOneofCase kind, - [CombinatorialValues(0, 403, 404)] int storageStatus, - [CombinatorialValues(StatusCode.OK, StatusCode.NotFound, StatusCode.Unavailable, StatusCode.PermissionDenied)] StatusCode settlementStatus) - { - // Arrange - const string PayloadToken = "blob:v1:bad"; - P.WorkItem failed = new() { CompletionToken = "failed" }; - switch (kind) - { - case P.WorkItem.RequestOneofCase.ActivityRequest: - failed.ActivityRequest = new P.ActivityRequest - { - Input = PayloadToken, OrchestrationInstance = new P.OrchestrationInstance { InstanceId = "failed-instance" }, - }; - break; - case P.WorkItem.RequestOneofCase.OrchestratorRequest: - failed.OrchestratorRequest = new P.OrchestratorRequest - { - InstanceId = "failed-instance", ExecutionId = "failed-execution", - PastEvents = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent - { - OrchestrationInstance = new P.OrchestrationInstance { InstanceId = "failed-instance", ExecutionId = "failed-execution" }, - } } }, - NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Name = "input", Input = PayloadToken } } }, - }; - break; - case P.WorkItem.RequestOneofCase.EntityRequest: - failed.EntityRequest = new P.EntityBatchRequest { EntityState = PayloadToken }; - break; - case P.WorkItem.RequestOneofCase.EntityRequestV2: - failed.EntityRequestV2 = new P.EntityRequest { EntityState = PayloadToken }; - break; - } - - if (failed.OrchestratorRequest is { } orchestration) - { - EventSentEvent acquire = Assert.IsType(EntityConversions.EncodeLockRequested( - new P.HistoryEvent - { - EntityLockRequested = new P.EntityLockRequestedEvent - { - CriticalSectionId = "00000000-0000-0000-0000-000000000001", - LockSet = { "@counter@one" }, - ParentInstanceId = orchestration.InstanceId, - }, - }, - new OrchestrationInstance { InstanceId = orchestration.InstanceId, ExecutionId = orchestration.ExecutionId })); - orchestration.PastEvents.Add(new P.HistoryEvent - { - EventSent = new P.EventSentEvent { InstanceId = acquire.InstanceId, Name = acquire.Name, Input = acquire.Input }, - }); - } - - Queue deliveries = new([failed, new P.WorkItem { HealthPing = new P.HealthPing() }]); - TaskCompletionSource pingDispatched = new(TaskCreationOptions.RunContinuationsAsynchronously); - P.WorkItem current = null!; - Mock> reader = new(); - reader.SetupGet(r => r.Current).Returns(() => current); - reader.Setup(r => r.MoveNext(It.IsAny())).Returns(async (CancellationToken cancellation) => - { - if (deliveries.TryDequeue(out P.WorkItem? item)) - { - current = item; - return true; - } - - pingDispatched.TrySetResult(); - await Task.Delay(Timeout.Infinite, cancellation); - return false; - }); - Mock store = new(); - store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value == PayloadToken); - Exception failure = storageStatus == 0 ? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed."); - store.Setup(s => s.DownloadAsync(PayloadToken, It.IsAny())).ThrowsAsync(failure); - bool terminal = storageStatus == 404 && kind is P.WorkItem.RequestOneofCase.ActivityRequest or P.WorkItem.RequestOneofCase.OrchestratorRequest; - string expectedOperation = terminal - ? kind == P.WorkItem.RequestOneofCase.ActivityRequest ? "CompleteActivityTask" : "CompleteOrchestratorTask" - : kind switch - { - P.WorkItem.RequestOneofCase.ActivityRequest => "AbandonTaskActivityWorkItem", - P.WorkItem.RequestOneofCase.OrchestratorRequest => "AbandonTaskOrchestratorWorkItem", - _ => "AbandonTaskEntityWorkItem", - }; - List settlements = []; - ScriptedWorkerCallInvoker invoker = new( - (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), - (_, _) => CreateServerStreamingCall(reader.Object)) - { - UnaryResponse = (operation, request) => - { - Assert.Equal(expectedOperation, operation); - settlements.Add(request); - (string Token, object Reply) result = request switch - { - P.AbandonActivityTaskRequest r => (r.CompletionToken, new P.AbandonActivityTaskResponse()), - P.AbandonOrchestrationTaskRequest r => (r.CompletionToken, new P.AbandonOrchestrationTaskResponse()), - P.AbandonEntityTaskRequest r => (r.CompletionToken, new P.AbandonEntityTaskResponse()), - P.ActivityResponse r => (r.CompletionToken, new P.CompleteTaskResponse()), - P.OrchestratorResponse r => (r.CompletionToken, new P.CompleteTaskResponse()), - _ => throw new InvalidOperationException($"Unexpected request {request.GetType()}."), - }; - Assert.Equal("failed", result.Token); - if (settlementStatus != StatusCode.OK) - { - throw new RpcException(new Status(settlementStatus, "Settlement failed.")); - } - - return result.Reply; - }, - }; - GrpcDurableTaskWorkerOptions options = new(); - options.Internal.TransientRetryMaxAttempts = 1; - options.Internal.ReconnectBackoffBase = options.Internal.ReconnectBackoffCap = TimeSpan.Zero; - object processor = CreateProcessor(CreateWorker(options), new P.TaskHubSidecarService.TaskHubSidecarServiceClient( - invoker.Intercept(new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions())))); - using CancellationTokenSource shutdown = new(); - - // Act - Task processing = InvokeProcessorExecuteAsync(processor, shutdown.Token); - try - { - await pingDispatched.Task.WaitAsync(TimeSpan.FromSeconds(5)); - } - finally - { - shutdown.Cancel(); - await processing; - } - - // Assert - object settlement = Assert.Single(settlements); - if (settlement is P.ActivityResponse activityFailure) - { - Assert.True(activityFailure.FailureDetails.IsNonRetriable); - } - else if (settlement is P.OrchestratorResponse orchestrationFailure) - { - P.OrchestratorAction release = Assert.Single(orchestrationFailure.Actions, a => a.SendEvent is not null); - Assert.Equal("@counter@one", release.SendEvent.Instance.InstanceId); - Assert.Equal("release", release.SendEvent.Name); - P.CompleteOrchestrationAction action = Assert.Single(orchestrationFailure.Actions, a => a.CompleteOrchestration is not null).CompleteOrchestration; - Assert.Equal(P.OrchestrationStatus.Failed, action.OrchestrationStatus); - Assert.True(action.FailureDetails.IsNonRetriable); - } - - Assert.Equal(settlementStatus is StatusCode.OK or StatusCode.NotFound ? 1 : 2, invoker.GetWorkItemsCallCount); - store.Verify(s => s.DownloadAsync(PayloadToken, It.IsAny()), Times.Once); - - } - [Fact] public async Task ExecuteAsync_ConnectFailureThreshold_RecreatesConfiguredChannel() { @@ -2355,8 +2199,6 @@ public ScriptedWorkerCallInvoker( public int GetWorkItemsCallCount => Volatile.Read(ref this.getWorkItemsCallCount); - public Func? UnaryResponse { get; init; } - public override TResponse BlockingUnaryCall(Method method, string? host, CallOptions options, TRequest request) { throw new NotSupportedException(); @@ -2370,11 +2212,6 @@ public override AsyncUnaryCall AsyncUnaryCall(Me return (AsyncUnaryCall)(object)call; } - if (this.UnaryResponse is { } respond) - { - return CreateUnaryCall(Task.FromResult((TResponse)respond(method.Name, request))); - } - throw new NotSupportedException($"Unexpected unary method {method.FullName}."); } diff --git a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs index f3008385d..4bc5bb9d4 100644 --- a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs +++ b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs @@ -222,6 +222,59 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti result.Outcome.Should().Be(WorkItemStreamOutcome.Shutdown); } + [Fact] + public async Task HydrationFailure_RearmsTimerAndRecoversBeforeReadingNextItemAsync() + { + // Arrange + P.WorkItem failed = new() + { + CompletionToken = "failed-token", + OrchestratorRequest = new P.OrchestratorRequest + { + NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Input = "partially resolved" } } }, + }, + }; + WorkItemPayloadException failure = new(failed, new InvalidOperationException("Download failed.")); + P.WorkItem healthy = new() { HealthPing = new P.HealthPing() }; + Mock> reader = new(MockBehavior.Strict); + reader.SetupSequence(r => r.MoveNext(It.IsAny())) + .ThrowsAsync(failure) + .ReturnsAsync(true) + .ReturnsAsync(false); + reader.SetupGet(r => r.Current).Returns(healthy); + List events = new(); + List dispatched = new(); + + // Act + WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( + stream: reader.Object, + silentDisconnectTimeout: TimeSpan.FromSeconds(5), + onItem: item => + { + events.Add("dispatch"); + dispatched.Add(item); + }, + onFirstMessage: () => events.Add("first"), + cancellation: CancellationToken.None, + onSilentDisconnectTimerArmed: () => events.Add("armed"), + onPayloadFailure: (exception, cancellation) => + { + Assert.Same(failure, exception); + Assert.Same(failed, exception.WorkItem); + Assert.True(cancellation.CanBeCanceled); + events.Add("abandon"); + return Task.CompletedTask; + }); + + // Assert + Assert.Equal(WorkItemStreamOutcome.GracefulDrain, result.Outcome); + Assert.True(result.FirstMessageObserved); + Assert.Equal(new[] { "armed", "armed", "first", "abandon", "armed", "dispatch" }, events); + Assert.Same(healthy, Assert.Single(dispatched)); + reader.VerifyGet(r => r.Current, Times.Once); + reader.Verify(r => r.MoveNext(It.IsAny()), Times.Exactly(3)); + } + static IAsyncStreamReader EmptyStream() => StreamOf(); static IAsyncStreamReader StreamOf(params P.WorkItem[] items) => new SequenceStreamReader(items); From e70fac2ab5d07b3240aba265ec9c4edb4b42c9c7 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 10 Nov 2026 15:57:49 -0800 Subject: [PATCH 4/9] refactor: reduce abandon-only hydration patch Remove the standalone 536-line scripted sidecar and reuse existing worker and integration test harnesses. Leave ordinary background failure handling unchanged, remove unnecessary test friend access and duplicate documentation, and retain delivery-local abandonment for every hydration failure. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Extensions/AzureBlobPayloads/README.md | 14 - src/Grpc/Grpc.csproj | 1 - .../Grpc/GrpcDurableTaskWorker.Processor.cs | 172 +++--- src/Worker/Grpc/Worker.Grpc.csproj | 1 - .../LargePayloadTests.cs | 76 +++ .../PayloadHydrationFailureTests.cs | 536 ------------------ .../Grpc.Tests/GrpcDurableTaskWorkerTests.cs | 106 ++++ .../Grpc.Tests/WorkItemStreamConsumerTests.cs | 53 -- 8 files changed, 288 insertions(+), 671 deletions(-) delete mode 100644 test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs diff --git a/src/Extensions/AzureBlobPayloads/README.md b/src/Extensions/AzureBlobPayloads/README.md index e6fd9f6d1..146700caf 100644 --- a/src/Extensions/AzureBlobPayloads/README.md +++ b/src/Extensions/AzureBlobPayloads/README.md @@ -170,20 +170,6 @@ orphan blobs: real I/O failures, cancellation during uploads, and uncertain backend submission can still leave unrecorded uploads. No catch-and-delete rollback is performed. -## Incoming work-item hydration failures - -If downloading an incoming work item's payload fails, the worker abandons only -that delivery's completion token and continues reading the same work-item stream. -This includes missing blobs, authorization failures, and transient I/O errors. -The failed delivery is not dispatched, even if some fields were already hydrated, -and remains retryable; this does not introduce terminal poison-payload settlement. - -An abandonment response of `NotFound` means ownership was already lost and does -not disconnect the stream. Other abandonment failures retain connection-level -recovery after the worker's existing retry policy. Worker shutdown cancellation -is unchanged. Unary responses and streamed history still propagate hydration -errors to their existing callers. - ## SDK and dashboard boundaries The client and worker gRPC interceptors resolve orchestration inputs, external diff --git a/src/Grpc/Grpc.csproj b/src/Grpc/Grpc.csproj index 4798b8bac..f2e1ef3e2 100644 --- a/src/Grpc/Grpc.csproj +++ b/src/Grpc/Grpc.csproj @@ -24,7 +24,6 @@ - diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 0052fd920..e18cca78d 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -281,13 +281,6 @@ static OrchestrationRuntimeState BuildRuntimeState( return runtimeState; } - static string GetWorkItemInstanceId(P.WorkItem? workItem) => - workItem?.OrchestratorRequest?.InstanceId ?? - workItem?.ActivityRequest?.OrchestrationInstance?.InstanceId ?? - workItem?.EntityRequest?.InstanceId ?? - workItem?.EntityRequestV2?.InstanceId ?? - string.Empty; - async ValueTask> GetPastEventsAsync( P.OrchestratorRequest orchestratorRequest, CancellationToken cancellation) @@ -542,8 +535,86 @@ void RunBackgroundTask(P.WorkItem? workItem, Func handler, CancellationTok } catch (Exception ex) { - this.Logger.UnexpectedError(ex, GetWorkItemInstanceId(workItem)); - await this.AbandonWorkItemAsync(workItem, cancellation); + string instanceId = + workItem?.OrchestratorRequest?.InstanceId ?? + workItem?.ActivityRequest?.OrchestrationInstance?.InstanceId ?? + workItem?.EntityRequest?.InstanceId ?? + workItem?.EntityRequestV2?.InstanceId ?? + string.Empty; + this.Logger.UnexpectedError(ex, instanceId); + + if (workItem?.OrchestratorRequest != null) + { + this.Logger.AbandoningOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskOrchestratorWorkItemAsync( + new P.AbandonOrchestrationTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskOrchestratorWorkItemAsync), + cancellation); + this.Logger.AbandonedOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); + } + else if (workItem?.ActivityRequest != null) + { + this.Logger.AbandoningActivityWorkItem( + instanceId, + workItem.ActivityRequest.Name, + workItem.ActivityRequest.TaskId, + workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskActivityWorkItemAsync( + new P.AbandonActivityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskActivityWorkItemAsync), + cancellation); + this.Logger.AbandonedActivityWorkItem( + instanceId, + workItem.ActivityRequest.Name, + workItem.ActivityRequest.TaskId, + workItem.CompletionToken ?? string.Empty); + } + else if (workItem?.EntityRequest != null) + { + this.Logger.AbandoningEntityWorkItem( + workItem.EntityRequest.InstanceId, + workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskEntityWorkItemAsync), + cancellation); + this.Logger.AbandonedEntityWorkItem( + workItem.EntityRequest.InstanceId, + workItem.CompletionToken ?? string.Empty); + } + else if (workItem?.EntityRequestV2 != null) + { + this.Logger.AbandoningEntityWorkItem( + workItem.EntityRequestV2.InstanceId, + workItem.CompletionToken ?? string.Empty); + await this.ExecuteWithRetryAsync( + async () => await this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest + { + CompletionToken = workItem.CompletionToken, + }, + cancellationToken: cancellation), + nameof(this.client.AbandonTaskEntityWorkItemAsync), + cancellation); + this.Logger.AbandonedEntityWorkItem( + workItem.EntityRequestV2.InstanceId, + workItem.CompletionToken ?? string.Empty); + } } }); } @@ -551,10 +622,34 @@ void RunBackgroundTask(P.WorkItem? workItem, Func handler, CancellationTok async Task OnWorkItemPayloadFailureAsync(WorkItemPayloadException exception, CancellationToken cancellation) { P.WorkItem workItem = exception.WorkItem; - this.Logger.UnexpectedError(exception, GetWorkItemInstanceId(workItem)); + string instanceId = + workItem.OrchestratorRequest?.InstanceId ?? + workItem.ActivityRequest?.OrchestrationInstance?.InstanceId ?? + workItem.EntityRequest?.InstanceId ?? + workItem.EntityRequestV2?.InstanceId ?? + string.Empty; + this.Logger.UnexpectedError(exception, instanceId); try { - await this.AbandonWorkItemAsync(workItem, cancellation); + await this.ExecuteWithRetryAsync( + () => workItem.RequestCase switch + { + P.WorkItem.RequestOneofCase.OrchestratorRequest => + this.client.AbandonTaskOrchestratorWorkItemAsync( + new P.AbandonOrchestrationTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + P.WorkItem.RequestOneofCase.ActivityRequest => + this.client.AbandonTaskActivityWorkItemAsync( + new P.AbandonActivityTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + P.WorkItem.RequestOneofCase.EntityRequest or P.WorkItem.RequestOneofCase.EntityRequestV2 => + this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + _ => throw new InvalidOperationException($"Cannot abandon unresolved {workItem.RequestCase}."), + }, + "AbandonUnresolvedWorkItem", + cancellation); } catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) { @@ -563,61 +658,6 @@ async Task OnWorkItemPayloadFailureAsync(WorkItemPayloadException exception, Can } } - async Task AbandonWorkItemAsync(P.WorkItem? workItem, CancellationToken cancellation) - { - string instanceId = GetWorkItemInstanceId(workItem); - if (workItem?.OrchestratorRequest != null) - { - this.Logger.AbandoningOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskOrchestratorWorkItemAsync( - new P.AbandonOrchestrationTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskOrchestratorWorkItemAsync), - cancellation); - this.Logger.AbandonedOrchestratorWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.ActivityRequest != null) - { - this.Logger.AbandoningActivityWorkItem( - instanceId, - workItem.ActivityRequest.Name, - workItem.ActivityRequest.TaskId, - workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskActivityWorkItemAsync( - new P.AbandonActivityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskActivityWorkItemAsync), - cancellation); - this.Logger.AbandonedActivityWorkItem( - instanceId, - workItem.ActivityRequest.Name, - workItem.ActivityRequest.TaskId, - workItem.CompletionToken ?? string.Empty); - } - else if (workItem?.EntityRequest != null || workItem?.EntityRequestV2 != null) - { - this.Logger.AbandoningEntityWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - await this.ExecuteWithRetryAsync( - async () => await this.client.AbandonTaskEntityWorkItemAsync( - new P.AbandonEntityTaskRequest - { - CompletionToken = workItem.CompletionToken, - }, - cancellationToken: cancellation), - nameof(this.client.AbandonTaskEntityWorkItemAsync), - cancellation); - this.Logger.AbandonedEntityWorkItem(instanceId, workItem.CompletionToken ?? string.Empty); - } - } - async Task OnRunOrchestratorAsync( P.OrchestratorRequest request, string completionToken, diff --git a/src/Worker/Grpc/Worker.Grpc.csproj b/src/Worker/Grpc/Worker.Grpc.csproj index 0fa7b856b..2b5d41138 100644 --- a/src/Worker/Grpc/Worker.Grpc.csproj +++ b/src/Worker/Grpc/Worker.Grpc.csproj @@ -22,7 +22,6 @@ - diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index f29b71062..a35f77827 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.IO; using System.Text.Json; +using Azure; using FluentAssertions; using Grpc.Core; using Grpc.Core.Interceptors; @@ -11,6 +13,7 @@ using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; +using Moq; using Xunit.Abstractions; using P = Microsoft.DurableTask.Protobuf; @@ -18,6 +21,79 @@ namespace Microsoft.DurableTask.Grpc.Tests; public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) : IntegrationTestBase(output, sidecarFixture) { + [Theory] + [InlineData(0)] + [InlineData(403)] + [InlineData(404)] + public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int storageStatus) + { + // Arrange + string token = $"test-blob://missing-{Guid.NewGuid():N}"; + TaskName orchestration = nameof(HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync); + TaskName activity = "HydrationActivity"; + TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource downloadFailed = new(TaskCreationOptions.RunContinuationsAsynchronously); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value == token); + store.Setup(s => s.UploadAsync(It.IsAny(), It.IsAny())).ReturnsAsync(token); + store.Setup(s => s.DownloadAsync(token, It.IsAny())).Returns(() => + { + downloadFailed.TrySetResult(); + Exception failure = storageStatus == 0 + ? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed."); + return Task.FromException(failure); + }); + await using HostTestLifetime server = await this.StartWorkerAsync(worker => + { + worker.AddTasks(tasks => tasks + .AddOrchestratorFunc(orchestration, (context, input) => + context.CallActivityAsync(activity, input == "bad" ? new string('X', 1024) : input)) + .AddActivityFunc(activity, async (_, input) => + { + if (input == "running") + { + running.TrySetResult(); + await release.Task.WaitAsync(this.TimeoutToken); + } + + return input; + })); + worker.UseExternalizedPayloads(); + worker.Services.AddSingleton(store.Object); + }, servicesConfigure: services => services.AddExternalizedPayloadStore(options => + { + options.ThresholdBytes = 16; + options.ConnectionString = "UseDevelopmentStorage=true"; + })); + + try + { + // Act + string alreadyRunning = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "running"); + await running.Task.WaitAsync(this.TimeoutToken); + await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "bad"); + await downloadFailed.Task.WaitAsync(this.TimeoutToken); + string following = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "following"); + OrchestrationMetadata followingResult = await server.Client.WaitForInstanceCompletionAsync( + following, getInputsAndOutputs: true, this.TimeoutToken); + release.TrySetResult(); + OrchestrationMetadata runningResult = await server.Client.WaitForInstanceCompletionAsync( + alreadyRunning, getInputsAndOutputs: true, this.TimeoutToken); + + // Assert + Assert.Equal(OrchestrationRuntimeStatus.Completed, followingResult.RuntimeStatus); + Assert.Equal("following", followingResult.ReadOutputAs()); + Assert.Equal(OrchestrationRuntimeStatus.Completed, runningResult.RuntimeStatus); + Assert.Equal("running", runningResult.ReadOutputAs()); + Assert.Single(this.GetLogs(), log => log.EventId.Name == "EstablishedWorkItemConnection"); + } + finally + { + release.TrySetResult(); + } + } + [Theory] [InlineData(true)] [InlineData(false)] diff --git a/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs b/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs deleted file mode 100644 index 42a65e63b..000000000 --- a/test/Grpc.IntegrationTests/PayloadHydrationFailureTests.cs +++ /dev/null @@ -1,536 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Concurrent; -using System.IO; -using System.Net; -using System.Text.Json; -using System.Threading.Channels; -using Azure; -using Google.Protobuf.WellKnownTypes; -using Grpc.Core; -using Grpc.Core.Interceptors; -using Grpc.Net.Client; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Hosting.Server; -using Microsoft.AspNetCore.Hosting.Server.Features; -using Microsoft.AspNetCore.Server.Kestrel.Core; -using Microsoft.DurableTask.Worker; -using Microsoft.DurableTask.Worker.AzureManaged; -using Microsoft.DurableTask.Worker.Grpc; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Moq; -using P = Microsoft.DurableTask.Protobuf; - -namespace Microsoft.DurableTask.Grpc.Tests; - -public class PayloadHydrationFailureTests -{ - const string TokenPrefix = "test-blob://"; - const string BadToken = TokenPrefix + "bad"; - const string FailedCompletionToken = "failed-delivery"; - static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(15); - - public static IEnumerable HydrationFailures() - { - P.WorkItem.RequestOneofCase[] kinds = - [ - P.WorkItem.RequestOneofCase.ActivityRequest, - P.WorkItem.RequestOneofCase.OrchestratorRequest, - P.WorkItem.RequestOneofCase.EntityRequest, - P.WorkItem.RequestOneofCase.EntityRequestV2, - ]; - - foreach (P.WorkItem.RequestOneofCase kind in kinds) - { - foreach (int storageStatus in new[] { 404, 403, 0 }) - { - foreach (bool ownershipLost in new[] { false, true }) - { - foreach (bool useAzureManaged in new[] { false, true }) - { - yield return [kind, storageStatus, ownershipLost, useAzureManaged]; - } - } - } - } - } - - [Theory] - [MemberData(nameof(HydrationFailures))] - public async Task HydrationFailure_AbandonsOnlyFailedDeliveryAndKeepsStreamAsync( - P.WorkItem.RequestOneofCase kind, int storageStatus, bool ownershipLost, bool useAzureManaged) - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - sidecar.AbandonStatus = ownershipLost ? StatusCode.NotFound : StatusCode.OK; - Exception failure = storageStatus == 0 - ? new IOException("Transient payload download failure.") - : new RequestFailedException(storageStatus, "Payload download failed."); - Mock store = CreatePayloadStore(failure); - TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously); - TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); - ConcurrentQueue inputs = new(); - await using ServiceProvider provider = CreateWorker(sidecar, store.Object, async input => - { - inputs.Enqueue(input); - if (input == "running") - { - running.TrySetResult(); - await release.Task.WaitAsync(timeout.Token); - } - - return input; - }, useAzureManaged); - IHostedService worker = Assert.Single(provider.GetServices()); - await worker.StartAsync(timeout.Token); - - try - { - // Act - sidecar.Send(CreateHealthyActivity("running")); - await running.Task.WaitAsync(timeout.Token); - sidecar.Send(CreateFailedWorkItem(kind)); - sidecar.Send(new P.WorkItem { HealthPing = new P.HealthPing() }); - sidecar.Send(CreateHealthyActivity("following")); - P.ActivityResponse following = await sidecar.Completions.Reader.ReadAsync(timeout.Token); - release.TrySetResult(); - P.ActivityResponse alreadyRunning = await sidecar.Completions.Reader.ReadAsync(timeout.Token); - - // Assert - Assert.Equal(1, sidecar.StreamCount); - Assert.Equal(0, sidecar.ClosedStreamCount); - Assert.Equal(1, sidecar.HelloCount); - P.WorkItem.RequestOneofCase abandonKind = kind == P.WorkItem.RequestOneofCase.EntityRequestV2 - ? P.WorkItem.RequestOneofCase.EntityRequest - : kind; - Assert.Equal((abandonKind, FailedCompletionToken), Assert.Single(sidecar.Abandons)); - Assert.Equal(new[] { "running", "following" }, inputs.ToArray()); - Assert.Equal("following", following.CompletionToken); - Assert.Equal("\"following\"", following.Result); - Assert.Null(following.FailureDetails); - Assert.Equal("running", alreadyRunning.CompletionToken); - Assert.Equal("\"running\"", alreadyRunning.Result); - Assert.Null(alreadyRunning.FailureDetails); - store.Verify(s => s.DownloadAsync(BadToken, It.IsAny()), Times.Once); - } - finally - { - release.TrySetResult(); - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - } - } - - [Theory] - [InlineData(StatusCode.Unknown, false)] - [InlineData(StatusCode.Unavailable, false)] - [InlineData(StatusCode.PermissionDenied, false)] - [InlineData(StatusCode.Internal, false)] - [InlineData(StatusCode.Unavailable, true)] - public async Task TransportOrAbandonFailure_ReconnectsAsync(StatusCode status, bool streamFailure) - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - sidecar.AbandonStatus = status; - Mock store = CreatePayloadStore(new IOException("Payload download failed.")); - await using ServiceProvider provider = CreateWorker(sidecar, store.Object); - IHostedService worker = Assert.Single(provider.GetServices()); - await worker.StartAsync(timeout.Token); - - try - { - Assert.Equal(1, await sidecar.Streams.Reader.ReadAsync(timeout.Token)); - - // Act - if (streamFailure) - { - sidecar.FailStream(status); - } - else - { - sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); - } - - Assert.Equal(2, await sidecar.Streams.Reader.ReadAsync(timeout.Token)); - sidecar.Send(CreateHealthyActivity("following")); - P.ActivityResponse response = await sidecar.Completions.Reader.ReadAsync(timeout.Token); - - // Assert - Assert.Equal(2, sidecar.StreamCount); - Assert.Equal(2, sidecar.HelloCount); - Assert.Equal("following", response.CompletionToken); - Assert.Null(response.FailureDetails); - Assert.Equal(streamFailure ? 0 : 1, sidecar.Abandons.Count); - } - finally - { - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - } - } - - [Fact] - public async Task HydrationFailure_RedeliveryCompletesAfterPayloadRecoversAsync() - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - Mock store = CreatePayloadStore(new IOException("Payload download failed.")); - store.SetupSequence(s => s.DownloadAsync(BadToken, It.IsAny())) - .ThrowsAsync(new RequestFailedException(404, "Payload not found.")) - .ReturnsAsync("\"recovered\""); - await using ServiceProvider provider = CreateWorker(sidecar, store.Object); - IHostedService worker = Assert.Single(provider.GetServices()); - await worker.StartAsync(timeout.Token); - - try - { - // Act - sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); - await sidecar.AbandonStarted.Task.WaitAsync(timeout.Token); - P.WorkItem redelivery = CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest); - redelivery.CompletionToken = "redelivery"; - sidecar.Send(redelivery); - P.ActivityResponse response = await sidecar.Completions.Reader.ReadAsync(timeout.Token); - - // Assert - Assert.Equal(1, sidecar.StreamCount); - Assert.Equal((P.WorkItem.RequestOneofCase.ActivityRequest, FailedCompletionToken), Assert.Single(sidecar.Abandons)); - Assert.Equal("redelivery", response.CompletionToken); - Assert.Equal("\"recovered\"", response.Result); - Assert.Null(response.FailureDetails); - } - finally - { - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ShutdownDuringHydrationOrAbandon_DoesNotReconnectAsync(bool duringAbandon) - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - sidecar.HoldAbandon = duringAbandon; - TaskCompletionSource downloading = new(TaskCreationOptions.RunContinuationsAsynchronously); - Mock store = CreatePayloadStore(new IOException("Payload download failed.")); - if (!duringAbandon) - { - store.Setup(s => s.DownloadAsync(BadToken, It.IsAny())) - .Returns(async (string _, CancellationToken cancellation) => - { - downloading.TrySetResult(cancellation); - await Task.Delay(Timeout.Infinite, cancellation); - return string.Empty; - }); - } - - await using ServiceProvider provider = CreateWorker(sidecar, store.Object); - IHostedService worker = Assert.Single(provider.GetServices()); - await worker.StartAsync(timeout.Token); - - try - { - sidecar.Send(CreateFailedWorkItem(P.WorkItem.RequestOneofCase.ActivityRequest)); - CancellationToken downloadCancellation = default; - if (duringAbandon) - { - await sidecar.AbandonStarted.Task.WaitAsync(timeout.Token); - } - else - { - downloadCancellation = await downloading.Task.WaitAsync(timeout.Token); - } - - // Act - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - - // Assert - Assert.Equal(1, sidecar.StreamCount); - Assert.Equal(duringAbandon ? 1 : 0, sidecar.Abandons.Count); - Assert.False(sidecar.Completions.Reader.TryRead(out _)); - if (!duringAbandon) - { - Assert.True(downloadCancellation.IsCancellationRequested); - } - } - finally - { - await worker.StopAsync(CancellationToken.None).WaitAsync(TestTimeout); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task NonWorkItemHydrationFailure_PropagatesOriginalExceptionAsync(bool historyStream) - { - // Arrange - using CancellationTokenSource timeout = new(TestTimeout); - await using ScriptedSidecar sidecar = await ScriptedSidecar.StartAsync(); - RequestFailedException failure = new(403, "Payload download forbidden."); - Mock store = CreatePayloadStore(failure); - using GrpcChannel channel = GrpcChannel.ForAddress(sidecar.Address); - P.TaskHubSidecarService.TaskHubSidecarServiceClient client = new(channel.Intercept( - new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions()))); - - // Act - Exception? observed; - if (historyStream) - { - using AsyncServerStreamingCall call = client.StreamInstanceHistory( - new P.StreamInstanceHistoryRequest { InstanceId = "instance" }, cancellationToken: timeout.Token); - observed = await Record.ExceptionAsync(() => call.ResponseStream.MoveNext(timeout.Token)); - } - else - { - using AsyncUnaryCall call = client.GetInstanceAsync( - new P.GetInstanceRequest { InstanceId = "instance" }, cancellationToken: timeout.Token); - observed = await Record.ExceptionAsync(async () => await call.ResponseAsync); - } - - // Assert - Assert.Same(failure, observed); - Assert.Empty(sidecar.Abandons); - } - - static Mock CreatePayloadStore(Exception failure) - { - Mock store = new(); - store.Setup(s => s.IsKnownPayloadToken(It.IsAny())) - .Returns((string value) => value.StartsWith(TokenPrefix, StringComparison.Ordinal)); - store.Setup(s => s.DownloadAsync(It.IsAny(), It.IsAny())) - .Returns((string token, CancellationToken _) => token == BadToken - ? Task.FromException(failure) - : Task.FromResult(JsonSerializer.Serialize(token[TokenPrefix.Length..]))); - return store; - } - - static ServiceProvider CreateWorker( - ScriptedSidecar sidecar, - PayloadStore store, - Func>? runActivity = null, - bool useAzureManaged = false) - { - ServiceCollection services = new(); - services.AddLogging(logging => logging.ClearProviders()); - services.AddSingleton(store); - services.AddDurableTaskWorker(builder => - { - if (useAzureManaged) - { - builder.UseDurableTaskScheduler($"Endpoint={sidecar.Address};TaskHub=test;Authentication=None"); - } - else - { - builder.UseGrpc(sidecar.Address); - } - - builder.Services.Configure(builder.Name, options => - { - options.Internal.TransientRetryMaxAttempts = 1; - options.Internal.ReconnectBackoffBase = TimeSpan.Zero; - options.Internal.ReconnectBackoffCap = TimeSpan.Zero; - }); - builder.UseExternalizedPayloads(); - builder.AddTasks(tasks => tasks.AddActivityFunc( - "Healthy", (_, input) => runActivity?.Invoke(input!) ?? Task.FromResult(input!))); - }); - return services.BuildServiceProvider(); - } - - static P.WorkItem CreateHealthyActivity(string name) => new() - { - CompletionToken = name, - ActivityRequest = new P.ActivityRequest - { - Name = "Healthy", - TaskId = 1, - Input = TokenPrefix + name, - OrchestrationInstance = new P.OrchestrationInstance { InstanceId = name, ExecutionId = "execution" }, - }, - }; - - static P.WorkItem CreateFailedWorkItem(P.WorkItem.RequestOneofCase kind) - { - P.WorkItem item = new() { CompletionToken = FailedCompletionToken }; - switch (kind) - { - case P.WorkItem.RequestOneofCase.ActivityRequest: - item.ActivityRequest = CreateHealthyActivity("failed").ActivityRequest; - item.ActivityRequest.Input = BadToken; - break; - case P.WorkItem.RequestOneofCase.OrchestratorRequest: - item.OrchestratorRequest = new P.OrchestratorRequest - { - InstanceId = "failed", - PastEvents = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = TokenPrefix + "state" } } }, - NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Name = "event", Input = BadToken } } }, - }; - break; - case P.WorkItem.RequestOneofCase.EntityRequest: - item.EntityRequest = new P.EntityBatchRequest - { - InstanceId = "@entity@failed", - EntityState = TokenPrefix + "state", - Operations = { new P.OperationRequest { Operation = "operation", Input = BadToken } }, - }; - break; - case P.WorkItem.RequestOneofCase.EntityRequestV2: - item.EntityRequestV2 = new P.EntityRequest - { - InstanceId = "@entity@failed", - EntityState = TokenPrefix + "state", - OperationRequests = { new P.HistoryEvent { EntityOperationSignaled = new P.EntityOperationSignaledEvent { Input = BadToken } } }, - }; - break; - default: - throw new ArgumentOutOfRangeException(nameof(kind), kind, "Unsupported work item kind."); - } - - return item; - } - - sealed class ScriptedSidecar : P.TaskHubSidecarService.TaskHubSidecarServiceBase, IAsyncDisposable - { - readonly Channel, Task>> deliveries = - Channel.CreateUnbounded, Task>>(); - IHost host = null!; - int streamCount; - int closedStreamCount; - int helloCount; - - public string Address { get; private set; } = string.Empty; - - public StatusCode AbandonStatus { get; set; } - - public bool HoldAbandon { get; set; } - - public int StreamCount => Volatile.Read(ref this.streamCount); - - public int ClosedStreamCount => Volatile.Read(ref this.closedStreamCount); - - public int HelloCount => Volatile.Read(ref this.helloCount); - - public Channel Streams { get; } = Channel.CreateUnbounded(); - - public Channel Completions { get; } = Channel.CreateUnbounded(); - - public ConcurrentQueue<(P.WorkItem.RequestOneofCase Kind, string Token)> Abandons { get; } = new(); - - public TaskCompletionSource AbandonStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - - public static async Task StartAsync() - { - ScriptedSidecar sidecar = new(); - sidecar.host = Host.CreateDefaultBuilder() - .ConfigureLogging(logging => logging.ClearProviders()) - .ConfigureWebHostDefaults(web => web - .UseKestrel(options => options.Listen(IPAddress.Loopback, 0, endpoint => endpoint.Protocols = HttpProtocols.Http2)) - .ConfigureServices(services => - { - services.AddGrpc(); - services.AddSingleton(sidecar); - }) - .Configure(app => - { - app.UseRouting(); - app.UseEndpoints(endpoints => endpoints.MapGrpcService()); - })) - .Build(); - await sidecar.host.StartAsync().WaitAsync(TestTimeout); - sidecar.Address = Assert.Single(sidecar.host.Services.GetRequiredService() - .Features.Get()!.Addresses); - return sidecar; - } - - public void Send(P.WorkItem item) => Assert.True(this.deliveries.Writer.TryWrite(stream => stream.WriteAsync(item))); - - public void FailStream(StatusCode status) => Assert.True(this.deliveries.Writer.TryWrite( - _ => Task.FromException(new RpcException(new Status(status, "Stream transport failed."))))); - - public override Task Hello(Empty request, ServerCallContext context) - { - Interlocked.Increment(ref this.helloCount); - return Task.FromResult(new Empty()); - } - - public override async Task GetWorkItems(P.GetWorkItemsRequest request, IServerStreamWriter responseStream, ServerCallContext context) - { - await context.WriteResponseHeadersAsync(new Metadata()); - this.Streams.Writer.TryWrite(Interlocked.Increment(ref this.streamCount)); - try - { - await foreach (Func, Task> deliver in this.deliveries.Reader.ReadAllAsync(context.CancellationToken)) - { - await deliver(responseStream); - } - } - finally - { - Interlocked.Increment(ref this.closedStreamCount); - } - } - - public override Task CompleteActivityTask(P.ActivityResponse request, ServerCallContext context) - { - this.Completions.Writer.TryWrite(request); - return Task.FromResult(new P.CompleteTaskResponse()); - } - - public override async Task AbandonTaskActivityWorkItem(P.AbandonActivityTaskRequest request, ServerCallContext context) - { - await this.AbandonAsync(P.WorkItem.RequestOneofCase.ActivityRequest, request.CompletionToken, context); - return new P.AbandonActivityTaskResponse(); - } - - public override async Task AbandonTaskOrchestratorWorkItem(P.AbandonOrchestrationTaskRequest request, ServerCallContext context) - { - await this.AbandonAsync(P.WorkItem.RequestOneofCase.OrchestratorRequest, request.CompletionToken, context); - return new P.AbandonOrchestrationTaskResponse(); - } - - public override async Task AbandonTaskEntityWorkItem(P.AbandonEntityTaskRequest request, ServerCallContext context) - { - await this.AbandonAsync(P.WorkItem.RequestOneofCase.EntityRequest, request.CompletionToken, context); - return new P.AbandonEntityTaskResponse(); - } - - public override Task GetInstance(P.GetInstanceRequest request, ServerCallContext context) - => Task.FromResult(new P.GetInstanceResponse { OrchestrationState = new P.OrchestrationState { Input = BadToken } }); - - public override Task StreamInstanceHistory(P.StreamInstanceHistoryRequest request, IServerStreamWriter responseStream, ServerCallContext context) - => responseStream.WriteAsync(new P.HistoryChunk - { - Events = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = BadToken } } }, - }); - - public async ValueTask DisposeAsync() - { - using IHost host = this.host; - await host.StopAsync().WaitAsync(TestTimeout); - } - - async Task AbandonAsync(P.WorkItem.RequestOneofCase kind, string token, ServerCallContext context) - { - this.Abandons.Enqueue((kind, token)); - this.AbandonStarted.TrySetResult(); - if (this.HoldAbandon) - { - await Task.Delay(Timeout.Infinite, context.CancellationToken); - } - - if (this.AbandonStatus != StatusCode.OK) - { - throw new RpcException(new Status(this.AbandonStatus, "Abandon failed.")); - } - } - } -} diff --git a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs index 4b3505c16..3a8762cd0 100644 --- a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs +++ b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs @@ -5,10 +5,12 @@ using System.Diagnostics; using System.IO; using System.Reflection; +using Azure; using DurableTask.Core; using DurableTask.Core.History; using Google.Protobuf.WellKnownTypes; using Grpc.Core; +using Grpc.Core.Interceptors; using Microsoft.DurableTask; using Microsoft.DurableTask.Tests.Logging; using Microsoft.DurableTask.Tracing; @@ -49,6 +51,103 @@ public class GrpcDurableTaskWorkerTests .GetNestedType("Processor", BindingFlags.NonPublic)! .GetMethod("OnRunOrchestratorAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; + [Theory] + [CombinatorialData] + public async Task ProcessorExecuteAsync_HydrationFailure_AbandonsOnlyFailedDeliveryAsync( + [CombinatorialValues(P.WorkItem.RequestOneofCase.ActivityRequest, P.WorkItem.RequestOneofCase.OrchestratorRequest, + P.WorkItem.RequestOneofCase.EntityRequest, P.WorkItem.RequestOneofCase.EntityRequestV2)] P.WorkItem.RequestOneofCase kind, + [CombinatorialValues(0, 403, 404)] int storageStatus, + [CombinatorialValues(StatusCode.OK, StatusCode.NotFound, StatusCode.Unavailable, StatusCode.PermissionDenied)] StatusCode abandonStatus) + { + // Arrange + const string Token = "test-blob://failed"; + P.WorkItem failed = kind switch + { + P.WorkItem.RequestOneofCase.ActivityRequest => new() { ActivityRequest = new P.ActivityRequest { Input = Token } }, + P.WorkItem.RequestOneofCase.OrchestratorRequest => new() { OrchestratorRequest = new P.OrchestratorRequest + { + NewEvents = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = Token } } }, + } }, + P.WorkItem.RequestOneofCase.EntityRequest => new() { EntityRequest = new P.EntityBatchRequest { EntityState = Token } }, + _ => new() { EntityRequestV2 = new P.EntityRequest { EntityState = Token } }, + }; + failed.CompletionToken = "failed"; + Queue deliveries = new([failed, new P.WorkItem { HealthPing = new P.HealthPing() }]); + TaskCompletionSource pingDispatched = new(TaskCreationOptions.RunContinuationsAsynchronously); + P.WorkItem current = null!; + Mock> reader = new(); + reader.SetupGet(r => r.Current).Returns(() => current); + reader.Setup(r => r.MoveNext(It.IsAny())).Returns(async (CancellationToken cancellation) => + { + if (deliveries.TryDequeue(out P.WorkItem? item)) + { + current = item; + return true; + } + + pingDispatched.TrySetResult(); + await Task.Delay(Timeout.Infinite, cancellation); + return false; + }); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value == Token); + Exception failure = storageStatus == 0 ? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed."); + store.Setup(s => s.DownloadAsync(Token, It.IsAny())).ThrowsAsync(failure); + string expectedOperation = kind switch + { + P.WorkItem.RequestOneofCase.ActivityRequest => "AbandonTaskActivityWorkItem", + P.WorkItem.RequestOneofCase.OrchestratorRequest => "AbandonTaskOrchestratorWorkItem", + _ => "AbandonTaskEntityWorkItem", + }; + List abandoned = []; + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (_, _) => CreateServerStreamingCall(reader.Object)) + { + UnaryResponse = (operation, request) => + { + Assert.Equal(expectedOperation, operation); + (string Token, object Reply) result = request switch + { + P.AbandonActivityTaskRequest r => (r.CompletionToken, new P.AbandonActivityTaskResponse()), + P.AbandonOrchestrationTaskRequest r => (r.CompletionToken, new P.AbandonOrchestrationTaskResponse()), + P.AbandonEntityTaskRequest r => (r.CompletionToken, new P.AbandonEntityTaskResponse()), + _ => throw new InvalidOperationException($"Unexpected request {request.GetType()}."), + }; + abandoned.Add(result.Token); + if (abandonStatus != StatusCode.OK) + { + throw new RpcException(new Status(abandonStatus, "Abandon failed.")); + } + + return result.Reply; + }, + }; + GrpcDurableTaskWorkerOptions options = new(); + options.Internal.TransientRetryMaxAttempts = 1; + options.Internal.ReconnectBackoffBase = options.Internal.ReconnectBackoffCap = TimeSpan.Zero; + object processor = CreateProcessor(CreateWorker(options), new P.TaskHubSidecarService.TaskHubSidecarServiceClient( + invoker.Intercept(new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions())))); + using CancellationTokenSource shutdown = new(); + + // Act + Task processing = InvokeProcessorExecuteAsync(processor, shutdown.Token); + try + { + await pingDispatched.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally + { + shutdown.Cancel(); + await processing; + } + + // Assert + Assert.Equal(new[] { "failed" }, abandoned); + Assert.Equal(abandonStatus is StatusCode.OK or StatusCode.NotFound ? 1 : 2, invoker.GetWorkItemsCallCount); + store.Verify(s => s.DownloadAsync(Token, It.IsAny()), Times.Once); + } + [Fact] public async Task ExecuteAsync_ConnectFailureThreshold_RecreatesConfiguredChannel() { @@ -2199,6 +2298,8 @@ public ScriptedWorkerCallInvoker( public int GetWorkItemsCallCount => Volatile.Read(ref this.getWorkItemsCallCount); + public Func? UnaryResponse { get; init; } + public override TResponse BlockingUnaryCall(Method method, string? host, CallOptions options, TRequest request) { throw new NotSupportedException(); @@ -2212,6 +2313,11 @@ public override AsyncUnaryCall AsyncUnaryCall(Me return (AsyncUnaryCall)(object)call; } + if (this.UnaryResponse is { } respond) + { + return CreateUnaryCall(Task.FromResult((TResponse)respond(method.Name, request))); + } + throw new NotSupportedException($"Unexpected unary method {method.FullName}."); } diff --git a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs index 4bc5bb9d4..f3008385d 100644 --- a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs +++ b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs @@ -222,59 +222,6 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti result.Outcome.Should().Be(WorkItemStreamOutcome.Shutdown); } - [Fact] - public async Task HydrationFailure_RearmsTimerAndRecoversBeforeReadingNextItemAsync() - { - // Arrange - P.WorkItem failed = new() - { - CompletionToken = "failed-token", - OrchestratorRequest = new P.OrchestratorRequest - { - NewEvents = { new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Input = "partially resolved" } } }, - }, - }; - WorkItemPayloadException failure = new(failed, new InvalidOperationException("Download failed.")); - P.WorkItem healthy = new() { HealthPing = new P.HealthPing() }; - Mock> reader = new(MockBehavior.Strict); - reader.SetupSequence(r => r.MoveNext(It.IsAny())) - .ThrowsAsync(failure) - .ReturnsAsync(true) - .ReturnsAsync(false); - reader.SetupGet(r => r.Current).Returns(healthy); - List events = new(); - List dispatched = new(); - - // Act - WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - stream: reader.Object, - silentDisconnectTimeout: TimeSpan.FromSeconds(5), - onItem: item => - { - events.Add("dispatch"); - dispatched.Add(item); - }, - onFirstMessage: () => events.Add("first"), - cancellation: CancellationToken.None, - onSilentDisconnectTimerArmed: () => events.Add("armed"), - onPayloadFailure: (exception, cancellation) => - { - Assert.Same(failure, exception); - Assert.Same(failed, exception.WorkItem); - Assert.True(cancellation.CanBeCanceled); - events.Add("abandon"); - return Task.CompletedTask; - }); - - // Assert - Assert.Equal(WorkItemStreamOutcome.GracefulDrain, result.Outcome); - Assert.True(result.FirstMessageObserved); - Assert.Equal(new[] { "armed", "armed", "first", "abandon", "armed", "dispatch" }, events); - Assert.Same(healthy, Assert.Single(dispatched)); - reader.VerifyGet(r => r.Current, Times.Once); - reader.Verify(r => r.MoveNext(It.IsAny()), Times.Exactly(3)); - } - static IAsyncStreamReader EmptyStream() => StreamOf(); static IAsyncStreamReader StreamOf(params P.WorkItem[] items) => new SequenceStreamReader(items); From 52ed544cfcd3bf04bda19bb7841287fd7d21e466 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 11 Nov 2026 07:38:13 -0800 Subject: [PATCH 5/9] fix: pause the read watchdog during payload abandonment Use worker shutdown cancellation during delivery recovery and rearm the read watchdog before receiving again. Add gated shutdown and slow-abandon regressions; fix the scoped-payload assertion and prevent test-sidecar concurrency starvation on small CI hosts. Keep all hydration failures abandon-only. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Worker/Grpc/WorkItemStreamConsumer.cs | 12 +- .../BlobPayloadStoreRoutingTests.cs | 2 +- .../LargePayloadTests.cs | 12 +- .../Grpc.Tests/GrpcDurableTaskWorkerTests.cs | 116 +++++++++++++++++- 4 files changed, 128 insertions(+), 14 deletions(-) diff --git a/src/Worker/Grpc/WorkItemStreamConsumer.cs b/src/Worker/Grpc/WorkItemStreamConsumer.cs index d814f2a3b..74164612d 100644 --- a/src/Worker/Grpc/WorkItemStreamConsumer.cs +++ b/src/Worker/Grpc/WorkItemStreamConsumer.cs @@ -66,8 +66,8 @@ internal static class WorkItemStreamConsumer /// detection. /// /// - /// Optional delivery-local recovery for a payload resolution failure. Exceptions from recovery - /// propagate to the caller for connection-level handling. + /// Optional delivery-local recovery outside the read watchdog, using the worker shutdown token. + /// Exceptions from recovery propagate to the caller for connection-level handling. /// /// The classified outcome plus whether any message was observed. public static async Task ConsumeAsync( @@ -104,8 +104,6 @@ void ArmSilentDisconnectTimer() void ObserveMessage() { - ArmSilentDisconnectTimer(); - if (!firstMessageObserved) { firstMessageObserved = true; @@ -126,11 +124,15 @@ void ObserveMessage() } catch (WorkItemPayloadException ex) when (onPayloadFailure is not null) { + // No reads can consume health pings while abandonment is pending. + timeoutSource.CancelAfter(Timeout.InfiniteTimeSpan); ObserveMessage(); - await onPayloadFailure(ex, tokenSource.Token).ConfigureAwait(false); + await onPayloadFailure(ex, cancellation).ConfigureAwait(false); + ArmSilentDisconnectTimer(); continue; } + ArmSilentDisconnectTimer(); ObserveMessage(); onItem(stream.Current); diff --git a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs index 4bd862eb7..d737069ed 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs @@ -254,7 +254,7 @@ public async Task WorkItem_ResolvesHistoryUsingConfiguredScopeAsync( } else { - await moveNext.Should().ThrowAsync(); + (await moveNext.Should().ThrowAsync()).Which.InnerException.Should().BeOfType(); workItem.OrchestratorRequest.NewEvents[0].ExecutionStarted.Input.Should().Be(token); handler.Requests.Should().BeEmpty(); } diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index a35f77827..a5d83e71c 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -28,6 +28,7 @@ public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture side public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int storageStatus) { // Arrange + using GrpcSidecarFixture isolatedSidecar = new(); string token = $"test-blob://missing-{Guid.NewGuid():N}"; TaskName orchestration = nameof(HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync); TaskName activity = "HydrationActivity"; @@ -46,6 +47,7 @@ public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int s }); await using HostTestLifetime server = await this.StartWorkerAsync(worker => { + worker.UseGrpc(isolatedSidecar.Channel); worker.AddTasks(tasks => tasks .AddOrchestratorFunc(orchestration, (context, input) => context.CallActivityAsync(activity, input == "bad" ? new string('X', 1024) : input)) @@ -61,7 +63,7 @@ public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int s })); worker.UseExternalizedPayloads(); worker.Services.AddSingleton(store.Object); - }, servicesConfigure: services => services.AddExternalizedPayloadStore(options => + }, client => client.UseGrpc(isolatedSidecar.Channel), services => services.AddExternalizedPayloadStore(options => { options.ThresholdBytes = 16; options.ConnectionString = "UseDevelopmentStorage=true"; @@ -74,12 +76,14 @@ public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int s await running.Task.WaitAsync(this.TimeoutToken); await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "bad"); await downloadFailed.Task.WaitAsync(this.TimeoutToken); - string following = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "following"); - OrchestrationMetadata followingResult = await server.Client.WaitForInstanceCompletionAsync( - following, getInputsAndOutputs: true, this.TimeoutToken); + + // The test sidecar does not settle abandoned activities; free the healthy slot before starting another. release.TrySetResult(); OrchestrationMetadata runningResult = await server.Client.WaitForInstanceCompletionAsync( alreadyRunning, getInputsAndOutputs: true, this.TimeoutToken); + string following = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "following"); + OrchestrationMetadata followingResult = await server.Client.WaitForInstanceCompletionAsync( + following, getInputsAndOutputs: true, this.TimeoutToken); // Assert Assert.Equal(OrchestrationRuntimeStatus.Completed, followingResult.RuntimeStatus); diff --git a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs index 3a8762cd0..400e062e4 100644 --- a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs +++ b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs @@ -104,7 +104,7 @@ public async Task ProcessorExecuteAsync_HydrationFailure_AbandonsOnlyFailedDeliv (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), (_, _) => CreateServerStreamingCall(reader.Object)) { - UnaryResponse = (operation, request) => + UnaryResponse = (operation, request, _) => { Assert.Equal(expectedOperation, operation); (string Token, object Reply) result = request switch @@ -120,7 +120,7 @@ public async Task ProcessorExecuteAsync_HydrationFailure_AbandonsOnlyFailedDeliv throw new RpcException(new Status(abandonStatus, "Abandon failed.")); } - return result.Reply; + return Task.FromResult(result.Reply); }, }; GrpcDurableTaskWorkerOptions options = new(); @@ -148,6 +148,112 @@ public async Task ProcessorExecuteAsync_HydrationFailure_AbandonsOnlyFailedDeliv store.Verify(s => s.DownloadAsync(Token, It.IsAny()), Times.Once); } + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ProcessorExecuteAsync_PendingPayloadRecovery_PreservesShutdownAndReadTimeoutAsync( + bool pendingAbandon, bool resume) + { + // Arrange + using CancellationTokenSource shutdown = new(); + TaskCompletionSource pending = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + P.WorkItem failed = new() + { + CompletionToken = "failed", + ActivityRequest = new P.ActivityRequest { Input = "payload-token" }, + }; + int reads = 0; + CancellationToken readCancellation = default; + Mock> reader = new(); + reader.SetupGet(r => r.Current).Returns(failed); + reader.Setup(r => r.MoveNext(It.IsAny())).Returns(async (CancellationToken cancellation) => + { + readCancellation = cancellation; + if (++reads == 1) + { + return true; + } + + await Task.Delay(Timeout.Infinite, cancellation); + return false; + }); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken("payload-token")).Returns(true); + store.Setup(s => s.DownloadAsync("payload-token", It.IsAny())) + .Returns(async (string _, CancellationToken cancellation) => + { + if (pendingAbandon) + { + throw new IOException("Download failed."); + } + + await WaitForReleaseAsync(cancellation); + return "\"input\""; + }); + int abandons = 0; + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (_, _) => CreateServerStreamingCall(reader.Object)) + { + UnaryResponse = async (operation, request, options) => + { + Assert.Equal("AbandonTaskActivityWorkItem", operation); + Assert.Equal("failed", Assert.IsType(request).CompletionToken); + abandons++; + await WaitForReleaseAsync(options.CancellationToken); + return new P.AbandonActivityTaskResponse(); + }, + }; + GrpcDurableTaskWorkerOptions grpcOptions = new(); + grpcOptions.Internal.ChannelRecreateFailureThreshold = 1; + grpcOptions.Internal.SilentDisconnectTimeout = resume ? TimeSpan.FromMilliseconds(250) : TimeSpan.FromSeconds(30); + object processor = CreateProcessor(CreateWorker(grpcOptions), new P.TaskHubSidecarService.TaskHubSidecarServiceClient( + invoker.Intercept(new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions())))); + + // Act + Task processing = InvokeProcessorExecuteAsync(processor, shutdown.Token); + try + { + CancellationToken recoveryCancellation = await pending.Task.WaitAsync(TimeSpan.FromSeconds(5)); + if (resume) + { + await Task.Delay(grpcOptions.Internal.SilentDisconnectTimeout * 2); + Assert.False(readCancellation.IsCancellationRequested); + Assert.False(recoveryCancellation.IsCancellationRequested); + Assert.False(processing.IsCompleted); + release.TrySetResult(); + } + else + { + shutdown.Cancel(); + } + + ProcessorExitReason reason = await processing.WaitAsync(TimeSpan.FromSeconds(5)); + + // Assert + Assert.Equal(resume ? ProcessorExitReason.ChannelRecreateRequested : ProcessorExitReason.Shutdown, reason); + Assert.Equal(resume ? 2 : 1, reads); + Assert.Equal(pendingAbandon ? 1 : 0, abandons); + Assert.Equal(1, invoker.GetWorkItemsCallCount); + reader.VerifyGet(r => r.Current, Times.Once); + store.Verify(s => s.DownloadAsync("payload-token", It.IsAny()), Times.Once); + } + finally + { + shutdown.Cancel(); + release.TrySetResult(); + await processing.WaitAsync(TimeSpan.FromSeconds(5)); + } + + async Task WaitForReleaseAsync(CancellationToken cancellation) + { + pending.TrySetResult(cancellation); + await release.Task.WaitAsync(cancellation); + } + } + [Fact] public async Task ExecuteAsync_ConnectFailureThreshold_RecreatesConfiguredChannel() { @@ -2298,7 +2404,7 @@ public ScriptedWorkerCallInvoker( public int GetWorkItemsCallCount => Volatile.Read(ref this.getWorkItemsCallCount); - public Func? UnaryResponse { get; init; } + public Func>? UnaryResponse { get; init; } public override TResponse BlockingUnaryCall(Method method, string? host, CallOptions options, TRequest request) { @@ -2315,7 +2421,9 @@ public override AsyncUnaryCall AsyncUnaryCall(Me if (this.UnaryResponse is { } respond) { - return CreateUnaryCall(Task.FromResult((TResponse)respond(method.Name, request))); + return CreateUnaryCall(ResponseAsync()); + + async Task ResponseAsync() => (TResponse)await respond(method.Name, request, options); } throw new NotSupportedException($"Unexpected unary method {method.FullName}."); From 1675f41f52bec370027cb244a6d56a8d8bae0619 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 11 Nov 2026 08:19:06 -0800 Subject: [PATCH 6/9] fix: separate transport, download, and abandonment budgets Watch only transport receipt for silent disconnects. Give each active worker payload download its own 120-second cancellation budget while preserving bounded concurrency and drainage. Bound receive-loop abandonment and its existing retries/backoff together to 30 seconds. Keep downloads abandon-only and preserve public unary/history behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AzureBlobPayloadsSideCarInterceptor.cs | 70 +++-- .../Interceptors/PayloadInterceptor.cs | 89 +++++- src/Extensions/AzureBlobPayloads/README.md | 8 + src/Grpc/WorkItemPayloadException.cs | 11 + .../Grpc/GrpcDurableTaskWorker.Processor.cs | 26 +- src/Worker/Grpc/WorkItemStreamConsumer.cs | 16 +- ...zureBlobPayloadsSideCarInterceptorTests.cs | 132 ++++++++ .../Grpc.Tests/GrpcDurableTaskWorkerTests.cs | 292 ++++++++++++++++++ 8 files changed, 599 insertions(+), 45 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs b/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs index e7d4dfa9a..4a9bfe429 100644 --- a/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs +++ b/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs @@ -67,6 +67,10 @@ internal async Task PrepareOrchestratorResponseAsync(P.OrchestratorResponse resp } } + /// + internal override Task ResolveWorkItemPayloadsAsync(TResponse response, CancellationToken cancellation) + => this.ResolveResponsePayloadsCoreAsync(response, workItem: true, cancellation); + /// protected override async Task ExternalizeRequestPayloadsAsync(TRequest request, CancellationToken cancellation) { @@ -131,7 +135,10 @@ protected override async Task ExternalizeRequestPayloadsAsync(TRequest } /// - protected override async Task ResolveResponsePayloadsAsync(TResponse response, CancellationToken cancellation) + protected override Task ResolveResponsePayloadsAsync(TResponse response, CancellationToken cancellation) + => this.ResolveResponsePayloadsCoreAsync(response, workItem: response is P.WorkItem, cancellation); + + async Task ResolveResponsePayloadsCoreAsync(TResponse response, bool workItem, CancellationToken cancellation) { // Sidecar -> client/worker switch (response) @@ -151,7 +158,7 @@ protected override async Task ResolveResponsePayloadsAsync(TResponse { if (this.RequiresEventPayloadResolution(e)) { - operations.Add(() => this.ResolveEventPayloadsAsync(e, cancellation)); + operations.Add(() => this.ResolveEventPayloadsAsync(e, workItem, cancellation)); } } @@ -200,7 +207,7 @@ protected override async Task ResolveResponsePayloadsAsync(TResponse string? input = ar.Input; if (this.RequiresResolution(input)) { - operations.Add(async () => ar.Input = await this.MaybeResolveAsync(input, cancellation)); + operations.Add(async () => ar.Input = await this.ResolvePayloadAsync(input, workItem, cancellation)); } } @@ -211,7 +218,7 @@ protected override async Task ResolveResponsePayloadsAsync(TResponse { if (this.RequiresEventPayloadResolution(e)) { - operations.Add(() => this.ResolveEventPayloadsAsync(e, cancellation)); + operations.Add(() => this.ResolveEventPayloadsAsync(e, workItem, cancellation)); } } @@ -219,7 +226,7 @@ protected override async Task ResolveResponsePayloadsAsync(TResponse { if (this.RequiresEventPayloadResolution(e)) { - operations.Add(() => this.ResolveEventPayloadsAsync(e, cancellation)); + operations.Add(() => this.ResolveEventPayloadsAsync(e, workItem, cancellation)); } } } @@ -230,7 +237,7 @@ protected override async Task ResolveResponsePayloadsAsync(TResponse string? entityState = er1.EntityState; if (this.RequiresResolution(entityState)) { - operations.Add(async () => er1.EntityState = await this.MaybeResolveAsync(entityState, cancellation)); + operations.Add(async () => er1.EntityState = await this.ResolvePayloadAsync(entityState, workItem, cancellation)); } if (er1.Operations != null) @@ -240,7 +247,7 @@ protected override async Task ResolveResponsePayloadsAsync(TResponse string? input = op.Input; if (this.RequiresResolution(input)) { - operations.Add(async () => op.Input = await this.MaybeResolveAsync(input, cancellation)); + operations.Add(async () => op.Input = await this.ResolvePayloadAsync(input, workItem, cancellation)); } } } @@ -252,7 +259,7 @@ protected override async Task ResolveResponsePayloadsAsync(TResponse string? entityState = er2.EntityState; if (this.RequiresResolution(entityState)) { - operations.Add(async () => er2.EntityState = await this.MaybeResolveAsync(entityState, cancellation)); + operations.Add(async () => er2.EntityState = await this.ResolvePayloadAsync(entityState, workItem, cancellation)); } if (er2.OperationRequests != null) @@ -261,7 +268,7 @@ protected override async Task ResolveResponsePayloadsAsync(TResponse { if (this.RequiresEventPayloadResolution(opEvt)) { - operations.Add(() => this.ResolveEventPayloadsAsync(opEvt, cancellation)); + operations.Add(() => this.ResolveEventPayloadsAsync(opEvt, workItem, cancellation)); } } } @@ -601,126 +608,129 @@ bool RequiresEventPayloadResolution(P.HistoryEvent e) }; } - async Task ResolveEventPayloadsAsync(P.HistoryEvent e, CancellationToken cancellation) + Task ResolvePayloadAsync(string? value, bool workItem, CancellationToken cancellation) + => workItem ? this.MaybeResolveWorkItemAsync(value, cancellation) : this.MaybeResolveAsync(value, cancellation); + + async Task ResolveEventPayloadsAsync(P.HistoryEvent e, bool workItem, CancellationToken cancellation) { switch (e.EventTypeCase) { case P.HistoryEvent.EventTypeOneofCase.ExecutionStarted: if (e.ExecutionStarted is { } es) { - es.Input = await this.MaybeResolveAsync(es.Input, cancellation); + es.Input = await this.ResolvePayloadAsync(es.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.ExecutionCompleted: if (e.ExecutionCompleted is { } ec) { - ec.Result = await this.MaybeResolveAsync(ec.Result, cancellation); + ec.Result = await this.ResolvePayloadAsync(ec.Result, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.EventRaised: if (e.EventRaised is { } er) { - er.Input = await this.MaybeResolveAsync(er.Input, cancellation); + er.Input = await this.ResolvePayloadAsync(er.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.TaskScheduled: if (e.TaskScheduled is { } ts) { - ts.Input = await this.MaybeResolveAsync(ts.Input, cancellation); + ts.Input = await this.ResolvePayloadAsync(ts.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.TaskCompleted: if (e.TaskCompleted is { } tc) { - tc.Result = await this.MaybeResolveAsync(tc.Result, cancellation); + tc.Result = await this.ResolvePayloadAsync(tc.Result, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated: if (e.SubOrchestrationInstanceCreated is { } soc) { - soc.Input = await this.MaybeResolveAsync(soc.Input, cancellation); + soc.Input = await this.ResolvePayloadAsync(soc.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCompleted: if (e.SubOrchestrationInstanceCompleted is { } sox) { - sox.Result = await this.MaybeResolveAsync(sox.Result, cancellation); + sox.Result = await this.ResolvePayloadAsync(sox.Result, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.EventSent: if (e.EventSent is { } esent) { - esent.Input = await this.MaybeResolveAsync(esent.Input, cancellation); + esent.Input = await this.ResolvePayloadAsync(esent.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.GenericEvent: if (e.GenericEvent is { } ge) { - ge.Data = await this.MaybeResolveAsync(ge.Data, cancellation); + ge.Data = await this.ResolvePayloadAsync(ge.Data, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.ContinueAsNew: if (e.ContinueAsNew is { } can) { - can.Input = await this.MaybeResolveAsync(can.Input, cancellation); + can.Input = await this.ResolvePayloadAsync(can.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.ExecutionTerminated: if (e.ExecutionTerminated is { } et) { - et.Input = await this.MaybeResolveAsync(et.Input, cancellation); + et.Input = await this.ResolvePayloadAsync(et.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.ExecutionSuspended: if (e.ExecutionSuspended is { } esus) { - esus.Input = await this.MaybeResolveAsync(esus.Input, cancellation); + esus.Input = await this.ResolvePayloadAsync(esus.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.ExecutionResumed: if (e.ExecutionResumed is { } eres) { - eres.Input = await this.MaybeResolveAsync(eres.Input, cancellation); + eres.Input = await this.ResolvePayloadAsync(eres.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.ExecutionRewound: if (e.ExecutionRewound is { } erew) { - erew.Reason = await this.MaybeResolveAsync(erew.Reason, cancellation); + erew.Reason = await this.ResolvePayloadAsync(erew.Reason, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.EntityOperationSignaled: if (e.EntityOperationSignaled is { } eos) { - eos.Input = await this.MaybeResolveAsync(eos.Input, cancellation); + eos.Input = await this.ResolvePayloadAsync(eos.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.EntityOperationCalled: if (e.EntityOperationCalled is { } eoc) { - eoc.Input = await this.MaybeResolveAsync(eoc.Input, cancellation); + eoc.Input = await this.ResolvePayloadAsync(eoc.Input, workItem, cancellation); } break; case P.HistoryEvent.EventTypeOneofCase.EntityOperationCompleted: if (e.EntityOperationCompleted is { } ecomp) { - ecomp.Output = await this.MaybeResolveAsync(ecomp.Output, cancellation); + ecomp.Output = await this.ResolvePayloadAsync(ecomp.Output, workItem, cancellation); } break; @@ -729,9 +739,9 @@ async Task ResolveEventPayloadsAsync(P.HistoryEvent e, CancellationToken cancell { // A HistoryState event is one bounded operation, so its three fields remain // sequential. Concurrent history operations always target different messages. - os.Input = await this.MaybeResolveAsync(os.Input, cancellation); - os.Output = await this.MaybeResolveAsync(os.Output, cancellation); - os.CustomStatus = await this.MaybeResolveAsync(os.CustomStatus, cancellation); + os.Input = await this.ResolvePayloadAsync(os.Input, workItem, cancellation); + os.Output = await this.ResolvePayloadAsync(os.Output, workItem, cancellation); + os.CustomStatus = await this.ResolvePayloadAsync(os.CustomStatus, workItem, cancellation); } break; diff --git a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs index 9207456a3..1d74f0ff6 100644 --- a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs +++ b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs @@ -28,6 +28,8 @@ public abstract class PayloadInterceptor( readonly PayloadStore payloadStore = payloadStore; readonly LargePayloadStorageOptions options = options; + Action? DownloadTimeoutStartedForTest { get; set; } + // Unary: externalize on request, resolve on response /// @@ -141,12 +143,22 @@ public override AsyncServerStreamingCall AsyncServerStreamingCall call = continuation(request, context); + CancellationTokenSource lifetime = CancellationTokenSource.CreateLinkedTokenSource(context.Options.CancellationToken); + bool workerPayloads = request is P.GetWorkItemsRequest or P.StreamInstanceHistoryRequest { ForWorkItemProcessing: true }; + int disposed = 0; - IAsyncStreamReader wrapped = new TransformingStreamReader(call.ResponseStream, async (msg, ct) => + async ValueTask ResolveAsync(TResponse msg, CancellationToken ct) { try { - await this.ResolveResponsePayloadsAsync(msg, ct); + if (workerPayloads) + { + await this.ResolveWorkItemPayloadsAsync(msg, ct); + } + else + { + await this.ResolveResponsePayloadsAsync(msg, ct); + } } catch (Exception ex) when (msg is P.WorkItem workItem && workItem.RequestCase is P.WorkItem.RequestOneofCase.ActivityRequest @@ -159,16 +171,43 @@ or P.WorkItem.RequestOneofCase.EntityRequestV2 } return msg; - }); + } + + IAsyncStreamReader wrapped = new TransformingStreamReader( + call.ResponseStream, ResolveAsync, lifetime.Token); + + void Dispose() + { + if (Interlocked.Exchange(ref disposed, 1) == 0) + { + try + { + lifetime.Cancel(); + } + finally + { + lifetime.Dispose(); + call.Dispose(); + } + } + } return new AsyncServerStreamingCall( wrapped, call.ResponseHeadersAsync, call.GetStatus, call.GetTrailers, - call.Dispose); + Dispose); } + /// Resolves worker response payloads with per-download timeout handling. + /// The response type. + /// The received worker response. + /// Worker shutdown and call-disposal cancellation. + /// The payload resolution task. + internal virtual Task ResolveWorkItemPayloadsAsync(TResponse response, CancellationToken cancellation) + => this.ResolveResponsePayloadsAsync(response, cancellation); + /// /// Externalizes large payloads in request messages. /// @@ -220,6 +259,33 @@ or P.WorkItem.RequestOneofCase.EntityRequestV2 return await this.payloadStore.DownloadAsync(value!, cancellation); } + /// Resolves one worker payload with an independent budget that begins at download dispatch. + /// The value to resolve. + /// Worker shutdown and call-disposal cancellation. + /// The resolved value, or the unchanged value when it is not a token. + private protected async Task MaybeResolveWorkItemAsync(string? value, CancellationToken cancellation) + { + if (!this.RequiresResolution(value)) + { + return value; + } + + TimeSpan budget = TimeSpan.FromSeconds(120); + using CancellationTokenSource timeout = new(budget); + using CancellationTokenSource download = CancellationTokenSource.CreateLinkedTokenSource(cancellation, timeout.Token); + try + { + this.DownloadTimeoutStartedForTest?.Invoke(timeout, budget); + string payload = await this.payloadStore.DownloadAsync(value!, download.Token).ConfigureAwait(false); + download.Token.ThrowIfCancellationRequested(); + return payload; + } + catch (OperationCanceledException ex) when (timeout.IsCancellationRequested && !cancellation.IsCancellationRequested) + { + throw new TimeoutException("Work-item payload download exceeded its 120-second budget.", ex); + } + } + /// /// Externalizes a payload whose UTF-8 size has already been validated. /// @@ -279,7 +345,8 @@ private protected bool RequiresResolution(string? value) sealed class TransformingStreamReader( IAsyncStreamReader inner, - Func> transform) : IAsyncStreamReader + Func> transform, + CancellationToken lifetime) : IAsyncStreamReader, IWorkItemStreamReader { readonly IAsyncStreamReader inner = inner; readonly Func> transform = transform; @@ -288,14 +355,22 @@ sealed class TransformingStreamReader( public async Task MoveNext(CancellationToken cancellationToken) { - bool hasNext = await this.inner.MoveNext(cancellationToken); + bool hasNext = await this.MoveNextTransportAsync(cancellationToken); if (!hasNext) { return false; } - this.Current = await this.transform(this.inner.Current, cancellationToken); + await this.ResolveCurrentAsync(cancellationToken); return true; } + + public Task MoveNextTransportAsync(CancellationToken cancellation) => this.inner.MoveNext(cancellation); + + public async Task ResolveCurrentAsync(CancellationToken cancellation) + { + using CancellationTokenSource resolution = CancellationTokenSource.CreateLinkedTokenSource(cancellation, lifetime); + this.Current = await this.transform(this.inner.Current, resolution.Token); + } } } diff --git a/src/Extensions/AzureBlobPayloads/README.md b/src/Extensions/AzureBlobPayloads/README.md index 146700caf..c9670f284 100644 --- a/src/Extensions/AzureBlobPayloads/README.md +++ b/src/Extensions/AzureBlobPayloads/README.md @@ -178,6 +178,14 @@ entity payloads through `PayloadStore.DownloadAsync`. `BlobPayloadStore` enforce the same policy there and in direct `DeleteAsync` calls and auto-purge activities. Uploads are unaffected by broader read/delete scopes. +For worker deliveries, the receive watchdog times only transport reads. Each +active payload download has its own 120-second budget, not an aggregate limit +for the message or history. Receive-loop abandonment has one 30-second budget +across all attempts and backoff; expiry reconnects the stream. Failed deliveries +remain retryable. Worker shutdown and call disposal cancel active downloads. +Custom stores must honor cancellation; started downloads are always drained. +Management unary/history reads retain their existing cancellation behavior. + Custom `PayloadStore` implementations own their own access policy; these options govern `BlobPayloadStore`. This package does not implement a Storage proxy service or a dashboard. Dashboard reads using the signed-in user's delegated Storage diff --git a/src/Grpc/WorkItemPayloadException.cs b/src/Grpc/WorkItemPayloadException.cs index 6386098a6..7e0129488 100644 --- a/src/Grpc/WorkItemPayloadException.cs +++ b/src/Grpc/WorkItemPayloadException.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Grpc.Core; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask; @@ -18,3 +19,13 @@ internal sealed class WorkItemPayloadException(P.WorkItem workItem, Exception in /// public P.WorkItem WorkItem { get; } = workItem; } + +/// Separates transport receipt from payload hydration on a worker stream. +internal interface IWorkItemStreamReader +{ + /// Reads the next transport message without hydrating it. + Task MoveNextTransportAsync(CancellationToken cancellation); + + /// Hydrates the received message before exposing it as the current item. + Task ResolveCurrentAsync(CancellationToken cancellation); +} diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index e18cca78d..f3572d427 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -56,6 +56,8 @@ public Processor(GrpcDurableTaskWorker worker, TaskHubSidecarServiceClient clien ILogger Logger => this.worker.logger; + Action? AbandonTimeoutStartedForTest { get; set; } + public async Task ExecuteAsync(CancellationToken cancellation) { // Tracks consecutive failures against the same channel. Reset only after the stream @@ -629,29 +631,43 @@ async Task OnWorkItemPayloadFailureAsync(WorkItemPayloadException exception, Can workItem.EntityRequestV2?.InstanceId ?? string.Empty; this.Logger.UnexpectedError(exception, instanceId); + TimeSpan budget = TimeSpan.FromSeconds(30); + DateTime deadline = DateTime.UtcNow.Add(budget); + using CancellationTokenSource timeout = new(budget); + using CancellationTokenSource abandon = CancellationTokenSource.CreateLinkedTokenSource(cancellation, timeout.Token); try { + this.AbandonTimeoutStartedForTest?.Invoke(timeout, budget); await this.ExecuteWithRetryAsync( () => workItem.RequestCase switch { P.WorkItem.RequestOneofCase.OrchestratorRequest => this.client.AbandonTaskOrchestratorWorkItemAsync( new P.AbandonOrchestrationTaskRequest { CompletionToken = workItem.CompletionToken }, - cancellationToken: cancellation).ResponseAsync, + deadline: deadline, + cancellationToken: abandon.Token).ResponseAsync, P.WorkItem.RequestOneofCase.ActivityRequest => this.client.AbandonTaskActivityWorkItemAsync( new P.AbandonActivityTaskRequest { CompletionToken = workItem.CompletionToken }, - cancellationToken: cancellation).ResponseAsync, + deadline: deadline, + cancellationToken: abandon.Token).ResponseAsync, P.WorkItem.RequestOneofCase.EntityRequest or P.WorkItem.RequestOneofCase.EntityRequestV2 => this.client.AbandonTaskEntityWorkItemAsync( new P.AbandonEntityTaskRequest { CompletionToken = workItem.CompletionToken }, - cancellationToken: cancellation).ResponseAsync, + deadline: deadline, + cancellationToken: abandon.Token).ResponseAsync, _ => throw new InvalidOperationException($"Cannot abandon unresolved {workItem.RequestCase}."), }, "AbandonUnresolvedWorkItem", - cancellation); + abandon.Token); + abandon.Token.ThrowIfCancellationRequested(); + } + catch (Exception ex) when (timeout.IsCancellationRequested && !cancellation.IsCancellationRequested + && ex is OperationCanceledException or RpcException) + { + throw new TimeoutException("Payload failure abandonment exceeded its total 30-second budget.", ex); } - catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) + catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound && !cancellation.IsCancellationRequested) { // Ownership loss is delivery-local; other abandon failures must reach the reconnect loop. this.Logger.WorkItemOwnershipLost(workItem.RequestCase.ToString(), workItem.CompletionToken, ex); diff --git a/src/Worker/Grpc/WorkItemStreamConsumer.cs b/src/Worker/Grpc/WorkItemStreamConsumer.cs index 74164612d..d1d4e24db 100644 --- a/src/Worker/Grpc/WorkItemStreamConsumer.cs +++ b/src/Worker/Grpc/WorkItemStreamConsumer.cs @@ -101,6 +101,7 @@ void ArmSilentDisconnectTimer() cancellation, timeoutSource.Token); bool firstMessageObserved = false; + IWorkItemStreamReader? payloadReader = stream as IWorkItemStreamReader; void ObserveMessage() { @@ -117,14 +118,24 @@ void ObserveMessage() { try { - if (!await stream.MoveNext(tokenSource.Token).ConfigureAwait(false)) + bool hasNext = payloadReader is null + ? await stream.MoveNext(tokenSource.Token).ConfigureAwait(false) + : await payloadReader.MoveNextTransportAsync(tokenSource.Token).ConfigureAwait(false); + if (!hasNext) { break; } + + // The read watchdog measures only transport inactivity, never storage work. + timeoutSource.CancelAfter(Timeout.InfiniteTimeSpan); + ObserveMessage(); + if (payloadReader is not null) + { + await payloadReader.ResolveCurrentAsync(cancellation).ConfigureAwait(false); + } } catch (WorkItemPayloadException ex) when (onPayloadFailure is not null) { - // No reads can consume health pings while abandonment is pending. timeoutSource.CancelAfter(Timeout.InfiniteTimeSpan); ObserveMessage(); await onPayloadFailure(ex, cancellation).ConfigureAwait(false); @@ -133,7 +144,6 @@ void ObserveMessage() } ArmSilentDisconnectTimer(); - ObserveMessage(); onItem(stream.Current); } diff --git a/test/Grpc.IntegrationTests/AzureBlobPayloadsSideCarInterceptorTests.cs b/test/Grpc.IntegrationTests/AzureBlobPayloadsSideCarInterceptorTests.cs index 71a08a1c8..b9ceebc98 100644 --- a/test/Grpc.IntegrationTests/AzureBlobPayloadsSideCarInterceptorTests.cs +++ b/test/Grpc.IntegrationTests/AzureBlobPayloadsSideCarInterceptorTests.cs @@ -6,6 +6,9 @@ using Azure; using FluentAssertions; using Google.Protobuf; +using Grpc.Core; +using Grpc.Core.Interceptors; +using Moq; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Grpc.Tests; @@ -34,6 +37,116 @@ public sealed class AzureBlobPayloadsSideCarInterceptorTests static readonly PropertyInfo SharedMessageLockAcquiredForTestProperty = typeof(AzureBlobPayloadsSideCarInterceptor) .GetProperty("SharedMessageLockAcquiredForTest", BindingFlags.Instance | BindingFlags.NonPublic)!; + [Theory] + [InlineData("work-item")] + [InlineData("worker-history")] + [InlineData("management-history")] + [InlineData("unary")] + public async Task ResolvePayloads_WorkerDownloadsHaveIndependentBudgetsAsync(string responseKind) + { + // Arrange: advance a local clock 119 seconds per successful download. The three + // fields of one HistoryState operation must not share a 120-second aggregate budget. + TimeSpan elapsed = TimeSpan.Zero; + TimeSpan expires = TimeSpan.MaxValue; + CancellationTokenSource? active = null; + List deadlines = []; + TrackingPayloadStore store = new(downloadAsync: (token, cancellation) => + { + elapsed += TimeSpan.FromSeconds(119); + if (elapsed >= expires) + { + active!.Cancel(); + } + + cancellation.ThrowIfCancellationRequested(); + return Task.FromResult(token["test-blob://".Length..]); + }); + AzureBlobPayloadsSideCarInterceptor interceptor = new(store, CreateOptions()); + typeof(PayloadInterceptor).GetProperty("DownloadTimeoutStartedForTest", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(interceptor, (Action)((source, budget) => + { + Assert.Equal(TimeSpan.FromSeconds(120), budget); + active = source; + expires = elapsed + budget; + deadlines.Add(source); + })); + P.OrchestrationState state = new() + { + Input = "test-blob://input", + Output = "test-blob://output", + CustomStatus = "test-blob://status", + }; + P.HistoryEvent historyEvent = new() { HistoryState = new P.HistoryStateEvent { OrchestrationState = state } }; + + // Act + if (responseKind == "work-item") + { + using AsyncServerStreamingCall call = CreateStreamingCall(interceptor, new P.GetWorkItemsRequest(), new P.WorkItem + { + OrchestratorRequest = new P.OrchestratorRequest { PastEvents = { historyEvent } }, + }); + Assert.True(await call.ResponseStream.MoveNext(default)); + } + else if (responseKind == "unary") + { + Method method = new( + MethodType.Unary, "test", "GetInstance", + Marshallers.Create(_ => [], _ => new()), + Marshallers.Create(_ => [], _ => new())); + using AsyncUnaryCall call = interceptor.AsyncUnaryCall( + new P.GetInstanceRequest(), + new ClientInterceptorContext(method, null, default), + (_, _) => new AsyncUnaryCall( + Task.FromResult(new P.GetInstanceResponse { OrchestrationState = state }), + Task.FromResult(new Metadata()), () => Status.DefaultSuccess, () => new Metadata(), () => { })); + await call.ResponseAsync; + } + else + { + using AsyncServerStreamingCall call = CreateStreamingCall( + interceptor, + new P.StreamInstanceHistoryRequest { ForWorkItemProcessing = responseKind == "worker-history" }, + new P.HistoryChunk { Events = { historyEvent } }); + Assert.True(await call.ResponseStream.MoveNext(default)); + } + + // Assert + Assert.Equal(TimeSpan.FromSeconds(357), elapsed); + Assert.Equal(responseKind is "work-item" or "worker-history" ? 3 : 0, deadlines.Distinct().Count()); + Assert.Equal("input", state.Input); + Assert.Equal("output", state.Output); + Assert.Equal("status", state.CustomStatus); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StreamingCall_DisposeCancelsActiveHydrationAsync(bool workerHistory) + { + // Arrange + TaskCompletionSource downloading = new(TaskCreationOptions.RunContinuationsAsynchronously); + TrackingPayloadStore store = new(downloadAsync: async (_, cancellation) => + { + downloading.TrySetResult(); + await Task.Delay(Timeout.Infinite, cancellation); + return string.Empty; + }); + AzureBlobPayloadsSideCarInterceptor interceptor = new(store, CreateOptions()); + using AsyncServerStreamingCall call = CreateStreamingCall( + interceptor, + new P.StreamInstanceHistoryRequest { ForWorkItemProcessing = workerHistory }, + new P.HistoryChunk { Events = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = "test-blob://input" } } } }); + + // Act + Task next = call.ResponseStream.MoveNext(default); + await downloading.Task.WaitAsync(TimeSpan.FromSeconds(5)); + call.Dispose(); + + // Assert + await Assert.ThrowsAnyAsync(() => next.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(1, store.DownloadCount); + } + [Fact] public async Task ResolveResponsePayloadsAsync_HistoryChunk_ResolvesEventsInOrderWithBoundedConcurrency() { @@ -2445,6 +2558,25 @@ static IReadOnlyList GetHistoryEventPayloads(P.HistoryEvent historyEvent static LargePayloadStorageOptions CreateOptions() => new() { ThresholdBytes = 1 }; + static AsyncServerStreamingCall CreateStreamingCall( + AzureBlobPayloadsSideCarInterceptor interceptor, TRequest request, TResponse response) + where TRequest : class, new() + where TResponse : class, new() + { + Mock> reader = new(); + reader.SetupSequence(r => r.MoveNext(It.IsAny())).ReturnsAsync(true).ReturnsAsync(false); + reader.SetupGet(r => r.Current).Returns(response); + Method method = new( + MethodType.ServerStreaming, "test", "Read", + Marshallers.Create(_ => [], _ => new()), + Marshallers.Create(_ => [], _ => new())); + return interceptor.AsyncServerStreamingCall( + request, + new ClientInterceptorContext(method, null, default), + (_, _) => new AsyncServerStreamingCall( + reader.Object, Task.FromResult(new Metadata()), () => Status.DefaultSuccess, () => new Metadata(), () => { })); + } + static Task ExternalizeAsync(AzureBlobPayloadsSideCarInterceptor interceptor, TRequest request, CancellationToken cancellation) => (Task)ExternalizeMethodDefinition.MakeGenericMethod(typeof(TRequest)).Invoke(interceptor, [request, cancellation])!; diff --git a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs index 400e062e4..b904f9217 100644 --- a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs +++ b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs @@ -254,6 +254,298 @@ async Task WaitForReleaseAsync(CancellationToken cancellation) } } + [Fact] + public async Task ConsumeAsync_LateMessage_DoesNotTimeOutHealthyHydrationAsync() + { + // Arrange + TimeSpan readTimeout = TimeSpan.FromMilliseconds(500); + TaskCompletionSource downloading = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + P.WorkItem item = new() { ActivityRequest = new P.ActivityRequest { Input = "payload-token" } }; + int reads = 0; + Mock> reader = new(); + reader.SetupGet(r => r.Current).Returns(item); + reader.Setup(r => r.MoveNext(It.IsAny())) + .Returns(async (CancellationToken cancellation) => + { + if (++reads != 1) + { + return false; + } + + await Task.Delay(readTimeout / 2, cancellation); + return true; + }); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken("payload-token")).Returns(true); + store.Setup(s => s.DownloadAsync("payload-token", It.IsAny())) + .Returns(async (string _, CancellationToken cancellation) => + { + downloading.TrySetResult(); + await release.Task.WaitAsync(cancellation); + return "\"hydrated\""; + }); + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (_, _) => CreateServerStreamingCall(reader.Object)); + P.TaskHubSidecarService.TaskHubSidecarServiceClient client = new(invoker.Intercept( + new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions()))); + using CancellationTokenSource shutdown = new(); + using AsyncServerStreamingCall call = client.GetWorkItems(new(), cancellationToken: shutdown.Token); + List received = []; + int observations = 0; + + // Act + Task consume = WorkItemStreamConsumer.ConsumeAsync( + call.ResponseStream, readTimeout, received.Add, () => observations++, shutdown.Token); + try + { + await downloading.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Task.Delay(readTimeout); + Assert.False(consume.IsCompleted); + Assert.Equal(1, observations); + release.TrySetResult(); + WorkItemStreamResult result = await consume.WaitAsync(TimeSpan.FromSeconds(5)); + + // Assert + Assert.Equal(WorkItemStreamOutcome.GracefulDrain, result.Outcome); + Assert.Equal("\"hydrated\"", Assert.Single(received).ActivityRequest.Input); + Assert.Equal(1, invoker.GetWorkItemsCallCount); + } + finally + { + shutdown.Cancel(); + release.TrySetResult(); + await consume; + } + } + + [Fact] + public async Task ProcessorExecuteAsync_DownloadTimeout_DrainsThenAbandonsOnSameStreamAsync() + { + // Arrange + using CancellationTokenSource shutdown = new(); + TaskCompletionSource allDownloading = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource downloadCancelled = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource drain = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource healthyCompleted = new(TaskCreationOptions.RunContinuationsAsynchronously); + P.WorkItem failed = new() + { + CompletionToken = "failed", + OrchestratorRequest = new P.OrchestratorRequest + { + NewEvents = + { + new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = "blocked-token" } }, + new P.HistoryEvent { EventRaised = new P.EventRaisedEvent { Input = "stalled-token" } }, + }, + }, + }; + P.WorkItem healthy = new() + { + CompletionToken = "healthy", + ActivityRequest = new P.ActivityRequest + { + Name = "MyActivity", Input = "\"healthy\"", + OrchestrationInstance = new P.OrchestrationInstance { InstanceId = "instance", ExecutionId = "execution" }, + }, + }; + Queue deliveries = new([failed, healthy]); + P.WorkItem current = null!; + Mock> reader = new(); + reader.SetupGet(r => r.Current).Returns(() => current); + reader.Setup(r => r.MoveNext(It.IsAny())).Returns(async (CancellationToken cancellation) => + { + if (deliveries.TryDequeue(out P.WorkItem? item)) + { + current = item; + return true; + } + + await Task.Delay(Timeout.Infinite, cancellation); + return false; + }); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value.EndsWith("-token")); + store.Setup(s => s.DownloadAsync(It.IsAny(), It.IsAny())) + .Returns(async (string token, CancellationToken cancellation) => + { + if (token == "blocked-token") + { + await drain.Task.WaitAsync(cancellation); + return "\"drained\""; + } + + allDownloading.TrySetResult(); + try + { + await Task.Delay(Timeout.Infinite, cancellation); + } + finally + { + downloadCancelled.TrySetResult(); + } + + return string.Empty; + }); + AzureBlobPayloadsSideCarInterceptor interceptor = new(store.Object, new LargePayloadStorageOptions()); + List deadlines = []; + typeof(PayloadInterceptor).GetProperty("DownloadTimeoutStartedForTest", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(interceptor, (Action)((source, budget) => + { + Assert.Equal(TimeSpan.FromSeconds(120), budget); + deadlines.Add(source); + })); + int abandons = 0; + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (_, _) => CreateServerStreamingCall(reader.Object)) + { + UnaryResponse = (operation, request, _) => + { + if (operation == "CompleteActivityTask") + { + healthyCompleted.TrySetResult(Assert.IsType(request)); + return Task.FromResult(new P.CompleteTaskResponse()); + } + + Assert.Equal("AbandonTaskOrchestratorWorkItem", operation); + Assert.Equal("failed", Assert.IsType(request).CompletionToken); + abandons++; + return Task.FromResult(new P.AbandonOrchestrationTaskResponse()); + }, + }; + using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); + Mock factory = new(); + ITaskActivity? activity = new TestActivity(); + factory.Setup(f => f.TryCreateActivity(It.IsAny(), It.IsAny(), out activity)).Returns(true); + GrpcDurableTaskWorker worker = CreateWorker(new(), new(), NullLoggerFactory.Instance, factory.Object, services); + object processor = CreateProcessor(worker, new P.TaskHubSidecarService.TaskHubSidecarServiceClient(invoker.Intercept(interceptor))); + + // Act + Task processing = InvokeProcessorExecuteAsync(processor, shutdown.Token); + try + { + await allDownloading.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(2, deadlines.Count); + deadlines[1].Cancel(); + await downloadCancelled.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.False(processing.IsCompleted); + Assert.Equal(0, abandons); + drain.TrySetResult(); + P.ActivityResponse completed = await healthyCompleted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Assert + Assert.Equal("healthy", completed.CompletionToken); + Assert.Equal("\"healthy\"", completed.Result); + Assert.Null(completed.FailureDetails); + Assert.Equal(1, abandons); + Assert.Equal(1, invoker.GetWorkItemsCallCount); + Assert.Equal("\"drained\"", failed.OrchestratorRequest.NewEvents[0].ExecutionStarted.Input); + Assert.Equal("stalled-token", failed.OrchestratorRequest.NewEvents[1].EventRaised.Input); + } + finally + { + shutdown.Cancel(); + drain.TrySetResult(); + await processing.WaitAsync(TimeSpan.FromSeconds(5)); + } + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public async Task ProcessorExecuteAsync_AbandonTimeout_BoundsRpcRetriesAndBackoffAsync(int mode) + { + // Arrange + using CancellationTokenSource shutdown = new(); + TaskCompletionSource abandoning = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource reconnected = new(TaskCreationOptions.RunContinuationsAsynchronously); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken("payload-token")).Returns(true); + store.Setup(s => s.DownloadAsync("payload-token", It.IsAny())).ThrowsAsync(new IOException("Download failed.")); + List attempts = []; + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (number, _) => + { + if (number == 1) + { + return CreateServerStreamingCall(new SequenceAsyncStreamReader(new P.WorkItem + { + CompletionToken = "failed", + ActivityRequest = new P.ActivityRequest { Input = "payload-token" }, + })); + } + + reconnected.TrySetResult(); + return CreateServerStreamingCall(new HangingAsyncStreamReader(throwAsRpc: false)); + }) + { + UnaryResponse = async (operation, request, options) => + { + Assert.Equal("AbandonTaskActivityWorkItem", operation); + Assert.Equal("failed", Assert.IsType(request).CompletionToken); + attempts.Add(options); + if (mode == 2 || (mode == 1 && attempts.Count == 1)) + { + if (mode == 2) + { + abandoning.TrySetResult(); + } + + throw new RpcException(new Status(StatusCode.Unavailable, "Retry abandon.")); + } + + abandoning.TrySetResult(); + await Task.Delay(Timeout.Infinite, options.CancellationToken); + return new P.AbandonActivityTaskResponse(); + }, + }; + GrpcDurableTaskWorkerOptions grpcOptions = new(); + grpcOptions.Internal.SilentDisconnectTimeout = TimeSpan.Zero; + grpcOptions.Internal.ReconnectBackoffBase = grpcOptions.Internal.ReconnectBackoffCap = TimeSpan.Zero; + grpcOptions.Internal.TransientRetryBackoffBase = grpcOptions.Internal.TransientRetryBackoffCap = + mode == 2 ? TimeSpan.FromMinutes(1) : TimeSpan.Zero; + object processor = CreateProcessor(CreateWorker(grpcOptions), new P.TaskHubSidecarService.TaskHubSidecarServiceClient( + invoker.Intercept(new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions())))); + CancellationTokenSource? deadline = null; + int budgets = 0; + processor.GetType().GetProperty("AbandonTimeoutStartedForTest", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(processor, (Action)((source, budget) => + { + Assert.Equal(TimeSpan.FromSeconds(30), budget); + deadline = source; + budgets++; + })); + + // Act + DateTime started = DateTime.UtcNow; + Task processing = InvokeProcessorExecuteAsync(processor, shutdown.Token); + try + { + await abandoning.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, budgets); + Assert.Equal(mode == 1 ? 2 : 1, attempts.Count); + DateTime rpcDeadline = Assert.Single(attempts.Select(attempt => attempt.Deadline).Distinct())!.Value; + Assert.InRange(rpcDeadline, started.AddSeconds(29), DateTime.UtcNow.AddSeconds(30)); + Assert.Single(attempts.Select(attempt => attempt.CancellationToken).Distinct()); + Assert.IsType(deadline).Cancel(); + await reconnected.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Assert + Assert.All(attempts, attempt => Assert.True(attempt.CancellationToken.IsCancellationRequested)); + Assert.Equal(mode == 1 ? 2 : 1, attempts.Count); + Assert.Equal(2, invoker.GetWorkItemsCallCount); + } + finally + { + shutdown.Cancel(); + await processing.WaitAsync(TimeSpan.FromSeconds(5)); + } + } + [Fact] public async Task ExecuteAsync_ConnectFailureThreshold_RecreatesConfiguredChannel() { From 325f6bbdb714ef21b7749550fa070d8474d23da8 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 11 Nov 2026 09:27:36 -0800 Subject: [PATCH 7/9] fix: clarify payload contracts and own each read deadline Move worker-extension infrastructure out of the protobuf assembly and remove its friend access. Preserve protected interceptor overrides and async behavior, simplify call disposal and common-token allocation, and remove timeout test callbacks. Use standard clock scheduling for deterministic deadline, cancellation, and external-subclass regressions; successful transport reads no longer poison later reads with a fired watchdog. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Directory.Packages.props | 1 + ...rkerBuilderExtensions.AzureBlobPayloads.cs | 7 +- .../AzureBlobPayloadsSideCarInterceptor.cs | 4 +- .../Interceptors/PayloadInterceptor.cs | 64 ++++++--- src/Grpc/Grpc.csproj | 5 - src/Grpc/WorkItemPayloadException.cs | 31 ---- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 21 ++- .../Grpc/Internal/WorkItemPayloadException.cs | 37 +++++ src/Worker/Grpc/WorkItemStreamConsumer.cs | 135 +++++++----------- .../BlobPayloadStoreRoutingTests.cs | 3 +- ...zureBlobPayloadsSideCarInterceptorTests.cs | 33 ++--- .../Grpc.IntegrationTests.csproj | 4 + .../Grpc.Tests/GrpcDurableTaskWorkerTests.cs | 101 +++++++------ .../Grpc.Tests/PayloadInterceptorTests.cs | 103 +++++++++++++ .../Grpc.Tests/WorkItemStreamConsumerTests.cs | 65 ++++----- .../Grpc.Tests/Worker.Grpc.Tests.csproj | 1 + 16 files changed, 366 insertions(+), 249 deletions(-) delete mode 100644 src/Grpc/WorkItemPayloadException.cs create mode 100644 src/Worker/Grpc/Internal/WorkItemPayloadException.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 15630f4f4..feacb1feb 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -85,6 +85,7 @@ + diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index 62ec0f193..474a81337 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -101,7 +101,12 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB // Register an interceptor rather than moving Channel onto an intercepted CallInvoker. // Clearing Channel would disable the worker's gRPC channel recreation, and requiring a // pre-built Channel/CallInvoker would rule out the Address-only configuration. - AzureBlobPayloadsSideCarInterceptor interceptor = new(store, opts); + AzureBlobPayloadsSideCarInterceptor interceptor = new(store, opts) + { +#if NET8_0_OR_GREATER + TimeProvider = services.GetService() ?? TimeProvider.System, +#endif + }; opt.Interceptors.Add(interceptor); // Storage failures must become terminal responses before the worker releases entity locks. diff --git a/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs b/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs index 4a9bfe429..01e1a1fbf 100644 --- a/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs +++ b/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs @@ -135,8 +135,8 @@ protected override async Task ExternalizeRequestPayloadsAsync(TRequest } /// - protected override Task ResolveResponsePayloadsAsync(TResponse response, CancellationToken cancellation) - => this.ResolveResponsePayloadsCoreAsync(response, workItem: response is P.WorkItem, cancellation); + protected override async Task ResolveResponsePayloadsAsync(TResponse response, CancellationToken cancellation) + => await this.ResolveResponsePayloadsCoreAsync(response, workItem: response is P.WorkItem, cancellation); async Task ResolveResponsePayloadsCoreAsync(TResponse response, bool workItem, CancellationToken cancellation) { diff --git a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs index 1d74f0ff6..ef18335a5 100644 --- a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs +++ b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs @@ -4,6 +4,7 @@ using System.Text; using Grpc.Core; using Grpc.Core.Interceptors; +using Microsoft.DurableTask.Worker.Grpc.Internal; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask; @@ -28,7 +29,10 @@ public abstract class PayloadInterceptor( readonly PayloadStore payloadStore = payloadStore; readonly LargePayloadStorageOptions options = options; - Action? DownloadTimeoutStartedForTest { get; set; } +#if NET8_0_OR_GREATER + /// Gets the clock used for active worker download deadlines. + internal TimeProvider TimeProvider { get; init; } = TimeProvider.System; +#endif // Unary: externalize on request, resolve on response @@ -174,21 +178,17 @@ or P.WorkItem.RequestOneofCase.EntityRequestV2 } IAsyncStreamReader wrapped = new TransformingStreamReader( - call.ResponseStream, ResolveAsync, lifetime.Token); + call.ResponseStream, ResolveAsync, lifetime.Token, context.Options.CancellationToken); void Dispose() { if (Interlocked.Exchange(ref disposed, 1) == 0) { - try + using (lifetime) + using (call) { lifetime.Cancel(); } - finally - { - lifetime.Dispose(); - call.Dispose(); - } } } @@ -271,11 +271,14 @@ internal virtual Task ResolveWorkItemPayloadsAsync(TResponse response } TimeSpan budget = TimeSpan.FromSeconds(120); +#if NET8_0_OR_GREATER + using CancellationTokenSource timeout = new(budget, this.TimeProvider); +#else using CancellationTokenSource timeout = new(budget); +#endif using CancellationTokenSource download = CancellationTokenSource.CreateLinkedTokenSource(cancellation, timeout.Token); try { - this.DownloadTimeoutStartedForTest?.Invoke(timeout, budget); string payload = await this.payloadStore.DownloadAsync(value!, download.Token).ConfigureAwait(false); download.Token.ThrowIfCancellationRequested(); return payload; @@ -346,31 +349,50 @@ private protected bool RequiresResolution(string? value) sealed class TransformingStreamReader( IAsyncStreamReader inner, Func> transform, - CancellationToken lifetime) : IAsyncStreamReader, IWorkItemStreamReader + CancellationToken lifetime, + CancellationToken callCancellation) : IAsyncStreamReader, IWorkItemStreamReader { readonly IAsyncStreamReader inner = inner; readonly Func> transform = transform; public T Current { get; private set; } = default!; - public async Task MoveNext(CancellationToken cancellationToken) + public Task MoveNext(CancellationToken cancellationToken) => + this.MoveNextAsync(null, cancellationToken, cancellationToken); + + public async Task MoveNextAsync( + Action? onMessageReceived, CancellationToken readCancellation, CancellationToken payloadCancellation) { - bool hasNext = await this.MoveNextTransportAsync(cancellationToken); + bool hasNext; + if (this.inner is IWorkItemStreamReader payloadReader) + { + hasNext = await payloadReader.MoveNextAsync(onMessageReceived, readCancellation, payloadCancellation).ConfigureAwait(false); + } + else + { + hasNext = await this.inner.MoveNext(readCancellation).ConfigureAwait(false); + if (hasNext) + { + onMessageReceived?.Invoke(); + } + } + if (!hasNext) { return false; } - await this.ResolveCurrentAsync(cancellationToken); - return true; - } - - public Task MoveNextTransportAsync(CancellationToken cancellation) => this.inner.MoveNext(cancellation); + if (!payloadCancellation.CanBeCanceled || payloadCancellation == callCancellation || payloadCancellation == lifetime) + { + this.Current = await this.transform(this.inner.Current, lifetime).ConfigureAwait(false); + } + else + { + using CancellationTokenSource resolution = CancellationTokenSource.CreateLinkedTokenSource(payloadCancellation, lifetime); + this.Current = await this.transform(this.inner.Current, resolution.Token).ConfigureAwait(false); + } - public async Task ResolveCurrentAsync(CancellationToken cancellation) - { - using CancellationTokenSource resolution = CancellationTokenSource.CreateLinkedTokenSource(cancellation, lifetime); - this.Current = await this.transform(this.inner.Current, resolution.Token); + return true; } } } diff --git a/src/Grpc/Grpc.csproj b/src/Grpc/Grpc.csproj index f2e1ef3e2..e433e01ef 100644 --- a/src/Grpc/Grpc.csproj +++ b/src/Grpc/Grpc.csproj @@ -21,9 +21,4 @@ - - - - - diff --git a/src/Grpc/WorkItemPayloadException.cs b/src/Grpc/WorkItemPayloadException.cs deleted file mode 100644 index 7e0129488..000000000 --- a/src/Grpc/WorkItemPayloadException.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Grpc.Core; -using P = Microsoft.DurableTask.Protobuf; - -namespace Microsoft.DurableTask; - -/// -/// A received delivery whose payloads could not be resolved. The item is only safe to abandon, not dispatch. -/// -/// The failed delivery, including its original completion token. -/// The payload resolution failure. -internal sealed class WorkItemPayloadException(P.WorkItem workItem, Exception innerException) - : Exception("Failed to resolve payloads for a work-item delivery.", innerException) -{ - /// - /// Gets the failed delivery. Its payload fields may be partially resolved. - /// - public P.WorkItem WorkItem { get; } = workItem; -} - -/// Separates transport receipt from payload hydration on a worker stream. -internal interface IWorkItemStreamReader -{ - /// Reads the next transport message without hydrating it. - Task MoveNextTransportAsync(CancellationToken cancellation); - - /// Hydrates the received message before exposing it as the current item. - Task ResolveCurrentAsync(CancellationToken cancellation); -} diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index f3572d427..4994000d3 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -38,6 +38,9 @@ class Processor readonly TaskHubSidecarServiceClient client; readonly DurableTaskShimFactory shimFactory; readonly GrpcDurableTaskWorkerOptions.InternalOptions internalOptions; +#if NET8_0_OR_GREATER + readonly TimeProvider timeProvider; +#endif readonly DTCore.IExceptionPropertiesProvider? exceptionPropertiesProvider; [Obsolete("Experimental")] readonly IOrchestrationFilter? orchestrationFilter; @@ -48,6 +51,9 @@ public Processor(GrpcDurableTaskWorker worker, TaskHubSidecarServiceClient clien this.client = client; this.shimFactory = new DurableTaskShimFactory(this.worker.grpcOptions, this.worker.loggerFactory); this.internalOptions = this.worker.grpcOptions.Internal; +#if NET8_0_OR_GREATER + this.timeProvider = worker.services.GetService() ?? TimeProvider.System; +#endif this.orchestrationFilter = orchestrationFilter; this.exceptionPropertiesProvider = exceptionPropertiesProvider is not null ? new ExceptionPropertiesProviderAdapter(exceptionPropertiesProvider) @@ -56,8 +62,6 @@ public Processor(GrpcDurableTaskWorker worker, TaskHubSidecarServiceClient clien ILogger Logger => this.worker.logger; - Action? AbandonTimeoutStartedForTest { get; set; } - public async Task ExecuteAsync(CancellationToken cancellation) { // Tracks consecutive failures against the same channel. Reset only after the stream @@ -397,6 +401,9 @@ async Task ProcessWorkItemsAsync( workItem => this.DispatchWorkItem(workItem, cancellation), onFirstMessage, cancellation, +#if NET8_0_OR_GREATER + timeProvider: this.timeProvider, +#endif onPayloadFailure: this.OnWorkItemPayloadFailureAsync); switch (result.Outcome) @@ -632,12 +639,16 @@ async Task OnWorkItemPayloadFailureAsync(WorkItemPayloadException exception, Can string.Empty; this.Logger.UnexpectedError(exception, instanceId); TimeSpan budget = TimeSpan.FromSeconds(30); +#if NET8_0_OR_GREATER + DateTime deadline = this.timeProvider.GetUtcNow().UtcDateTime.Add(budget); + using CancellationTokenSource timeout = new(budget, this.timeProvider); +#else DateTime deadline = DateTime.UtcNow.Add(budget); using CancellationTokenSource timeout = new(budget); +#endif using CancellationTokenSource abandon = CancellationTokenSource.CreateLinkedTokenSource(cancellation, timeout.Token); try { - this.AbandonTimeoutStartedForTest?.Invoke(timeout, budget); await this.ExecuteWithRetryAsync( () => workItem.RequestCase switch { @@ -1318,7 +1329,11 @@ async Task ExecuteWithRetryAsync( try { +#if NET8_0_OR_GREATER + await Task.Delay(backoff, this.timeProvider, cancellationToken); +#else await Task.Delay(backoff, cancellationToken); +#endif } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { diff --git a/src/Worker/Grpc/Internal/WorkItemPayloadException.cs b/src/Worker/Grpc/Internal/WorkItemPayloadException.cs new file mode 100644 index 000000000..abb5216c1 --- /dev/null +++ b/src/Worker/Grpc/Internal/WorkItemPayloadException.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Worker.Grpc.Internal; + +/// Allows a worker extension to separate transport and payload cancellation on a work-item stream. +/// +/// Infrastructure contract used by payload extensions. This is not a supported application API. +/// +public interface IWorkItemStreamReader +{ + /// Receives and hydrates the next delivery using separate cancellation boundaries. + /// Invoked after a successful transport read, before payload resolution. + /// Cancellation for the underlying transport read. + /// Worker cancellation for payload resolution. + /// Whether a fully hydrated delivery is available. + Task MoveNextAsync(Action? onMessageReceived, CancellationToken readCancellation, CancellationToken payloadCancellation); +} + +/// +/// A received delivery whose payloads could not be resolved. The item is only safe to abandon, not dispatch. +/// +/// The failed delivery, including its original completion token. +/// The payload resolution failure. +/// +/// Infrastructure contract between worker extensions and the gRPC worker. This is not a supported application API. +/// +public sealed class WorkItemPayloadException(P.WorkItem workItem, Exception innerException) + : Exception("Failed to resolve payloads for a work-item delivery.", innerException) +{ + /// + /// Gets the failed delivery. Its payload fields may be partially resolved. + /// + public P.WorkItem WorkItem { get; } = workItem; +} diff --git a/src/Worker/Grpc/WorkItemStreamConsumer.cs b/src/Worker/Grpc/WorkItemStreamConsumer.cs index d1d4e24db..e45dfa78e 100644 --- a/src/Worker/Grpc/WorkItemStreamConsumer.cs +++ b/src/Worker/Grpc/WorkItemStreamConsumer.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Microsoft.DurableTask.Worker.Grpc.Internal; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Worker.Grpc; @@ -40,66 +41,43 @@ internal static class WorkItemStreamConsumer // values are still treated as "effectively infinite" without depending on framework-specific edge cases. static readonly TimeSpan MaxSupportedCancelAfterTimeout = TimeSpan.FromMilliseconds(int.MaxValue - 1d); - /// - /// Consume a work-item stream until shutdown, silent disconnect, or graceful drain. - /// - /// The response reader. Each read receives the linked cancellation token. - /// - /// How long to wait between successive items (or health pings) before treating the stream as - /// silently disconnected. Non-positive values disable detection entirely. - /// - /// - /// Synchronous per-item dispatch. Invoked once per delivered work item, after the silent-disconnect - /// timer has been re-armed. - /// - /// - /// Optional callback invoked exactly once when the first message is observed. Used by callers to - /// reset retry counters that should only count consecutive transport failures. - /// - /// Outer worker cancellation token. - /// - /// Test-only observability hook invoked synchronously every time the silent-disconnect timer is - /// (re-)armed -- once before the read loop starts, and once per item, immediately before that - /// item is dispatched. Always in production; lets tests prove the - /// per-item reset actually happens (and in what order relative to dispatch) without depending on - /// real elapsed time. Never invoked when disables - /// detection. - /// - /// - /// Optional delivery-local recovery outside the read watchdog, using the worker shutdown token. - /// Exceptions from recovery propagate to the caller for connection-level handling. - /// - /// The classified outcome plus whether any message was observed. +#if NET8_0_OR_GREATER + /// Consumes a work-item stream until shutdown, silent disconnect, or graceful drain. + /// The response reader. + /// Transport inactivity timeout; non-positive values disable detection. + /// Synchronous dispatch after successful payload resolution. + /// Invoked once on the first successful transport receipt. + /// Worker shutdown cancellation. + /// Clock used for read deadlines. + /// Delivery recovery using worker cancellation; its errors propagate to the caller. + /// The termination outcome and whether a message was received. +#else + /// Consumes a work-item stream until shutdown, silent disconnect, or graceful drain. + /// The response reader. + /// Transport inactivity timeout; non-positive values disable detection. + /// Synchronous dispatch after successful payload resolution. + /// Invoked once on the first successful transport receipt. + /// Worker shutdown cancellation. + /// Delivery recovery using worker cancellation; its errors propagate to the caller. + /// The termination outcome and whether a message was received. +#endif public static async Task ConsumeAsync( IAsyncStreamReader stream, TimeSpan silentDisconnectTimeout, Action onItem, Action? onFirstMessage, CancellationToken cancellation, - Action? onSilentDisconnectTimerArmed = null, +#if NET8_0_OR_GREATER + TimeProvider? timeProvider = null, +#endif Func? onPayloadFailure = null) { bool silentDisconnectEnabled = silentDisconnectTimeout > TimeSpan.Zero; - // Clamp enormous values once up-front so the timer-reset path can simply re-arm the same window. + // Use a fresh deadline for each read. Successful transport receipt wins even if its + // timeout fires before the awaiting continuation runs; it must not poison later reads. TimeSpan effectiveTimeout = ClampCancelAfterTimeout(silentDisconnectTimeout); - using CancellationTokenSource timeoutSource = new(); - void ArmSilentDisconnectTimer() - { - if (silentDisconnectEnabled) - { - timeoutSource.CancelAfter(effectiveTimeout); - onSilentDisconnectTimerArmed?.Invoke(); - } - } - - // Arm once before reading so the initial gap before the first message is also bounded. - ArmSilentDisconnectTimer(); - - using CancellationTokenSource tokenSource = CancellationTokenSource.CreateLinkedTokenSource( - cancellation, timeoutSource.Token); - bool firstMessageObserved = false; IWorkItemStreamReader? payloadReader = stream as IWorkItemStreamReader; @@ -116,34 +94,49 @@ void ObserveMessage() { while (true) { +#if NET8_0_OR_GREATER + using CancellationTokenSource timeoutSource = new( + silentDisconnectEnabled ? effectiveTimeout : Timeout.InfiniteTimeSpan, + timeProvider ?? TimeProvider.System); +#else + using CancellationTokenSource timeoutSource = new( + silentDisconnectEnabled ? effectiveTimeout : Timeout.InfiniteTimeSpan); +#endif + using CancellationTokenSource tokenSource = CancellationTokenSource.CreateLinkedTokenSource( + cancellation, timeoutSource.Token); + void OnMessageReceived() + { + timeoutSource.CancelAfter(Timeout.InfiniteTimeSpan); + ObserveMessage(); + } + try { bool hasNext = payloadReader is null ? await stream.MoveNext(tokenSource.Token).ConfigureAwait(false) - : await payloadReader.MoveNextTransportAsync(tokenSource.Token).ConfigureAwait(false); + : await payloadReader.MoveNextAsync(OnMessageReceived, tokenSource.Token, cancellation).ConfigureAwait(false); if (!hasNext) { break; } - // The read watchdog measures only transport inactivity, never storage work. - timeoutSource.CancelAfter(Timeout.InfiniteTimeSpan); - ObserveMessage(); - if (payloadReader is not null) - { - await payloadReader.ResolveCurrentAsync(cancellation).ConfigureAwait(false); - } + OnMessageReceived(); } catch (WorkItemPayloadException ex) when (onPayloadFailure is not null) { - timeoutSource.CancelAfter(Timeout.InfiniteTimeSpan); - ObserveMessage(); + OnMessageReceived(); await onPayloadFailure(ex, cancellation).ConfigureAwait(false); - ArmSilentDisconnectTimer(); continue; } - - ArmSilentDisconnectTimer(); + catch (OperationCanceledException) when (timeoutSource.IsCancellationRequested && !cancellation.IsCancellationRequested) + { + return new WorkItemStreamResult(WorkItemStreamOutcome.SilentDisconnect, firstMessageObserved); + } + catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled + && timeoutSource.IsCancellationRequested && !cancellation.IsCancellationRequested) + { + return new WorkItemStreamResult(WorkItemStreamOutcome.SilentDisconnect, firstMessageObserved); + } onItem(stream.Current); } @@ -152,32 +145,12 @@ void ObserveMessage() { // Worker is shutting down. } - catch (OperationCanceledException) when (timeoutSource.IsCancellationRequested) - { - // Silent-disconnect timer fired and grpc-dotnet surfaced cancellation as OCE - // (when GrpcChannelOptions.ThrowOperationCanceledOnCancellation == true). - } - catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled - && timeoutSource.IsCancellationRequested - && !cancellation.IsCancellationRequested) - { - // Silent-disconnect timer fired mid-MoveNext. By default - // (GrpcChannelOptions.ThrowOperationCanceledOnCancellation == false), grpc-dotnet - // surfaces the linked cancellation as RpcException(Cancelled) rather than OCE. - // Without this catch the exception would propagate past the silent-disconnect - // branch and the recreate path would never fire. - } if (cancellation.IsCancellationRequested) { return new WorkItemStreamResult(WorkItemStreamOutcome.Shutdown, firstMessageObserved); } - if (timeoutSource.IsCancellationRequested) - { - return new WorkItemStreamResult(WorkItemStreamOutcome.SilentDisconnect, firstMessageObserved); - } - return new WorkItemStreamResult(WorkItemStreamOutcome.GracefulDrain, firstMessageObserved); } diff --git a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs index d737069ed..b397e8d96 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs @@ -254,7 +254,8 @@ public async Task WorkItem_ResolvesHistoryUsingConfiguredScopeAsync( } else { - (await moveNext.Should().ThrowAsync()).Which.InnerException.Should().BeOfType(); + (await moveNext.Should().ThrowAsync()) + .Which.InnerException.Should().BeOfType(); workItem.OrchestratorRequest.NewEvents[0].ExecutionStarted.Input.Should().Be(token); handler.Requests.Should().BeEmpty(); } diff --git a/test/Grpc.IntegrationTests/AzureBlobPayloadsSideCarInterceptorTests.cs b/test/Grpc.IntegrationTests/AzureBlobPayloadsSideCarInterceptorTests.cs index b9ceebc98..abd5fa6e4 100644 --- a/test/Grpc.IntegrationTests/AzureBlobPayloadsSideCarInterceptorTests.cs +++ b/test/Grpc.IntegrationTests/AzureBlobPayloadsSideCarInterceptorTests.cs @@ -8,6 +8,7 @@ using Google.Protobuf; using Grpc.Core; using Grpc.Core.Interceptors; +using Microsoft.Extensions.Time.Testing; using Moq; using P = Microsoft.DurableTask.Protobuf; @@ -44,32 +45,18 @@ public sealed class AzureBlobPayloadsSideCarInterceptorTests [InlineData("unary")] public async Task ResolvePayloads_WorkerDownloadsHaveIndependentBudgetsAsync(string responseKind) { - // Arrange: advance a local clock 119 seconds per successful download. The three - // fields of one HistoryState operation must not share a 120-second aggregate budget. - TimeSpan elapsed = TimeSpan.Zero; - TimeSpan expires = TimeSpan.MaxValue; - CancellationTokenSource? active = null; - List deadlines = []; + // Arrange + FakeTimeProvider clock = new(); + DateTimeOffset started = clock.GetUtcNow(); + List downloads = []; TrackingPayloadStore store = new(downloadAsync: (token, cancellation) => { - elapsed += TimeSpan.FromSeconds(119); - if (elapsed >= expires) - { - active!.Cancel(); - } - + downloads.Add(cancellation); + clock.Advance(TimeSpan.FromSeconds(119)); cancellation.ThrowIfCancellationRequested(); return Task.FromResult(token["test-blob://".Length..]); }); - AzureBlobPayloadsSideCarInterceptor interceptor = new(store, CreateOptions()); - typeof(PayloadInterceptor).GetProperty("DownloadTimeoutStartedForTest", BindingFlags.Instance | BindingFlags.NonPublic)! - .SetValue(interceptor, (Action)((source, budget) => - { - Assert.Equal(TimeSpan.FromSeconds(120), budget); - active = source; - expires = elapsed + budget; - deadlines.Add(source); - })); + AzureBlobPayloadsSideCarInterceptor interceptor = new(store, CreateOptions()) { TimeProvider = clock }; P.OrchestrationState state = new() { Input = "test-blob://input", @@ -111,8 +98,8 @@ public async Task ResolvePayloads_WorkerDownloadsHaveIndependentBudgetsAsync(str } // Assert - Assert.Equal(TimeSpan.FromSeconds(357), elapsed); - Assert.Equal(responseKind is "work-item" or "worker-history" ? 3 : 0, deadlines.Distinct().Count()); + Assert.Equal(TimeSpan.FromSeconds(357), clock.GetUtcNow() - started); + Assert.Equal(responseKind is "work-item" or "worker-history" ? 3 : 1, downloads.Distinct().Count()); Assert.Equal("input", state.Input); Assert.Equal("output", state.Output); Assert.Equal("status", state.CustomStatus); diff --git a/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj b/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj index 3007a1014..ce7e2bc17 100644 --- a/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj +++ b/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj @@ -4,6 +4,10 @@ net10.0 + + + + diff --git a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs index b904f9217..7cfc4b1ff 100644 --- a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs +++ b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs @@ -19,6 +19,8 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Worker.Grpc.Tests; @@ -159,6 +161,8 @@ public async Task ProcessorExecuteAsync_PendingPayloadRecovery_PreservesShutdown using CancellationTokenSource shutdown = new(); TaskCompletionSource pending = new(TaskCreationOptions.RunContinuationsAsynchronously); TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource nextRead = new(TaskCreationOptions.RunContinuationsAsynchronously); + FakeTimeProvider clock = new(); P.WorkItem failed = new() { CompletionToken = "failed", @@ -176,6 +180,7 @@ public async Task ProcessorExecuteAsync_PendingPayloadRecovery_PreservesShutdown return true; } + nextRead.TrySetResult(); await Task.Delay(Timeout.Infinite, cancellation); return false; }); @@ -209,7 +214,10 @@ public async Task ProcessorExecuteAsync_PendingPayloadRecovery_PreservesShutdown GrpcDurableTaskWorkerOptions grpcOptions = new(); grpcOptions.Internal.ChannelRecreateFailureThreshold = 1; grpcOptions.Internal.SilentDisconnectTimeout = resume ? TimeSpan.FromMilliseconds(250) : TimeSpan.FromSeconds(30); - object processor = CreateProcessor(CreateWorker(grpcOptions), new P.TaskHubSidecarService.TaskHubSidecarServiceClient( + using ServiceProvider services = new ServiceCollection().AddSingleton(clock).BuildServiceProvider(); + object processor = CreateProcessor( + CreateWorker(grpcOptions, new(), NullLoggerFactory.Instance, Mock.Of(), services), + new P.TaskHubSidecarService.TaskHubSidecarServiceClient( invoker.Intercept(new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions())))); // Act @@ -219,11 +227,13 @@ public async Task ProcessorExecuteAsync_PendingPayloadRecovery_PreservesShutdown CancellationToken recoveryCancellation = await pending.Task.WaitAsync(TimeSpan.FromSeconds(5)); if (resume) { - await Task.Delay(grpcOptions.Internal.SilentDisconnectTimeout * 2); + clock.Advance(grpcOptions.Internal.SilentDisconnectTimeout * 2); Assert.False(readCancellation.IsCancellationRequested); Assert.False(recoveryCancellation.IsCancellationRequested); Assert.False(processing.IsCompleted); release.TrySetResult(); + await nextRead.Task.WaitAsync(TimeSpan.FromSeconds(5)); + clock.Advance(grpcOptions.Internal.SilentDisconnectTimeout); } else { @@ -254,11 +264,14 @@ async Task WaitForReleaseAsync(CancellationToken cancellation) } } - [Fact] - public async Task ConsumeAsync_LateMessage_DoesNotTimeOutHealthyHydrationAsync() + [Theory] + [InlineData(119)] + [InlineData(120)] + public async Task ConsumeAsync_LateMessage_DoesNotTimeOutHealthyHydrationAsync(int receiptSeconds) { // Arrange - TimeSpan readTimeout = TimeSpan.FromMilliseconds(500); + FakeTimeProvider clock = new(); + TimeSpan readTimeout = TimeSpan.FromSeconds(120); TaskCompletionSource downloading = new(TaskCreationOptions.RunContinuationsAsynchronously); TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); P.WorkItem item = new() { ActivityRequest = new P.ActivityRequest { Input = "payload-token" } }; @@ -266,15 +279,17 @@ public async Task ConsumeAsync_LateMessage_DoesNotTimeOutHealthyHydrationAsync() Mock> reader = new(); reader.SetupGet(r => r.Current).Returns(item); reader.Setup(r => r.MoveNext(It.IsAny())) - .Returns(async (CancellationToken cancellation) => + .Returns((CancellationToken cancellation) => { if (++reads != 1) { - return false; + return Task.FromResult(false); } - await Task.Delay(readTimeout / 2, cancellation); - return true; + // A successful read can win even when its deadline fires before its continuation runs. + clock.Advance(TimeSpan.FromSeconds(receiptSeconds)); + Assert.Equal(receiptSeconds == 120, cancellation.IsCancellationRequested); + return Task.FromResult(true); }); Mock store = new(); store.Setup(s => s.IsKnownPayloadToken("payload-token")).Returns(true); @@ -297,11 +312,11 @@ public async Task ConsumeAsync_LateMessage_DoesNotTimeOutHealthyHydrationAsync() // Act Task consume = WorkItemStreamConsumer.ConsumeAsync( - call.ResponseStream, readTimeout, received.Add, () => observations++, shutdown.Token); + call.ResponseStream, readTimeout, received.Add, () => observations++, shutdown.Token, timeProvider: clock); try { await downloading.Task.WaitAsync(TimeSpan.FromSeconds(5)); - await Task.Delay(readTimeout); + clock.Advance(TimeSpan.FromSeconds(2)); Assert.False(consume.IsCompleted); Assert.Equal(1, observations); release.TrySetResult(); @@ -324,6 +339,7 @@ public async Task ConsumeAsync_LateMessage_DoesNotTimeOutHealthyHydrationAsync() public async Task ProcessorExecuteAsync_DownloadTimeout_DrainsThenAbandonsOnSameStreamAsync() { // Arrange + FakeTimeProvider clock = new(); using CancellationTokenSource shutdown = new(); TaskCompletionSource allDownloading = new(TaskCreationOptions.RunContinuationsAsynchronously); TaskCompletionSource downloadCancelled = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -372,8 +388,16 @@ public async Task ProcessorExecuteAsync_DownloadTimeout_DrainsThenAbandonsOnSame { if (token == "blocked-token") { - await drain.Task.WaitAsync(cancellation); - return "\"drained\""; + try + { + await Task.Delay(Timeout.Infinite, cancellation); + } + finally + { + await drain.Task; + } + + return string.Empty; } allDownloading.TrySetResult(); @@ -388,14 +412,6 @@ public async Task ProcessorExecuteAsync_DownloadTimeout_DrainsThenAbandonsOnSame return string.Empty; }); - AzureBlobPayloadsSideCarInterceptor interceptor = new(store.Object, new LargePayloadStorageOptions()); - List deadlines = []; - typeof(PayloadInterceptor).GetProperty("DownloadTimeoutStartedForTest", BindingFlags.Instance | BindingFlags.NonPublic)! - .SetValue(interceptor, (Action)((source, budget) => - { - Assert.Equal(TimeSpan.FromSeconds(120), budget); - deadlines.Add(source); - })); int abandons = 0; ScriptedWorkerCallInvoker invoker = new( (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), @@ -415,11 +431,19 @@ public async Task ProcessorExecuteAsync_DownloadTimeout_DrainsThenAbandonsOnSame return Task.FromResult(new P.AbandonOrchestrationTaskResponse()); }, }; - using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); + ServiceCollection registrations = new(); + registrations.AddSingleton(clock); + registrations.AddSingleton(store.Object); + DefaultDurableTaskWorkerBuilder builder = new(null, registrations); + builder.UseGrpc(options => options.CallInvoker = invoker); + builder.UseExternalizedPayloads(); + using ServiceProvider services = registrations.BuildServiceProvider(); + GrpcDurableTaskWorkerOptions grpcOptions = services.GetRequiredService>().Get(Options.DefaultName); + AzureBlobPayloadsSideCarInterceptor interceptor = Assert.Single(grpcOptions.Interceptors.OfType()); Mock factory = new(); ITaskActivity? activity = new TestActivity(); factory.Setup(f => f.TryCreateActivity(It.IsAny(), It.IsAny(), out activity)).Returns(true); - GrpcDurableTaskWorker worker = CreateWorker(new(), new(), NullLoggerFactory.Instance, factory.Object, services); + GrpcDurableTaskWorker worker = CreateWorker(grpcOptions, new(), NullLoggerFactory.Instance, factory.Object, services); object processor = CreateProcessor(worker, new P.TaskHubSidecarService.TaskHubSidecarServiceClient(invoker.Intercept(interceptor))); // Act @@ -427,8 +451,9 @@ public async Task ProcessorExecuteAsync_DownloadTimeout_DrainsThenAbandonsOnSame try { await allDownloading.Task.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.Equal(2, deadlines.Count); - deadlines[1].Cancel(); + clock.Advance(TimeSpan.FromSeconds(119)); + Assert.False(downloadCancelled.Task.IsCompleted); + clock.Advance(TimeSpan.FromSeconds(1)); await downloadCancelled.Task.WaitAsync(TimeSpan.FromSeconds(5)); Assert.False(processing.IsCompleted); Assert.Equal(0, abandons); @@ -441,7 +466,7 @@ public async Task ProcessorExecuteAsync_DownloadTimeout_DrainsThenAbandonsOnSame Assert.Null(completed.FailureDetails); Assert.Equal(1, abandons); Assert.Equal(1, invoker.GetWorkItemsCallCount); - Assert.Equal("\"drained\"", failed.OrchestratorRequest.NewEvents[0].ExecutionStarted.Input); + Assert.Equal("blocked-token", failed.OrchestratorRequest.NewEvents[0].ExecutionStarted.Input); Assert.Equal("stalled-token", failed.OrchestratorRequest.NewEvents[1].EventRaised.Input); } finally @@ -459,6 +484,7 @@ public async Task ProcessorExecuteAsync_DownloadTimeout_DrainsThenAbandonsOnSame public async Task ProcessorExecuteAsync_AbandonTimeout_BoundsRpcRetriesAndBackoffAsync(int mode) { // Arrange + FakeTimeProvider clock = new(); using CancellationTokenSource shutdown = new(); TaskCompletionSource abandoning = new(TaskCreationOptions.RunContinuationsAsynchronously); TaskCompletionSource reconnected = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -508,30 +534,25 @@ public async Task ProcessorExecuteAsync_AbandonTimeout_BoundsRpcRetriesAndBackof grpcOptions.Internal.ReconnectBackoffBase = grpcOptions.Internal.ReconnectBackoffCap = TimeSpan.Zero; grpcOptions.Internal.TransientRetryBackoffBase = grpcOptions.Internal.TransientRetryBackoffCap = mode == 2 ? TimeSpan.FromMinutes(1) : TimeSpan.Zero; - object processor = CreateProcessor(CreateWorker(grpcOptions), new P.TaskHubSidecarService.TaskHubSidecarServiceClient( + using ServiceProvider services = new ServiceCollection().AddSingleton(clock).BuildServiceProvider(); + object processor = CreateProcessor( + CreateWorker(grpcOptions, new(), NullLoggerFactory.Instance, Mock.Of(), services), + new P.TaskHubSidecarService.TaskHubSidecarServiceClient( invoker.Intercept(new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions())))); - CancellationTokenSource? deadline = null; - int budgets = 0; - processor.GetType().GetProperty("AbandonTimeoutStartedForTest", BindingFlags.Instance | BindingFlags.NonPublic)! - .SetValue(processor, (Action)((source, budget) => - { - Assert.Equal(TimeSpan.FromSeconds(30), budget); - deadline = source; - budgets++; - })); // Act - DateTime started = DateTime.UtcNow; + DateTime started = clock.GetUtcNow().UtcDateTime; Task processing = InvokeProcessorExecuteAsync(processor, shutdown.Token); try { await abandoning.Task.WaitAsync(TimeSpan.FromSeconds(5)); - Assert.Equal(1, budgets); Assert.Equal(mode == 1 ? 2 : 1, attempts.Count); DateTime rpcDeadline = Assert.Single(attempts.Select(attempt => attempt.Deadline).Distinct())!.Value; - Assert.InRange(rpcDeadline, started.AddSeconds(29), DateTime.UtcNow.AddSeconds(30)); + Assert.Equal(started.AddSeconds(30), rpcDeadline); Assert.Single(attempts.Select(attempt => attempt.CancellationToken).Distinct()); - Assert.IsType(deadline).Cancel(); + clock.Advance(TimeSpan.FromSeconds(29)); + Assert.All(attempts, attempt => Assert.False(attempt.CancellationToken.IsCancellationRequested)); + clock.Advance(TimeSpan.FromSeconds(1)); await reconnected.Task.WaitAsync(TimeSpan.FromSeconds(5)); // Assert diff --git a/test/Worker/Grpc.Tests/PayloadInterceptorTests.cs b/test/Worker/Grpc.Tests/PayloadInterceptorTests.cs index a092f145e..5426104ee 100644 --- a/test/Worker/Grpc.Tests/PayloadInterceptorTests.cs +++ b/test/Worker/Grpc.Tests/PayloadInterceptorTests.cs @@ -4,6 +4,7 @@ using System.Text; using Grpc.Core; using Grpc.Core.Interceptors; +using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Worker.Grpc.Tests; @@ -19,6 +20,108 @@ public class PayloadInterceptorTests StringMarshaller, StringMarshaller); + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AsyncServerStreamingCall_ExternalSubclass_UsesExistingResolveOverrideAsync(bool history) + { + // Arrange: this test assembly is not a friend of AzureBlobPayloads. + int resolved = 0; + TestPayloadInterceptor interceptor = CreateInterceptor(onResolve: async (_, _) => + { + await Task.Yield(); + resolved++; + }); + + // Act + if (history) + { + using AsyncServerStreamingCall call = CreateStreamingCall( + interceptor, + new P.StreamInstanceHistoryRequest { ForWorkItemProcessing = true }, + new P.HistoryChunk()); + Assert.True(await call.ResponseStream.MoveNext(default)); + } + else + { + using AsyncServerStreamingCall call = CreateStreamingCall( + interceptor, new P.GetWorkItemsRequest(), new P.WorkItem()); + Assert.True(await call.ResponseStream.MoveNext(default)); + } + + // Assert + Assert.Equal(1, resolved); + } + + [Fact] + public async Task AsyncServerStreamingCall_CommonCallToken_ReusesResolutionLifetimeAsync() + { + // Arrange + using CancellationTokenSource cancellation = new(); + List tokens = []; + TestPayloadInterceptor interceptor = CreateInterceptor(onResolve: (_, token) => + { + tokens.Add(token); + return Task.CompletedTask; + }); + using AsyncServerStreamingCall call = CreateStreamingCall( + interceptor, new P.GetWorkItemsRequest(), new P.WorkItem(), cancellation.Token, count: 2); + + // Act + Assert.True(await call.ResponseStream.MoveNext(cancellation.Token)); + Assert.True(await call.ResponseStream.MoveNext(cancellation.Token)); + cancellation.Cancel(); + + // Assert + Assert.Equal(2, tokens.Count); + Assert.Single(tokens.Distinct()); + Assert.All(tokens, token => Assert.True(token.IsCancellationRequested)); + } + + [Fact] + public async Task AsyncServerStreamingCall_DistinctReadToken_CancelsCustomResolutionAsync() + { + // Arrange + using CancellationTokenSource callCancellation = new(); + using CancellationTokenSource readCancellation = new(); + TaskCompletionSource resolving = new(TaskCreationOptions.RunContinuationsAsynchronously); + TestPayloadInterceptor interceptor = CreateInterceptor(onResolve: async (_, cancellation) => + { + resolving.TrySetResult(); + await Task.Delay(Timeout.Infinite, cancellation); + }); + using AsyncServerStreamingCall call = CreateStreamingCall( + interceptor, new P.GetWorkItemsRequest(), new P.WorkItem(), callCancellation.Token); + + // Act + Task next = call.ResponseStream.MoveNext(readCancellation.Token); + await resolving.Task.WaitAsync(TimeSpan.FromSeconds(5)); + readCancellation.Cancel(); + + // Assert + await Assert.ThrowsAnyAsync(() => next.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.False(callCancellation.IsCancellationRequested); + } + + static AsyncServerStreamingCall CreateStreamingCall( + TestPayloadInterceptor interceptor, TRequest request, TResponse response, CancellationToken cancellation = default, int count = 1) + where TRequest : class, new() + where TResponse : class, new() + { + Mock> reader = new(); + reader.Setup(r => r.MoveNext(It.IsAny())).ReturnsAsync(() => count-- > 0); + reader.SetupGet(r => r.Current).Returns(response); + Method method = new( + MethodType.ServerStreaming, "test", "Read", + Marshallers.Create(_ => [], _ => new()), + Marshallers.Create(_ => [], _ => new())); + return interceptor.AsyncServerStreamingCall( + request, + new ClientInterceptorContext(method, null, new CallOptions(cancellationToken: cancellation)), + (_, _) => new AsyncServerStreamingCall( + reader.Object, Task.FromResult(new Metadata()), () => Status.DefaultSuccess, () => new Metadata(), () => { })); + } + [Fact] public void AsyncUnaryCall_SynchronousExternalization_RunsInlineWithoutThreadHop() { diff --git a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs index f3008385d..2b9844475 100644 --- a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs +++ b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs @@ -3,6 +3,7 @@ using Grpc.Core; using Microsoft.DurableTask.Worker.Grpc; +using Microsoft.Extensions.Time.Testing; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Worker.Grpc.Tests; @@ -137,55 +138,37 @@ public async Task OuterCancellation_WithRpcCancelledFromStream_PropagatesExcepti [Fact] public async Task PerItem_HeartbeatReset_KeepsTimerAlive() { - // Proves the per-item timer reset -- not just a single arm at loop start -- is what keeps the - // stream alive. Earlier versions of this test tried to prove the reset by racing real per-item - // delays (each comfortably under the timeout) against the real silent-disconnect timeout (so - // their sum comfortably exceeded it). That was still flaky under CI scheduling pressure: any - // continuation between the "item processed" signal and the next write could be delayed by the - // thread pool/scheduler, silently inflating an intended-short gap past the timeout even though - // production was correct. - // - // This version removes wall-clock timing from the assertion entirely. ConsumeAsync exposes a - // test-only observability hook that fires every time the silent-disconnect timer is (re-)armed: - // once before the read loop starts, and once per item, immediately before that item is - // dispatched to onItem. By recording the exact interleaving of "armed" and "item" events, the - // test proves the structural guarantee directly -- an arm precedes every item, and the total arm - // count is itemCount + 1 -- instead of inferring it from elapsed real time. A regression that - // only arms the timer once at loop start (and never re-arms it per item) fails this assertion - // deterministically, with no dependency on scheduler timing. - const int itemCount = 5; - List events = new(); - int itemIndex = 0; - - P.WorkItem[] items = new P.WorkItem[itemCount]; - for (int i = 0; i < itemCount; i++) + // Arrange + FakeTimeProvider clock = new(); + int remaining = 5; + int received = 0; + Mock> reader = new(); + reader.SetupGet(r => r.Current).Returns(new P.WorkItem { HealthPing = new P.HealthPing() }); + reader.Setup(r => r.MoveNext(It.IsAny())).Returns((CancellationToken cancellation) => { - items[i] = new P.WorkItem { HealthPing = new P.HealthPing() }; - } + if (remaining-- == 0) + { + return Task.FromResult(false); + } + + clock.Advance(TimeSpan.FromSeconds(119)); + cancellation.ThrowIfCancellationRequested(); + return Task.FromResult(true); + }); + // Act WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - stream: StreamOf(items), - silentDisconnectTimeout: TimeSpan.FromMilliseconds(500), - onItem: _ => events.Add($"item{itemIndex++}"), + stream: reader.Object, + silentDisconnectTimeout: TimeSpan.FromSeconds(120), + onItem: _ => received++, onFirstMessage: null, cancellation: CancellationToken.None, - onSilentDisconnectTimerArmed: () => events.Add("armed")); + timeProvider: clock); + // Assert result.Outcome.Should().Be(WorkItemStreamOutcome.GracefulDrain); result.FirstMessageObserved.Should().BeTrue(); - - // 1 initial arm (before the loop starts) + 1 re-arm per item. - events.Count(e => e == "armed").Should().Be(itemCount + 1); - - // Every item must be immediately preceded by its own re-arm, and the very first event overall - // is the initial pre-loop arm. - events[0].Should().Be("armed"); - for (int i = 0; i < itemCount; i++) - { - int armedIndex = 1 + (i * 2); - events[armedIndex].Should().Be("armed", "item {0} must be preceded by a timer re-arm", i); - events[armedIndex + 1].Should().Be($"item{i}"); - } + received.Should().Be(5); } [Fact] diff --git a/test/Worker/Grpc.Tests/Worker.Grpc.Tests.csproj b/test/Worker/Grpc.Tests/Worker.Grpc.Tests.csproj index f17df96bc..d9a34cbac 100644 --- a/test/Worker/Grpc.Tests/Worker.Grpc.Tests.csproj +++ b/test/Worker/Grpc.Tests/Worker.Grpc.Tests.csproj @@ -6,6 +6,7 @@ + From 5c1c38eb02b77903706fce9c9b269205a9dfce72 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 11 Nov 2026 09:35:56 -0800 Subject: [PATCH 8/9] fix(test): use the feed-cached native fake clock package Use Microsoft.Extensions.TimeProvider.Testing 10.0.0 for the net10 test projects. Its native net10 dependency group does not request the uncached Microsoft.Bcl.TimeProvider 8.0.0 package. Verified both test projects restore from an empty package cache using only the checked-in nuget.config. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index feacb1feb..d8a0cbad5 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -85,7 +85,7 @@ - + From fa5acb8ffcea7a45aa7a93e8598f04986b942668 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 16 Nov 2026 15:47:47 -0800 Subject: [PATCH 9/9] fix: count failed payload deliveries as stream activity Record real failed-delivery receipt without yielding partially hydrated work. Share first-message bookkeeping with healthy deliveries so active streams do not receive empty-drain channel-recreation credit. Give the isolated healthy-work regression two explicit worker/sidecar activity slots so it also runs on one CPU. Preserve abandon-only handling and the existing timeout policy. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 4 +- src/Worker/Grpc/WorkItemStreamConsumer.cs | 28 +++++---- .../GrpcSidecarFixture.cs | 5 +- .../LargePayloadTests.cs | 11 +++- .../Grpc.Tests/GrpcDurableTaskWorkerTests.cs | 61 +++++++++++++++++++ .../Grpc.Tests/WorkItemStreamConsumerTests.cs | 54 +++++++++++++--- 6 files changed, 138 insertions(+), 25 deletions(-) diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index bc5f0c220..6bf287816 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -391,7 +391,7 @@ async Task ProcessWorkItemsAsync( // Catch delivery failures inside the iterator; a faulted ReadAllAsync cannot be resumed. // A false transport read still ends this stream permanently. WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - ct => this.ReadWorkItemsAsync(stream.ResponseStream, ct), + (ct, onMessageReceived) => this.ReadWorkItemsAsync(stream.ResponseStream, onMessageReceived, ct), silentDisconnectTimeout, workItem => this.DispatchWorkItem(workItem, cancellation), onFirstMessage, @@ -431,6 +431,7 @@ async Task ProcessWorkItemsAsync( async IAsyncEnumerable ReadWorkItemsAsync( IAsyncStreamReader reader, + Action onMessageReceived, [EnumeratorCancellation] CancellationToken cancellation) { while (true) @@ -442,6 +443,7 @@ async Task ProcessWorkItemsAsync( } catch (WorkItemPayloadException ex) { + onMessageReceived(); await this.AbandonUnresolvedWorkItemAsync(ex, cancellation).ConfigureAwait(false); continue; } diff --git a/src/Worker/Grpc/WorkItemStreamConsumer.cs b/src/Worker/Grpc/WorkItemStreamConsumer.cs index 9c39ef7bc..62004bae8 100644 --- a/src/Worker/Grpc/WorkItemStreamConsumer.cs +++ b/src/Worker/Grpc/WorkItemStreamConsumer.cs @@ -44,8 +44,8 @@ internal static class WorkItemStreamConsumer /// Consume a work-item stream until shutdown, silent disconnect, or graceful drain. /// /// - /// Factory that opens the stream with the supplied linked-cancellation token. Production passes - /// ct => stream.ResponseStream.ReadAllAsync(ct); tests pass arbitrary fakes. + /// Factory that opens the stream with linked cancellation and a receipt callback. The callback + /// records real deliveries handled without yielding them; yielded items are observed automatically. /// /// /// How long to wait between successive items (or health pings) before treating the stream as @@ -70,7 +70,7 @@ internal static class WorkItemStreamConsumer /// /// The classified outcome plus whether any message was observed. public static async Task ConsumeAsync( - Func> openStream, + Func> openStream, TimeSpan silentDisconnectTimeout, Action onItem, Action? onFirstMessage, @@ -100,18 +100,22 @@ void ArmSilentDisconnectTimer() bool firstMessageObserved = false; - try + void ObserveMessage() { - await foreach (P.WorkItem workItem in openStream(tokenSource.Token).ConfigureAwait(false)) - { - ArmSilentDisconnectTimer(); + ArmSilentDisconnectTimer(); - if (!firstMessageObserved) - { - firstMessageObserved = true; - onFirstMessage?.Invoke(); - } + if (!firstMessageObserved) + { + firstMessageObserved = true; + onFirstMessage?.Invoke(); + } + } + try + { + await foreach (P.WorkItem workItem in openStream(tokenSource.Token, ObserveMessage).ConfigureAwait(false)) + { + ObserveMessage(); onItem(workItem); } } diff --git a/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs b/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs index cfefcd4d3..3dfd6affa 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs @@ -22,9 +22,12 @@ public sealed class GrpcSidecarFixture : IDisposable readonly IHost host; public GrpcSidecarFixture() + : this(new InMemoryOrchestrationService()) { - InMemoryOrchestrationService service = new(); + } + internal GrpcSidecarFixture(InMemoryOrchestrationService service) + { // Use a random port number to allow multiple instances to run in parallel string address = $"http://{ListenHost}:{Random.Shared.Next(30000, 40000)}"; this.host = Host.CreateDefaultBuilder() diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index 4b0cc2899..a06189e5f 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -10,11 +10,13 @@ using Microsoft.DurableTask.Client; using Microsoft.DurableTask; using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.Testing.Sidecar; using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; using Moq; using Xunit.Abstractions; +using IOrchestrationService = global::DurableTask.Core.IOrchestrationService; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Grpc.Tests; @@ -28,7 +30,7 @@ public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture side public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int storageStatus) { // Arrange - using GrpcSidecarFixture isolatedSidecar = new(); + using GrpcSidecarFixture isolatedSidecar = new(new TwoActivitySlotService()); string token = $"test-blob://missing-{Guid.NewGuid():N}"; TaskName orchestration = nameof(HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync); TaskName activity = "HydrationActivity"; @@ -48,6 +50,8 @@ public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int s await using HostTestLifetime server = await this.StartWorkerAsync(worker => { worker.UseGrpc(isolatedSidecar.Channel); + worker.Services.Configure(worker.Name, + options => options.Concurrency.MaximumConcurrentActivityWorkItems = 2); worker.AddTasks(tasks => tasks .AddOrchestratorFunc(orchestration, (context, input) => context.CallActivityAsync(activity, input == "bad" ? new string('X', 1024) : input)) @@ -990,6 +994,11 @@ public async Task LargeActivityOutput_ExceedsGrpcLimit_ExternalizesSuccessfully( Assert.Equal(largeOutput, output); } + sealed class TwoActivitySlotService : InMemoryOrchestrationService, IOrchestrationService + { + int IOrchestrationService.MaxConcurrentTaskActivityWorkItems => 2; + } + sealed class ForwardingStreamInterceptor : Interceptor { public override AsyncServerStreamingCall AsyncServerStreamingCall( diff --git a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs index 6c710c633..2743cba23 100644 --- a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs +++ b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs @@ -36,6 +36,9 @@ public class GrpcDurableTaskWorkerTests static readonly MethodInfo ProcessorConnectAsyncMethod = typeof(GrpcDurableTaskWorker) .GetNestedType("Processor", BindingFlags.NonPublic)! .GetMethod("ConnectAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; + static readonly MethodInfo ProcessWorkItemsAsyncMethod = typeof(GrpcDurableTaskWorker) + .GetNestedType("Processor", BindingFlags.NonPublic)! + .GetMethod("ProcessWorkItemsAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; static readonly MethodInfo DispatchWorkItemMethod = typeof(GrpcDurableTaskWorker) .GetNestedType("Processor", BindingFlags.NonPublic)! .GetMethod("DispatchWorkItem", BindingFlags.Instance | BindingFlags.NonPublic)!; @@ -148,6 +151,64 @@ public async Task ProcessorExecuteAsync_HydrationFailure_AbandonsOnlyFailedDeliv store.Verify(s => s.DownloadAsync(Token, It.IsAny()), Times.Once); } + [Theory] + [InlineData(2, false)] + [InlineData(2, true)] + [InlineData(0, false)] + public async Task ProcessWorkItemsAsync_FailedDeliveries_CountAsReceiptWithoutDispatchAsync(int failedCount, bool healthPing) + { + // Arrange + P.WorkItem[] items = Enumerable.Range(0, failedCount).Select(index => new P.WorkItem + { + CompletionToken = $"failed-{index}", + ActivityRequest = new P.ActivityRequest { Input = "missing-payload" }, + }).ToArray(); + if (healthPing) + { + items = [.. items, new P.WorkItem { HealthPing = new P.HealthPing() }]; + } + + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken("missing-payload")).Returns(true); + store.Setup(s => s.DownloadAsync("missing-payload", It.IsAny())) + .ThrowsAsync(new IOException("Download failed.")); + List abandoned = []; + int firstMessages = 0; + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (_, _) => CreateServerStreamingCall(new SequenceAsyncStreamReader(items))) + { + UnaryResponse = (operation, request, _) => + { + Assert.Equal(1, firstMessages); + Assert.Equal("AbandonTaskActivityWorkItem", operation); + abandoned.Add(Assert.IsType(request).CompletionToken); + return Task.FromResult(new P.AbandonActivityTaskResponse()); + }, + }; + int dispatched = 0; + int poisoned = 0; + GrpcDurableTaskWorkerOptions grpcOptions = new(); + grpcOptions.Internal.SilentDisconnectTimeout = TimeSpan.Zero; + grpcOptions.ConfigureActivityNotification(_ => dispatched++); + P.TaskHubSidecarService.TaskHubSidecarServiceClient client = new(invoker.Intercept( + new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions()))); + object processor = CreateProcessor(CreateWorker(grpcOptions), client); + using AsyncServerStreamingCall stream = client.GetWorkItems(new()); + + // Act + await (Task)ProcessWorkItemsAsyncMethod.Invoke(processor, + [stream, CancellationToken.None, (Action)(() => firstMessages++), (Action)(() => poisoned++)])!; + + // Assert + bool observed = failedCount > 0 || healthPing; + Assert.Equal(observed ? 1 : 0, firstMessages); + Assert.Equal(observed ? 0 : 1, poisoned); + Assert.Equal(0, dispatched); + Assert.Equal(Enumerable.Range(0, failedCount).Select(index => $"failed-{index}"), abandoned); + Assert.Equal(1, invoker.GetWorkItemsCallCount); + } + [Fact] public async Task ExecuteAsync_ConnectFailureThreshold_RecreatesConfiguredChannel() { diff --git a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs index ba9104c11..96de52af6 100644 --- a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs +++ b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs @@ -16,7 +16,7 @@ public class WorkItemStreamConsumerTests public async Task EmptyStream_ReturnsGracefulDrain() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => EmptyStream(), + openStream: (_, _) => EmptyStream(), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: _ => throw new InvalidOperationException("onItem should not be invoked"), onFirstMessage: () => throw new InvalidOperationException("onFirstMessage should not be invoked"), @@ -35,7 +35,7 @@ public async Task StreamWithItems_ReturnsGracefulDrain_AndFiresCallbacks() int firstMessageCount = 0; WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => StreamOf(item1, item2), + openStream: (_, _) => StreamOf(item1, item2), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: received.Add, onFirstMessage: () => firstMessageCount++, @@ -47,11 +47,35 @@ public async Task StreamWithItems_ReturnsGracefulDrain_AndFiresCallbacks() firstMessageCount.Should().Be(1); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NonYieldedReceipts_MarkStreamObservedWithoutDispatchAsync(bool yieldHealthyItem) + { + // Arrange + List received = []; + int firstMessages = 0; + + // Act + WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( + openStream: (_, observe) => StreamWithReceipts(observe, yieldHealthyItem), + silentDisconnectTimeout: TimeSpan.Zero, + onItem: received.Add, + onFirstMessage: () => firstMessages++, + cancellation: CancellationToken.None); + + // Assert + Assert.Equal(WorkItemStreamOutcome.GracefulDrain, result.Outcome); + Assert.True(result.FirstMessageObserved); + Assert.Equal(1, firstMessages); + Assert.Equal(yieldHealthyItem ? 1 : 0, received.Count); + } + [Fact] public async Task VeryLargeSilentDisconnectTimeout_IsClamped_AndStreamCanStillComplete() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => EmptyStream(), + openStream: (_, _) => EmptyStream(), silentDisconnectTimeout: TimeSpan.FromDays(365), onItem: _ => throw new InvalidOperationException("onItem should not be invoked"), onFirstMessage: null, @@ -65,7 +89,7 @@ public async Task VeryLargeSilentDisconnectTimeout_IsClamped_AndStreamCanStillCo public async Task HangingStream_SurfacingOce_ReturnsSilentDisconnect() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: false), silentDisconnectTimeout: ShortTimeout, onItem: _ => { }, onFirstMessage: null, @@ -85,7 +109,7 @@ public async Task HangingStream_SurfacingOce_ReturnsSilentDisconnect() public async Task HangingStream_SurfacingRpcCancelled_ReturnsSilentDisconnect() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: true), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: true), silentDisconnectTimeout: ShortTimeout, onItem: _ => { }, onFirstMessage: null, @@ -104,7 +128,7 @@ public async Task OuterCancellation_WithOceFromStream_ReturnsShutdown() outer.CancelAfter(ShortTimeout); WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: false), silentDisconnectTimeout: TimeSpan.FromSeconds(30), onItem: _ => { }, onFirstMessage: null, @@ -126,7 +150,7 @@ public async Task OuterCancellation_WithRpcCancelledFromStream_PropagatesExcepti outer.CancelAfter(ShortTimeout); Func act = () => WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: true), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: true), silentDisconnectTimeout: TimeSpan.FromSeconds(30), onItem: _ => { }, onFirstMessage: null, @@ -165,7 +189,7 @@ public async Task PerItem_HeartbeatReset_KeepsTimerAlive() } WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => StreamOf(items), + openStream: (_, _) => StreamOf(items), silentDisconnectTimeout: TimeSpan.FromMilliseconds(500), onItem: _ => events.Add($"item{itemIndex++}"), onFirstMessage: null, @@ -193,7 +217,7 @@ public async Task PerItem_HeartbeatReset_KeepsTimerAlive() public async Task UnrelatedRpcException_Propagates() { Func act = () => WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => ThrowingStream(new RpcException(new Status(StatusCode.Unavailable, "boom"))), + openStream: (_, _) => ThrowingStream(new RpcException(new Status(StatusCode.Unavailable, "boom"))), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: _ => { }, onFirstMessage: null, @@ -213,7 +237,7 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti // Act WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: false), silentDisconnectTimeout: TimeSpan.FromMilliseconds(timeoutMilliseconds), onItem: _ => { }, onFirstMessage: null, @@ -224,6 +248,16 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti } #pragma warning disable CS1998 // Async method lacks 'await' operators + static async IAsyncEnumerable StreamWithReceipts(Action observe, bool yieldHealthyItem) + { + observe(); + observe(); + if (yieldHealthyItem) + { + yield return new P.WorkItem { HealthPing = new P.HealthPing() }; + } + } + static async IAsyncEnumerable EmptyStream() { yield break;