diff --git a/CHANGELOG.md b/CHANGELOG.md index 859e258b..bf2ffe99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Isolate incoming payload hydration failures to the failed delivery, preserving the shared work-item stream and unrelated in-flight work ([#825](https://github.com/microsoft/durabletask-dotnet/issues/825)). - Avoid allocating outcome-observation continuations for already-successful gRPC client responses while preserving transport-failure tracking and channel recreation. - Validate all eligible externalized payload sizes before uploading any field of an orchestration response or entity batch, including nested carryover and rewind history. Deterministic size failures now precede hypothetical upload failures and cause no uploads or partial token substitutions ([#820](https://github.com/microsoft/durabletask-dotnet/issues/820)). diff --git a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs index 8f0ab337..5aa1b375 100644 --- a/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs +++ b/src/Extensions/AzureBlobPayloads/Interceptors/PayloadInterceptor.cs @@ -4,6 +4,8 @@ using System.Text; using Grpc.Core; using Grpc.Core.Interceptors; +using Microsoft.DurableTask.Worker.Grpc.Internal; +using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask; @@ -280,7 +282,20 @@ public async Task MoveNext(CancellationToken cancellationToken) return false; } - this.Current = await this.transform(this.inner.Current, cancellationToken); + try + { + this.Current = await this.transform(this.inner.Current, cancellationToken); + } + catch (Exception ex) when (this.inner.Current is P.WorkItem workItem + && workItem.RequestCase is P.WorkItem.RequestOneofCase.ActivityRequest + or P.WorkItem.RequestOneofCase.OrchestratorRequest + or P.WorkItem.RequestOneofCase.EntityRequest + or P.WorkItem.RequestOneofCase.EntityRequestV2 + && !cancellationToken.IsCancellationRequested) + { + throw new WorkItemPayloadException(workItem, ex); + } + return true; } } diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 33e34968..6bf28781 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using DurableTask.Core; using DurableTask.Core.Entities; @@ -387,12 +388,10 @@ async Task ProcessWorkItemsAsync( // window. If that fires we treat the stream as silently disconnected and reconnect. TimeSpan silentDisconnectTimeout = this.internalOptions.SilentDisconnectTimeout; - // NOTE: the consumer deliberately does NOT wrap its await foreach in an outer loop. - // The underlying IAsyncStreamReader is single-use — once the server terminates the stream - // (e.g. via a graceful HTTP/2 GOAWAY with OK trailers during a rolling upgrade), MoveNext - // returns false forever and re-entering await foreach would tight-spin with no yield. + // Catch delivery failures inside the iterator; a faulted ReadAllAsync cannot be resumed. + // A false transport read still ends this stream permanently. WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - ct => stream.ResponseStream.ReadAllAsync(ct), + (ct, onMessageReceived) => this.ReadWorkItemsAsync(stream.ResponseStream, onMessageReceived, ct), silentDisconnectTimeout, workItem => this.DispatchWorkItem(workItem, cancellation), onFirstMessage, @@ -430,6 +429,72 @@ async Task ProcessWorkItemsAsync( } } + async IAsyncEnumerable ReadWorkItemsAsync( + IAsyncStreamReader reader, + Action onMessageReceived, + [EnumeratorCancellation] CancellationToken cancellation) + { + while (true) + { + bool hasNext; + try + { + hasNext = await reader.MoveNext(cancellation).ConfigureAwait(false); + } + catch (WorkItemPayloadException ex) + { + onMessageReceived(); + await this.AbandonUnresolvedWorkItemAsync(ex, cancellation).ConfigureAwait(false); + continue; + } + + if (!hasNext) + { + yield break; + } + + yield return reader.Current; + } + } + + async Task AbandonUnresolvedWorkItemAsync(WorkItemPayloadException exception, CancellationToken cancellation) + { + cancellation.ThrowIfCancellationRequested(); + P.WorkItem workItem = exception.WorkItem; + string instanceId = workItem.OrchestratorRequest?.InstanceId + ?? workItem.ActivityRequest?.OrchestrationInstance?.InstanceId + ?? workItem.EntityRequest?.InstanceId + ?? workItem.EntityRequestV2?.InstanceId + ?? string.Empty; + this.Logger.UnexpectedError(exception, instanceId); + try + { + await this.ExecuteWithRetryAsync( + () => workItem.RequestCase switch + { + P.WorkItem.RequestOneofCase.OrchestratorRequest => + this.client.AbandonTaskOrchestratorWorkItemAsync( + new P.AbandonOrchestrationTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + P.WorkItem.RequestOneofCase.ActivityRequest => + this.client.AbandonTaskActivityWorkItemAsync( + new P.AbandonActivityTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + P.WorkItem.RequestOneofCase.EntityRequest or P.WorkItem.RequestOneofCase.EntityRequestV2 => + this.client.AbandonTaskEntityWorkItemAsync( + new P.AbandonEntityTaskRequest { CompletionToken = workItem.CompletionToken }, + cancellationToken: cancellation).ResponseAsync, + _ => throw new InvalidOperationException($"Cannot abandon unresolved {workItem.RequestCase}."), + }, + "AbandonUnresolvedWorkItem", + cancellation); + } + catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound && !cancellation.IsCancellationRequested) + { + this.Logger.WorkItemOwnershipLost(workItem.RequestCase.ToString(), workItem.CompletionToken, ex); + } + } + void DispatchWorkItem(P.WorkItem workItem, CancellationToken cancellation) { if (workItem.RequestCase == P.WorkItem.RequestOneofCase.OrchestratorRequest) diff --git a/src/Worker/Grpc/Internal/WorkItemPayloadException.cs b/src/Worker/Grpc/Internal/WorkItemPayloadException.cs new file mode 100644 index 00000000..91f5322d --- /dev/null +++ b/src/Worker/Grpc/Internal/WorkItemPayloadException.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Worker.Grpc.Internal; + +/// +/// A received delivery whose payloads could not be resolved. The item is only safe to abandon, not dispatch. +/// +/// The failed delivery, including its original completion token. +/// The payload resolution failure. +/// +/// Infrastructure contract between worker extensions and the gRPC worker. This is not a supported application API. +/// +public sealed class WorkItemPayloadException(P.WorkItem workItem, Exception innerException) + : Exception("Failed to resolve payloads for a work-item delivery.", innerException) +{ + /// + /// Gets the failed delivery. Its payload fields may be partially resolved. + /// + public P.WorkItem WorkItem { get; } = workItem; +} diff --git a/src/Worker/Grpc/Logs.cs b/src/Worker/Grpc/Logs.cs index f378a7fc..ea1efa34 100644 --- a/src/Worker/Grpc/Logs.cs +++ b/src/Worker/Grpc/Logs.cs @@ -107,5 +107,8 @@ static partial class Logs [LoggerMessage(EventId = 78, Level = LogLevel.Warning, Message = "Activity notification callback failed for phase '{Phase}'.")] public static partial void ActivityNotificationFailed(this ILogger logger, ActivityNotificationPhase phase, Exception exception); + + [LoggerMessage(EventId = 79, Level = LogLevel.Warning, Message = "Ownership of '{workItemType}' work item was lost before abandonment. Completion token = '{completionToken}'")] + public static partial void WorkItemOwnershipLost(this ILogger logger, string workItemType, string completionToken, Exception exception); } } diff --git a/src/Worker/Grpc/WorkItemStreamConsumer.cs b/src/Worker/Grpc/WorkItemStreamConsumer.cs index 9c39ef7b..62004bae 100644 --- a/src/Worker/Grpc/WorkItemStreamConsumer.cs +++ b/src/Worker/Grpc/WorkItemStreamConsumer.cs @@ -44,8 +44,8 @@ internal static class WorkItemStreamConsumer /// Consume a work-item stream until shutdown, silent disconnect, or graceful drain. /// /// - /// Factory that opens the stream with the supplied linked-cancellation token. Production passes - /// ct => stream.ResponseStream.ReadAllAsync(ct); tests pass arbitrary fakes. + /// Factory that opens the stream with linked cancellation and a receipt callback. The callback + /// records real deliveries handled without yielding them; yielded items are observed automatically. /// /// /// How long to wait between successive items (or health pings) before treating the stream as @@ -70,7 +70,7 @@ internal static class WorkItemStreamConsumer /// /// The classified outcome plus whether any message was observed. public static async Task ConsumeAsync( - Func> openStream, + Func> openStream, TimeSpan silentDisconnectTimeout, Action onItem, Action? onFirstMessage, @@ -100,18 +100,22 @@ void ArmSilentDisconnectTimer() bool firstMessageObserved = false; - try + void ObserveMessage() { - await foreach (P.WorkItem workItem in openStream(tokenSource.Token).ConfigureAwait(false)) - { - ArmSilentDisconnectTimer(); + ArmSilentDisconnectTimer(); - if (!firstMessageObserved) - { - firstMessageObserved = true; - onFirstMessage?.Invoke(); - } + if (!firstMessageObserved) + { + firstMessageObserved = true; + onFirstMessage?.Invoke(); + } + } + try + { + await foreach (P.WorkItem workItem in openStream(tokenSource.Token, ObserveMessage).ConfigureAwait(false)) + { + ObserveMessage(); onItem(workItem); } } diff --git a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs index 4bd862eb..b397e8d9 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreRoutingTests.cs @@ -254,7 +254,8 @@ public async Task WorkItem_ResolvesHistoryUsingConfiguredScopeAsync( } else { - await moveNext.Should().ThrowAsync(); + (await moveNext.Should().ThrowAsync()) + .Which.InnerException.Should().BeOfType(); workItem.OrchestratorRequest.NewEvents[0].ExecutionStarted.Input.Should().Be(token); handler.Requests.Should().BeEmpty(); } diff --git a/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs b/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs index cfefcd4d..3dfd6aff 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs @@ -22,9 +22,12 @@ public sealed class GrpcSidecarFixture : IDisposable readonly IHost host; public GrpcSidecarFixture() + : this(new InMemoryOrchestrationService()) { - InMemoryOrchestrationService service = new(); + } + internal GrpcSidecarFixture(InMemoryOrchestrationService service) + { // Use a random port number to allow multiple instances to run in parallel string address = $"http://{ListenHost}:{Random.Shared.Next(30000, 40000)}"; this.host = Host.CreateDefaultBuilder() diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index f29b7106..a06189e5 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -1,23 +1,109 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.IO; using System.Text.Json; +using Azure; using FluentAssertions; using Grpc.Core; using Grpc.Core.Interceptors; using Microsoft.DurableTask.Client; using Microsoft.DurableTask; using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.Testing.Sidecar; using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; +using Moq; using Xunit.Abstractions; +using IOrchestrationService = global::DurableTask.Core.IOrchestrationService; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Grpc.Tests; public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) : IntegrationTestBase(output, sidecarFixture) { + [Theory] + [InlineData(0)] + [InlineData(403)] + [InlineData(404)] + public async Task HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync(int storageStatus) + { + // Arrange + using GrpcSidecarFixture isolatedSidecar = new(new TwoActivitySlotService()); + string token = $"test-blob://missing-{Guid.NewGuid():N}"; + TaskName orchestration = nameof(HydrationFailure_KeepsHealthyActivitiesOnSameStreamAsync); + TaskName activity = "HydrationActivity"; + TaskCompletionSource running = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource downloadFailed = new(TaskCreationOptions.RunContinuationsAsynchronously); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value == token); + store.Setup(s => s.UploadAsync(It.IsAny(), It.IsAny())).ReturnsAsync(token); + store.Setup(s => s.DownloadAsync(token, It.IsAny())).Returns(() => + { + downloadFailed.TrySetResult(); + Exception failure = storageStatus == 0 + ? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed."); + return Task.FromException(failure); + }); + await using HostTestLifetime server = await this.StartWorkerAsync(worker => + { + worker.UseGrpc(isolatedSidecar.Channel); + worker.Services.Configure(worker.Name, + options => options.Concurrency.MaximumConcurrentActivityWorkItems = 2); + worker.AddTasks(tasks => tasks + .AddOrchestratorFunc(orchestration, (context, input) => + context.CallActivityAsync(activity, input == "bad" ? new string('X', 1024) : input)) + .AddActivityFunc(activity, async (_, input) => + { + if (input == "running") + { + running.TrySetResult(); + await release.Task.WaitAsync(this.TimeoutToken); + } + + return input; + })); + worker.UseExternalizedPayloads(); + worker.Services.AddSingleton(store.Object); + worker.Services.PostConfigure( + options => options.Interceptors.Insert(0, new ForwardingStreamInterceptor())); + }, client => client.UseGrpc(isolatedSidecar.Channel), services => services.AddExternalizedPayloadStore(options => + { + options.ThresholdBytes = 16; + options.ConnectionString = "UseDevelopmentStorage=true"; + })); + + try + { + // Act + string alreadyRunning = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "running"); + await running.Task.WaitAsync(this.TimeoutToken); + await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "bad"); + await downloadFailed.Task.WaitAsync(this.TimeoutToken); + + // The test sidecar does not settle abandoned activities; free the healthy slot before starting another. + release.TrySetResult(); + OrchestrationMetadata runningResult = await server.Client.WaitForInstanceCompletionAsync( + alreadyRunning, getInputsAndOutputs: true, this.TimeoutToken); + string following = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestration, "following"); + OrchestrationMetadata followingResult = await server.Client.WaitForInstanceCompletionAsync( + following, getInputsAndOutputs: true, this.TimeoutToken); + + // Assert + Assert.Equal(OrchestrationRuntimeStatus.Completed, followingResult.RuntimeStatus); + Assert.Equal("following", followingResult.ReadOutputAs()); + Assert.Equal(OrchestrationRuntimeStatus.Completed, runningResult.RuntimeStatus); + Assert.Equal("running", runningResult.ReadOutputAs()); + Assert.Single(this.GetLogs(), log => log.EventId.Name == "EstablishedWorkItemConnection"); + } + finally + { + release.TrySetResult(); + } + } + [Theory] [InlineData(true)] [InlineData(false)] @@ -908,6 +994,28 @@ public async Task LargeActivityOutput_ExceedsGrpcLimit_ExternalizesSuccessfully( Assert.Equal(largeOutput, output); } + sealed class TwoActivitySlotService : InMemoryOrchestrationService, IOrchestrationService + { + int IOrchestrationService.MaxConcurrentTaskActivityWorkItems => 2; + } + + sealed class ForwardingStreamInterceptor : Interceptor + { + public override AsyncServerStreamingCall AsyncServerStreamingCall( + TRequest request, + ClientInterceptorContext context, + AsyncServerStreamingCallContinuation continuation) + { + AsyncServerStreamingCall call = continuation(request, context); + Mock> reader = new(); + reader.Setup(r => r.MoveNext(It.IsAny())) + .Returns((CancellationToken cancellation) => call.ResponseStream.MoveNext(cancellation)); + reader.SetupGet(r => r.Current).Returns(() => call.ResponseStream.Current); + return new AsyncServerStreamingCall( + reader.Object, call.ResponseHeadersAsync, call.GetStatus, call.GetTrailers, call.Dispose); + } + } + sealed class RetryCompletionInterceptor(bool retryActivity) : Interceptor { public List Payloads { get; } = []; diff --git a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs index 4b3505c1..2743cba2 100644 --- a/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs +++ b/test/Worker/Grpc.Tests/GrpcDurableTaskWorkerTests.cs @@ -5,10 +5,12 @@ using System.Diagnostics; using System.IO; using System.Reflection; +using Azure; using DurableTask.Core; using DurableTask.Core.History; using Google.Protobuf.WellKnownTypes; using Grpc.Core; +using Grpc.Core.Interceptors; using Microsoft.DurableTask; using Microsoft.DurableTask.Tests.Logging; using Microsoft.DurableTask.Tracing; @@ -34,6 +36,9 @@ public class GrpcDurableTaskWorkerTests static readonly MethodInfo ProcessorConnectAsyncMethod = typeof(GrpcDurableTaskWorker) .GetNestedType("Processor", BindingFlags.NonPublic)! .GetMethod("ConnectAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; + static readonly MethodInfo ProcessWorkItemsAsyncMethod = typeof(GrpcDurableTaskWorker) + .GetNestedType("Processor", BindingFlags.NonPublic)! + .GetMethod("ProcessWorkItemsAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; static readonly MethodInfo DispatchWorkItemMethod = typeof(GrpcDurableTaskWorker) .GetNestedType("Processor", BindingFlags.NonPublic)! .GetMethod("DispatchWorkItem", BindingFlags.Instance | BindingFlags.NonPublic)!; @@ -49,6 +54,161 @@ public class GrpcDurableTaskWorkerTests .GetNestedType("Processor", BindingFlags.NonPublic)! .GetMethod("OnRunOrchestratorAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; + [Theory] + [CombinatorialData] + public async Task ProcessorExecuteAsync_HydrationFailure_AbandonsOnlyFailedDeliveryAsync( + [CombinatorialValues(P.WorkItem.RequestOneofCase.ActivityRequest, P.WorkItem.RequestOneofCase.OrchestratorRequest, + P.WorkItem.RequestOneofCase.EntityRequest, P.WorkItem.RequestOneofCase.EntityRequestV2)] P.WorkItem.RequestOneofCase kind, + [CombinatorialValues(0, 403, 404)] int storageStatus, + [CombinatorialValues(StatusCode.OK, StatusCode.NotFound, StatusCode.Unavailable, StatusCode.PermissionDenied)] StatusCode abandonStatus) + { + // Arrange + const string Token = "test-blob://failed"; + P.WorkItem failed = kind switch + { + P.WorkItem.RequestOneofCase.ActivityRequest => new() { ActivityRequest = new P.ActivityRequest { Input = Token } }, + P.WorkItem.RequestOneofCase.OrchestratorRequest => new() { OrchestratorRequest = new P.OrchestratorRequest + { + NewEvents = { new P.HistoryEvent { ExecutionStarted = new P.ExecutionStartedEvent { Input = Token } } }, + } }, + P.WorkItem.RequestOneofCase.EntityRequest => new() { EntityRequest = new P.EntityBatchRequest { EntityState = Token } }, + _ => new() { EntityRequestV2 = new P.EntityRequest { EntityState = Token } }, + }; + failed.CompletionToken = "failed"; + Queue deliveries = new([failed, new P.WorkItem { HealthPing = new P.HealthPing() }]); + TaskCompletionSource pingDispatched = new(TaskCreationOptions.RunContinuationsAsynchronously); + P.WorkItem current = null!; + Mock> reader = new(); + reader.SetupGet(r => r.Current).Returns(() => current); + reader.Setup(r => r.MoveNext(It.IsAny())).Returns(async (CancellationToken cancellation) => + { + if (deliveries.TryDequeue(out P.WorkItem? item)) + { + current = item; + return true; + } + + pingDispatched.TrySetResult(); + await Task.Delay(Timeout.Infinite, cancellation); + return false; + }); + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken(It.IsAny())).Returns((string value) => value == Token); + Exception failure = storageStatus == 0 ? new IOException("Download failed.") : new RequestFailedException(storageStatus, "Download failed."); + store.Setup(s => s.DownloadAsync(Token, It.IsAny())).ThrowsAsync(failure); + string expectedOperation = kind switch + { + P.WorkItem.RequestOneofCase.ActivityRequest => "AbandonTaskActivityWorkItem", + P.WorkItem.RequestOneofCase.OrchestratorRequest => "AbandonTaskOrchestratorWorkItem", + _ => "AbandonTaskEntityWorkItem", + }; + List abandoned = []; + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (_, _) => CreateServerStreamingCall(reader.Object)) + { + UnaryResponse = (operation, request, _) => + { + Assert.Equal(expectedOperation, operation); + (string Token, object Reply) result = request switch + { + P.AbandonActivityTaskRequest r => (r.CompletionToken, new P.AbandonActivityTaskResponse()), + P.AbandonOrchestrationTaskRequest r => (r.CompletionToken, new P.AbandonOrchestrationTaskResponse()), + P.AbandonEntityTaskRequest r => (r.CompletionToken, new P.AbandonEntityTaskResponse()), + _ => throw new InvalidOperationException($"Unexpected request {request.GetType()}."), + }; + abandoned.Add(result.Token); + if (abandonStatus != StatusCode.OK) + { + throw new RpcException(new Status(abandonStatus, "Abandon failed.")); + } + + return Task.FromResult(result.Reply); + }, + }; + GrpcDurableTaskWorkerOptions options = new(); + options.Internal.TransientRetryMaxAttempts = 1; + options.Internal.ReconnectBackoffBase = options.Internal.ReconnectBackoffCap = TimeSpan.Zero; + object processor = CreateProcessor(CreateWorker(options), new P.TaskHubSidecarService.TaskHubSidecarServiceClient( + invoker.Intercept(new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions())))); + using CancellationTokenSource shutdown = new(); + + // Act + Task processing = InvokeProcessorExecuteAsync(processor, shutdown.Token); + try + { + await pingDispatched.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + finally + { + shutdown.Cancel(); + await processing; + } + + // Assert + Assert.Equal(new[] { "failed" }, abandoned); + Assert.Equal(abandonStatus is StatusCode.OK or StatusCode.NotFound ? 1 : 2, invoker.GetWorkItemsCallCount); + store.Verify(s => s.DownloadAsync(Token, It.IsAny()), Times.Once); + } + + [Theory] + [InlineData(2, false)] + [InlineData(2, true)] + [InlineData(0, false)] + public async Task ProcessWorkItemsAsync_FailedDeliveries_CountAsReceiptWithoutDispatchAsync(int failedCount, bool healthPing) + { + // Arrange + P.WorkItem[] items = Enumerable.Range(0, failedCount).Select(index => new P.WorkItem + { + CompletionToken = $"failed-{index}", + ActivityRequest = new P.ActivityRequest { Input = "missing-payload" }, + }).ToArray(); + if (healthPing) + { + items = [.. items, new P.WorkItem { HealthPing = new P.HealthPing() }]; + } + + Mock store = new(); + store.Setup(s => s.IsKnownPayloadToken("missing-payload")).Returns(true); + store.Setup(s => s.DownloadAsync("missing-payload", It.IsAny())) + .ThrowsAsync(new IOException("Download failed.")); + List abandoned = []; + int firstMessages = 0; + ScriptedWorkerCallInvoker invoker = new( + (_, _) => CreateUnaryCall(Task.FromResult(new Empty())), + (_, _) => CreateServerStreamingCall(new SequenceAsyncStreamReader(items))) + { + UnaryResponse = (operation, request, _) => + { + Assert.Equal(1, firstMessages); + Assert.Equal("AbandonTaskActivityWorkItem", operation); + abandoned.Add(Assert.IsType(request).CompletionToken); + return Task.FromResult(new P.AbandonActivityTaskResponse()); + }, + }; + int dispatched = 0; + int poisoned = 0; + GrpcDurableTaskWorkerOptions grpcOptions = new(); + grpcOptions.Internal.SilentDisconnectTimeout = TimeSpan.Zero; + grpcOptions.ConfigureActivityNotification(_ => dispatched++); + P.TaskHubSidecarService.TaskHubSidecarServiceClient client = new(invoker.Intercept( + new AzureBlobPayloadsSideCarInterceptor(store.Object, new LargePayloadStorageOptions()))); + object processor = CreateProcessor(CreateWorker(grpcOptions), client); + using AsyncServerStreamingCall stream = client.GetWorkItems(new()); + + // Act + await (Task)ProcessWorkItemsAsyncMethod.Invoke(processor, + [stream, CancellationToken.None, (Action)(() => firstMessages++), (Action)(() => poisoned++)])!; + + // Assert + bool observed = failedCount > 0 || healthPing; + Assert.Equal(observed ? 1 : 0, firstMessages); + Assert.Equal(observed ? 0 : 1, poisoned); + Assert.Equal(0, dispatched); + Assert.Equal(Enumerable.Range(0, failedCount).Select(index => $"failed-{index}"), abandoned); + Assert.Equal(1, invoker.GetWorkItemsCallCount); + } + [Fact] public async Task ExecuteAsync_ConnectFailureThreshold_RecreatesConfiguredChannel() { @@ -2199,6 +2359,8 @@ public ScriptedWorkerCallInvoker( public int GetWorkItemsCallCount => Volatile.Read(ref this.getWorkItemsCallCount); + public Func>? UnaryResponse { get; init; } + public override TResponse BlockingUnaryCall(Method method, string? host, CallOptions options, TRequest request) { throw new NotSupportedException(); @@ -2212,6 +2374,13 @@ public override AsyncUnaryCall AsyncUnaryCall(Me return (AsyncUnaryCall)(object)call; } + if (this.UnaryResponse is { } respond) + { + return CreateUnaryCall(ResponseAsync()); + + async Task ResponseAsync() => (TResponse)await respond(method.Name, request, options); + } + throw new NotSupportedException($"Unexpected unary method {method.FullName}."); } diff --git a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs index ba9104c1..96de52af 100644 --- a/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs +++ b/test/Worker/Grpc.Tests/WorkItemStreamConsumerTests.cs @@ -16,7 +16,7 @@ public class WorkItemStreamConsumerTests public async Task EmptyStream_ReturnsGracefulDrain() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => EmptyStream(), + openStream: (_, _) => EmptyStream(), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: _ => throw new InvalidOperationException("onItem should not be invoked"), onFirstMessage: () => throw new InvalidOperationException("onFirstMessage should not be invoked"), @@ -35,7 +35,7 @@ public async Task StreamWithItems_ReturnsGracefulDrain_AndFiresCallbacks() int firstMessageCount = 0; WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => StreamOf(item1, item2), + openStream: (_, _) => StreamOf(item1, item2), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: received.Add, onFirstMessage: () => firstMessageCount++, @@ -47,11 +47,35 @@ public async Task StreamWithItems_ReturnsGracefulDrain_AndFiresCallbacks() firstMessageCount.Should().Be(1); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NonYieldedReceipts_MarkStreamObservedWithoutDispatchAsync(bool yieldHealthyItem) + { + // Arrange + List received = []; + int firstMessages = 0; + + // Act + WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( + openStream: (_, observe) => StreamWithReceipts(observe, yieldHealthyItem), + silentDisconnectTimeout: TimeSpan.Zero, + onItem: received.Add, + onFirstMessage: () => firstMessages++, + cancellation: CancellationToken.None); + + // Assert + Assert.Equal(WorkItemStreamOutcome.GracefulDrain, result.Outcome); + Assert.True(result.FirstMessageObserved); + Assert.Equal(1, firstMessages); + Assert.Equal(yieldHealthyItem ? 1 : 0, received.Count); + } + [Fact] public async Task VeryLargeSilentDisconnectTimeout_IsClamped_AndStreamCanStillComplete() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => EmptyStream(), + openStream: (_, _) => EmptyStream(), silentDisconnectTimeout: TimeSpan.FromDays(365), onItem: _ => throw new InvalidOperationException("onItem should not be invoked"), onFirstMessage: null, @@ -65,7 +89,7 @@ public async Task VeryLargeSilentDisconnectTimeout_IsClamped_AndStreamCanStillCo public async Task HangingStream_SurfacingOce_ReturnsSilentDisconnect() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: false), silentDisconnectTimeout: ShortTimeout, onItem: _ => { }, onFirstMessage: null, @@ -85,7 +109,7 @@ public async Task HangingStream_SurfacingOce_ReturnsSilentDisconnect() public async Task HangingStream_SurfacingRpcCancelled_ReturnsSilentDisconnect() { WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: true), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: true), silentDisconnectTimeout: ShortTimeout, onItem: _ => { }, onFirstMessage: null, @@ -104,7 +128,7 @@ public async Task OuterCancellation_WithOceFromStream_ReturnsShutdown() outer.CancelAfter(ShortTimeout); WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: false), silentDisconnectTimeout: TimeSpan.FromSeconds(30), onItem: _ => { }, onFirstMessage: null, @@ -126,7 +150,7 @@ public async Task OuterCancellation_WithRpcCancelledFromStream_PropagatesExcepti outer.CancelAfter(ShortTimeout); Func act = () => WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: true), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: true), silentDisconnectTimeout: TimeSpan.FromSeconds(30), onItem: _ => { }, onFirstMessage: null, @@ -165,7 +189,7 @@ public async Task PerItem_HeartbeatReset_KeepsTimerAlive() } WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => StreamOf(items), + openStream: (_, _) => StreamOf(items), silentDisconnectTimeout: TimeSpan.FromMilliseconds(500), onItem: _ => events.Add($"item{itemIndex++}"), onFirstMessage: null, @@ -193,7 +217,7 @@ public async Task PerItem_HeartbeatReset_KeepsTimerAlive() public async Task UnrelatedRpcException_Propagates() { Func act = () => WorkItemStreamConsumer.ConsumeAsync( - openStream: _ => ThrowingStream(new RpcException(new Status(StatusCode.Unavailable, "boom"))), + openStream: (_, _) => ThrowingStream(new RpcException(new Status(StatusCode.Unavailable, "boom"))), silentDisconnectTimeout: TimeSpan.FromSeconds(5), onItem: _ => { }, onFirstMessage: null, @@ -213,7 +237,7 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti // Act WorkItemStreamResult result = await WorkItemStreamConsumer.ConsumeAsync( - openStream: ct => HangingStream(ct, throwAsRpc: false), + openStream: (ct, _) => HangingStream(ct, throwAsRpc: false), silentDisconnectTimeout: TimeSpan.FromMilliseconds(timeoutMilliseconds), onItem: _ => { }, onFirstMessage: null, @@ -224,6 +248,16 @@ public async Task NonPositiveSilentDisconnectTimeout_OnlyShutdownEndsLoop(int ti } #pragma warning disable CS1998 // Async method lacks 'await' operators + static async IAsyncEnumerable StreamWithReceipts(Action observe, bool yieldHealthyItem) + { + observe(); + observe(); + if (yieldHealthyItem) + { + yield return new P.WorkItem { HealthPing = new P.HealthPing() }; + } + } + static async IAsyncEnumerable EmptyStream() { yield break;