From 21bb8c4b78bfa596bfdeed72c0dfc257e9a73312 Mon Sep 17 00:00:00 2001 From: Taewoong Kim Date: Thu, 12 Nov 2026 19:25:47 +0900 Subject: [PATCH 1/2] Do not drop frames when a Netty frame write is interrupted With Netty IO, writeFrame waits for a non-writable channel to become writable before writing the frame. If the thread was interrupted while waiting, the frame was silently dropped. basicPublish returned as if the message had been sent, but it never reached the broker. When other frames of the same command had already been written, the broker received an incomplete command and closed the connection with UNEXPECTED_FRAME. writeFrame now keeps waiting when the thread is interrupted and restores the interrupt flag afterwards. The enqueuing timeout still bounds the wait, so an interrupted thread can block for up to that timeout before the frame is written or the write fails. Signed-off-by: Taewoong Kim --- .../client/impl/NettyFrameHandlerFactory.java | 45 +++++++--- .../com/rabbitmq/client/test/NettyTest.java | 86 +++++++++++++++++++ 2 files changed, 118 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java b/src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java index 27fc0836b5..7af47c2a32 100644 --- a/src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java +++ b/src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java @@ -16,6 +16,7 @@ package com.rabbitmq.client.impl; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import com.rabbitmq.client.AMQP; @@ -370,25 +371,43 @@ public void writeFrame(Frame frame) throws IOException { // the channel became writable this.doWriteFrame(frame); } else { - try { - // the channel is still non-writable - // in case its writability flipped, we have a reference to a latch that has been - // counted down - // so, worst case scenario, we'll enqueue only one frame right away - boolean canWriteNow = latch.await(enqueuingTimeout.toMillis(), MILLISECONDS); - if (canWriteNow) { - this.doWriteFrame(frame); - } else { - throw new IOException("Frame enqueuing failed"); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // the channel is still non-writable + // in case its writability flipped, we have a reference to a latch that has been + // counted down + // so, worst case scenario, we'll enqueue only one frame right away + boolean canWriteNow = awaitUninterruptibly(latch, enqueuingTimeout); + if (canWriteNow) { + this.doWriteFrame(frame); + } else { + throw new IOException("Frame enqueuing failed"); } } } } } + // the other frames of the command may already be enqueued, so dropping this one on interrupt + // would leave a partial command on the wire + private static boolean awaitUninterruptibly(CountDownLatch latch, Duration timeout) { + boolean interrupted = false; + try { + long remainingNanos = MILLISECONDS.toNanos(timeout.toMillis()); + long end = System.nanoTime() + remainingNanos; + while (true) { + try { + return latch.await(remainingNanos, NANOSECONDS); + } catch (InterruptedException e) { + interrupted = true; + remainingNanos = end - System.nanoTime(); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + private void doWriteFrame(Frame frame) throws IOException { ByteBuffer bbPayload = frame.getByteBufferPayload(); if (bbPayload != null) { diff --git a/src/test/java/com/rabbitmq/client/test/NettyTest.java b/src/test/java/com/rabbitmq/client/test/NettyTest.java index ac7efa1a2a..f7368b0f21 100644 --- a/src/test/java/com/rabbitmq/client/test/NettyTest.java +++ b/src/test/java/com/rabbitmq/client/test/NettyTest.java @@ -19,11 +19,15 @@ import static org.assertj.core.api.Assertions.assertThat; import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Address; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; import com.rabbitmq.client.DefaultConsumer; import com.rabbitmq.client.Envelope; +import com.rabbitmq.client.impl.Frame; +import com.rabbitmq.client.impl.FrameHandler; +import com.rabbitmq.client.impl.NettyFrameHandlerFactory; import io.netty.channel.EventLoopGroup; import io.netty.channel.IoHandlerFactory; import io.netty.channel.MultiThreadIoEventLoopGroup; @@ -31,13 +35,23 @@ import io.netty.channel.epoll.EpollSocketChannel; import io.netty.channel.kqueue.KQueueIoHandler; import io.netty.channel.kqueue.KQueueSocketChannel; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledOnOs; @@ -90,6 +104,78 @@ private static void nativeIoTest( } } + @Test + void interruptedFrameWriteWaitsForWritability() throws Exception { + Frame frame = + new Frame(AMQP.FRAME_BODY, 1, ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8))); + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + frame.writeTo(new DataOutputStream(expected)); + + AtomicReference nettyChannel = new AtomicReference<>(); + NettyFrameHandlerFactory factory = + new NettyFrameHandlerFactory( + null, + nettyChannel::set, + null, + null, + Duration.ofSeconds(30), + 10_000, + null, + AMQP.FRAME_MIN_SIZE, + false, + sse -> false); + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket server = new ServerSocket(0, 1, loopback)) { + FrameHandler frameHandler = + factory.create( + new Address(loopback.getHostAddress(), server.getLocalPort()), "interrupt-test"); + try (Socket socket = server.accept()) { + io.netty.channel.Channel ch = nettyChannel.get(); + setWritable(ch, false); + + CountDownLatch writeDone = new CountDownLatch(1); + AtomicReference writeError = new AtomicReference<>(); + AtomicBoolean interruptedAfterWrite = new AtomicBoolean(false); + Thread writer = + new Thread( + () -> { + try { + frameHandler.writeFrame(frame); + } catch (Throwable e) { + writeError.set(e); + } + interruptedAfterWrite.set(Thread.currentThread().isInterrupted()); + writeDone.countDown(); + }); + writer.start(); + TestUtils.waitAtMost(() -> writer.getState() == Thread.State.TIMED_WAITING); + writer.interrupt(); + assertThat(writeDone.await(500, TimeUnit.MILLISECONDS)).isFalse(); + + setWritable(ch, true); + assertThat(writeDone).is(completed()); + assertThat(writeError.get()).isNull(); + assertThat(interruptedAfterWrite).isTrue(); + + frameHandler.flush(null); + socket.setSoTimeout(10_000); + byte[] received = new byte[expected.size()]; + new DataInputStream(socket.getInputStream()).readFully(received); + assertThat(received).isEqualTo(expected.toByteArray()); + } finally { + frameHandler.close(); + } + } + } + + private static void setWritable(io.netty.channel.Channel ch, boolean writable) throws Exception { + ch.eventLoop() + .submit(() -> ch.unsafe().outboundBuffer().setUserDefinedWritability(1, writable)) + .get(10, TimeUnit.SECONDS); + // Netty fires the writability event in a later event loop task + ch.eventLoop().submit(() -> {}).get(10, TimeUnit.SECONDS); + } + private static void publishConsume(ConnectionFactory cf) throws IOException, TimeoutException { try (Connection c = cf.newConnection()) { Channel ch1 = c.createChannel(); From 20c246ab05e4defa899c505c214836a257f4e875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnaud=20Cogolu=C3=A8gnes?= <514737+acogoluegnes@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:34:15 +0100 Subject: [PATCH 2/2] Release blocked Netty frame writers when the channel closes A thread waiting in writeFrame for a non-writable channel to become writable was released only by a writability change. Netty does not fire that event when the channel closes, so the thread stayed blocked until the enqueuing timeout elapsed. Now that the wait can no longer be interrupted, nothing could cut it short. The writability latch is now counted down when the channel becomes inactive. writeFrame then fails with an IOException instead of writing to a closed channel. --- .../client/impl/NettyFrameHandlerFactory.java | 5 ++ .../com/rabbitmq/client/test/NettyTest.java | 68 +++++++++++++++---- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java b/src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java index 7af47c2a32..d5af754ce8 100644 --- a/src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java +++ b/src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java @@ -377,6 +377,9 @@ public void writeFrame(Frame frame) throws IOException { // so, worst case scenario, we'll enqueue only one frame right away boolean canWriteNow = awaitUninterruptibly(latch, enqueuingTimeout); if (canWriteNow) { + if (!this.channel.isActive()) { + throw new IOException("Channel closed while waiting for frame enqueuing"); + } this.doWriteFrame(frame); } else { throw new IOException("Frame enqueuing failed"); @@ -611,6 +614,8 @@ public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exceptio @Override public void channelInactive(ChannelHandlerContext ctx) { + // the channel will not become writable again, release the threads waiting to write + this.writableLatch.get().countDown(); if (needToDispatchIoError()) { AMQConnection c = this.connection; LOGGER.debug("Dispatching shutdown when channel became inactive ({})", this.id); diff --git a/src/test/java/com/rabbitmq/client/test/NettyTest.java b/src/test/java/com/rabbitmq/client/test/NettyTest.java index f7368b0f21..6ed4336c99 100644 --- a/src/test/java/com/rabbitmq/client/test/NettyTest.java +++ b/src/test/java/com/rabbitmq/client/test/NettyTest.java @@ -112,18 +112,7 @@ void interruptedFrameWriteWaitsForWritability() throws Exception { frame.writeTo(new DataOutputStream(expected)); AtomicReference nettyChannel = new AtomicReference<>(); - NettyFrameHandlerFactory factory = - new NettyFrameHandlerFactory( - null, - nettyChannel::set, - null, - null, - Duration.ofSeconds(30), - 10_000, - null, - AMQP.FRAME_MIN_SIZE, - false, - sse -> false); + NettyFrameHandlerFactory factory = frameHandlerFactory(nettyChannel); InetAddress loopback = InetAddress.getLoopbackAddress(); try (ServerSocket server = new ServerSocket(0, 1, loopback)) { FrameHandler frameHandler = @@ -168,6 +157,61 @@ void interruptedFrameWriteWaitsForWritability() throws Exception { } } + @Test + void frameWriteWaitingForWritabilityFailsWhenChannelIsClosed() throws Exception { + Frame frame = + new Frame(AMQP.FRAME_BODY, 1, ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8))); + AtomicReference nettyChannel = new AtomicReference<>(); + NettyFrameHandlerFactory factory = frameHandlerFactory(nettyChannel); + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket server = new ServerSocket(0, 1, loopback)) { + FrameHandler frameHandler = + factory.create( + new Address(loopback.getHostAddress(), server.getLocalPort()), "close-test"); + try (Socket ignored = server.accept()) { + io.netty.channel.Channel ch = nettyChannel.get(); + setWritable(ch, false); + + CountDownLatch writeDone = new CountDownLatch(1); + AtomicReference writeError = new AtomicReference<>(); + Thread writer = + new Thread( + () -> { + try { + frameHandler.writeFrame(frame); + } catch (Throwable e) { + writeError.set(e); + } + writeDone.countDown(); + }); + writer.start(); + TestUtils.waitAtMost(() -> writer.getState() == Thread.State.TIMED_WAITING); + + ch.close().get(10, TimeUnit.SECONDS); + // the enqueuing timeout is 30 seconds, the writer must be released well before + assertThat(writeDone).is(completed()); + assertThat(writeError.get()).isInstanceOf(IOException.class); + } finally { + frameHandler.close(); + } + } + } + + private static NettyFrameHandlerFactory frameHandlerFactory( + AtomicReference nettyChannel) { + return new NettyFrameHandlerFactory( + null, + nettyChannel::set, + null, + null, + Duration.ofSeconds(30), + 10_000, + null, + AMQP.FRAME_MIN_SIZE, + false, + sse -> false); + } + private static void setWritable(io.netty.channel.Channel ch, boolean writable) throws Exception { ch.eventLoop() .submit(() -> ch.unsafe().outboundBuffer().setUserDefinedWritability(1, writable))