fix(software-live): play audio far from its video on the video's axis

CHANGELOG.md

@@ -10,7 +10,13 @@ the public-API contract.
 
 ## [Unreleased]
 
-_Nothing yet._
+### Fixed
+
+- **A live MPEG-TS source whose audio runs far from its video plays on one clock.** With audio more than a minute
+  behind video, FFmpeg's 33-bit wrap correction puts the audio about 26.5 h ahead. On the software path that audio set
+  the live edge, so the rewind buffer discarded each video keyframe as older than its window, recovery seeked about
+  once a second and the clock barely ran. `LiveAudioAxis` now moves audio more than 30 s from the video onto the
+  video's axis, aligned on arrival. `LiveAudioAxisTests` covers the wrap, seams and a still image over sound.
 
 ## [7.56.0] - 2026-11-03
 

Sources/AetherEngine/Native/SoftwarePlaybackHost.swift

@@ -3172,6 +3172,7 @@ final class SoftwarePlaybackHost {
         var frameIntervalSec = 0.0
         var discontinuityOffsetSec = 0.0
         var loggedSWDiscontinuity = false
+        var audioAxis = LiveAudioAxis()
         var lastSeenExtradata: Data? = nil
         let replayReader = LiveReplayPacketReader(videoIndex: videoStreamIndex, audioIndex: audioStreamIndex,
             videoTimeBase: videoTimeBaseSeconds, audioTimeBase: audioTimeBaseSeconds)
@@ -3256,6 +3257,14 @@ final class SoftwarePlaybackHost {
                 packet.pointee.pts = packet.pointee.dts
             }
 
+            // Audio on another axis than the video joins it before anything reads its timestamps.
+            if !applyLiveAudioAxis(&audioAxis, to: packet, videoIndex: videoStreamIndex, audioIndex: audioStreamIndex,
+                                   videoTimeBase: videoTimeBaseSeconds, audioTimeBase: audioTimeBaseSeconds) {
+                av_packet_unref(packet)
+                av_packet_free_safe(packet)
+                return true
+            }
+
             // Live PTS-discontinuity: same accrual as combined loop.
             if streamIdx == videoStreamIndex, videoTimeBaseSeconds > 0,
                packet.pointee.pts != Int64.min {
@@ -3338,6 +3347,34 @@ final class SoftwarePlaybackHost {
         }
     }
 
+    /// Moves a live audio packet onto the video's axis (`LiveAudioAxis`). False when it is to be dropped.
+    nonisolated private static func applyLiveAudioAxis(
+        _ axis: inout LiveAudioAxis,
+        to packet: UnsafeMutablePointer<AVPacket>,
+        videoIndex: Int32,
+        audioIndex: Int32,
+        videoTimeBase: Double,
+        audioTimeBase: Double
+    ) -> Bool {
+        guard videoIndex >= 0, audioIndex >= 0, packet.pointee.pts != Int64.min else { return true }
+        if packet.pointee.stream_index == videoIndex {
+            if videoTimeBase > 0 { axis.noteVideo(ptsSeconds: Double(packet.pointee.pts) * videoTimeBase) }
+            return true
+        }
+        guard packet.pointee.stream_index == audioIndex, audioTimeBase > 0 else { return true }
+        switch axis.audio(ptsSeconds: Double(packet.pointee.pts) * audioTimeBase) {
+        case .drop:
+            return false
+        case .play(let shiftSeconds):
+            if shiftSeconds != 0 {
+                let ticks = SourceTimestampBounds.roundedTicks(shiftSeconds / audioTimeBase)
+                packet.pointee.pts = SourceTimestampBounds.shifted(packet.pointee.pts, back: ticks)
+                packet.pointee.dts = SourceTimestampBounds.shifted(packet.pointee.dts, back: ticks)
+            }
+            return true
+        }
+    }
+
     // MARK: - Live feeder loop (DVR sessions)
 
     /// Ring cursor -> decoders -> renderer with back-pressure. Pause = timeshift (reader keeps filling); DVR seek = cursor move.
@@ -3895,6 +3932,7 @@ final class SoftwarePlaybackHost {
         var frameIntervalSec = 0.0
         var discontinuityOffsetSec = 0.0
         var loggedSeamCount = 0
+        var audioAxis = LiveAudioAxis()
         let replayReader = isLive && directLiveSource == nil ? LiveReplayPacketReader(videoIndex: videoStreamIndex,
             audioIndex: audioStreamIndex, videoTimeBase: videoTimeBaseSeconds,
             audioTimeBase: audioTimeBaseSeconds) : nil
@@ -4330,6 +4368,14 @@ final class SoftwarePlaybackHost {
 
             let streamIdx = packet.pointee.stream_index
 
+            // Audio on another axis than the video joins it before anything reads its timestamps.
+            if isLive, !applyLiveAudioAxis(&audioAxis, to: packet, videoIndex: videoStreamIndex, audioIndex: audioStreamIndex,
+                                           videoTimeBase: videoTimeBaseSeconds, audioTimeBase: audioTimeBaseSeconds) {
+                av_packet_unref(packet)
+                av_packet_free_safe(packet)
+                return true
+            }
+
             // Discontinuity runs before any timestamp read; offset applied to both streams.
             if foldTimeline, streamIdx == videoStreamIndex, videoTimeBaseSeconds > 0,
                packet.pointee.pts != Int64.min {

Sources/AetherEngine/Video/LiveAudioAxis.swift

@@ -0,0 +1,79 @@
+import Foundation
+
+/// A live source whose audio stamps sit far from its video stamps plays on one clock only once the
+/// audio is moved onto the video's axis. The packets arrive interleaved, so the video packet last
+/// read says where the audio belongs.
+///
+/// An MPEG-TS source does this when its audio runs more than a minute behind its video: FFmpeg's
+/// 33-bit wrap correction takes the audio for a stream that wrapped and adds 95,443 s to it. Left
+/// there the audio sets the live edge, the rewind buffer discards every video packet as older than
+/// its window, and the clock stands on an axis no picture is on.
+struct LiveAudioAxis {
+    enum Decision: Equatable {
+        /// Plays, this many seconds earlier than stamped.
+        case play(shiftSeconds: Double)
+        case drop
+    }
+
+    /// Further apart than this, audio and video are not one timeline. A muxer's interleave is seconds.
+    static let farApartSeconds = 30.0
+    /// A source's clocks that disagree for this many audio packets (about 4 s of AAC) have moved
+    /// apart for good; fewer is one stream crossing a seam ahead of the other.
+    static let audioPacketsBeforeMoving = 200
+    /// A picture this sparse is a still image over sound, which says nothing about the audio's axis.
+    static let videoPacketsBeforeMoving = 20
+    /// With no video by this much audio there may be none coming.
+    static let audioSecondsWithoutVideo = 5.0
+
+    private var lastVideoSeconds = Double.nan
+    private var firstAudioSeconds = Double.nan
+    private var shiftSeconds = 0.0
+    private var hasCompared = false
+    private var disagreeingAudioPackets = 0
+    private var videoPacketsWhileDisagreeing = 0
+
+    mutating func noteVideo(ptsSeconds: Double) {
+        lastVideoSeconds = ptsSeconds
+        if disagreeingAudioPackets > 0 {
+            videoPacketsWhileDisagreeing += 1
+        }
+    }
+
+    mutating func audio(ptsSeconds: Double) -> Decision {
+        guard !lastVideoSeconds.isNaN else {
+            if firstAudioSeconds.isNaN { firstAudioSeconds = ptsSeconds }
+            return ptsSeconds - firstAudioSeconds < Self.audioSecondsWithoutVideo
+                ? .drop
+                : .play(shiftSeconds: 0)
+        }
+        guard abs(ptsSeconds - shiftSeconds - lastVideoSeconds) > Self.farApartSeconds else {
+            hasCompared = true
+            disagreeingAudioPackets = 0
+            videoPacketsWhileDisagreeing = 0
+            return .play(shiftSeconds: shiftSeconds)
+        }
+        guard hasCompared else {
+            hasCompared = true
+            moveOntoVideo(ptsSeconds)
+            return .play(shiftSeconds: shiftSeconds)
+        }
+        disagreeingAudioPackets += 1
+        if disagreeingAudioPackets >= Self.audioPacketsBeforeMoving,
+           videoPacketsWhileDisagreeing >= Self.videoPacketsBeforeMoving {
+            moveOntoVideo(ptsSeconds)
+            return .play(shiftSeconds: shiftSeconds)
+        }
+        return videoPacketsWhileDisagreeing > 0 ? .drop : .play(shiftSeconds: shiftSeconds)
+    }
+
+    private mutating func moveOntoVideo(_ ptsSeconds: Double) {
+        shiftSeconds = ptsSeconds - lastVideoSeconds
+        disagreeingAudioPackets = 0
+        videoPacketsWhileDisagreeing = 0
+        EngineLog.emit(
+            "[LiveAudioAxis] audio stamps are \(String(format: "%.2f", shiftSeconds))s from video; "
+            + "playing the audio on the video's axis",
+            category: .swPlayback
+        )
+    }
+}

Tests/AetherEngineTests/LiveAudioAxisTests.swift

@@ -0,0 +1,78 @@
+import Testing
+@testable import AetherEngine
+
+/// A live source's audio and video stamps can sit a day apart; the audio then has to play on the
+/// video's axis, while audio and video that merely cross a seam at different moments stay as stamped.
+@Suite("Live audio axis")
+struct LiveAudioAxisTests {
+    /// What FFmpeg's wrap correction makes of audio 274 s behind its video.
+    private static let wrappedOffset = 95_169.6
+
+    @Test("audio a day from its video plays on the video's axis from the first packet")
+    func wrappedAudioMovesAtOnce() {
+        var axis = LiveAudioAxis()
+        axis.noteVideo(ptsSeconds: 13_914)
+        #expect(axis.audio(ptsSeconds: 13_914 + Self.wrappedOffset) == .play(shiftSeconds: Self.wrappedOffset))
+        axis.noteVideo(ptsSeconds: 13_915)
+        #expect(axis.audio(ptsSeconds: 13_915 + Self.wrappedOffset) == .play(shiftSeconds: Self.wrappedOffset))
+    }
+
+    @Test("audio near its video keeps its stamps")
+    func nearbyAudioStays() {
+        var axis = LiveAudioAxis()
+        axis.noteVideo(ptsSeconds: 100)
+        #expect(axis.audio(ptsSeconds: 103) == .play(shiftSeconds: 0))
+    }
+
+    @Test("audio before the first video waits for it, for a while")
+    func audioBeforeVideo() {
+        var axis = LiveAudioAxis()
+        #expect(axis.audio(ptsSeconds: 500) == .drop)
+        #expect(axis.audio(ptsSeconds: 504.9) == .drop)
+        #expect(axis.audio(ptsSeconds: 505) == .play(shiftSeconds: 0))
+    }
+
+    @Test("a seam the audio crosses first drops only until the video follows")
+    func seamCrossedByAudioFirst() {
+        var axis = LiveAudioAxis()
+        axis.noteVideo(ptsSeconds: 100)
+        #expect(axis.audio(ptsSeconds: 100) == .play(shiftSeconds: 0))
+        #expect(axis.audio(ptsSeconds: 5_000) == .play(shiftSeconds: 0), "no video has moved yet")
+        axis.noteVideo(ptsSeconds: 100.04)
+        #expect(axis.audio(ptsSeconds: 5_000.02) == .drop)
+        axis.noteVideo(ptsSeconds: 5_000.04)
+        #expect(axis.audio(ptsSeconds: 5_000.06) == .play(shiftSeconds: 0))
+    }
+
+    @Test("clocks that move apart mid-session move the audio once both have shown it for good")
+    func clocksMoveApartForGood() {
+        var axis = LiveAudioAxis()
+        axis.noteVideo(ptsSeconds: 100)
+        #expect(axis.audio(ptsSeconds: 100) == .play(shiftSeconds: 0))
+        var video = 100.0
+        var audio = 1_100.0
+        var last = LiveAudioAxis.Decision.drop
+        for _ in 0..<LiveAudioAxis.audioPacketsBeforeMoving {
+            video += 0.02
+            audio += 0.02
+            axis.noteVideo(ptsSeconds: video)
+            last = axis.audio(ptsSeconds: audio)
+        }
+        #expect(last == .play(shiftSeconds: audio - video))
+    }
+
+    /// One picture a minute over continuous sound: the sound is never dropped or moved.
+    @Test("sound under a sparse still image is left alone")
+    func sparseVideoKeepsSound() {
+        var axis = LiveAudioAxis()
+        axis.noteVideo(ptsSeconds: 0)
+        var audio = 0.0
+        for frame in 1...5 {
+            while audio < Double(frame) * 60 {
+                audio += 0.02
+                #expect(axis.audio(ptsSeconds: audio) == .play(shiftSeconds: 0))
+            }
+            axis.noteVideo(ptsSeconds: Double(frame) * 60)
+        }
+    }
+}