From 6c81b3646dd66d349c7ac70fd560d411686dff87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santos?= Date: Tue, 3 Nov 2026 21:57:06 +0100 Subject: [PATCH] fix(software-live): play audio far from its video on the video's axis --- CHANGELOG.md | 8 +- .../Native/SoftwarePlaybackHost.swift | 46 +++++++++++ .../AetherEngine/Video/LiveAudioAxis.swift | 79 +++++++++++++++++++ .../LiveAudioAxisTests.swift | 78 ++++++++++++++++++ 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 Sources/AetherEngine/Video/LiveAudioAxis.swift create mode 100644 Tests/AetherEngineTests/LiveAudioAxisTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 6390cd398..0454a7f73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,13 @@ the public-API contract. ## [Unreleased] -_Nothing yet._ +### Fixed + +- **A live MPEG-TS source whose audio runs far from its video plays on one clock.** With audio more than a minute + behind video, FFmpeg's 33-bit wrap correction puts the audio about 26.5 h ahead. On the software path that audio set + the live edge, so the rewind buffer discarded each video keyframe as older than its window, recovery seeked about + once a second and the clock barely ran. `LiveAudioAxis` now moves audio more than 30 s from the video onto the + video's axis, aligned on arrival. `LiveAudioAxisTests` covers the wrap, seams and a still image over sound. ## [7.56.0] - 2026-11-03 diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index ded76ccc6..b3393f95c 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -3172,6 +3172,7 @@ final class SoftwarePlaybackHost { var frameIntervalSec = 0.0 var discontinuityOffsetSec = 0.0 var loggedSWDiscontinuity = false + var audioAxis = LiveAudioAxis() var lastSeenExtradata: Data? = nil let replayReader = LiveReplayPacketReader(videoIndex: videoStreamIndex, audioIndex: audioStreamIndex, videoTimeBase: videoTimeBaseSeconds, audioTimeBase: audioTimeBaseSeconds) @@ -3256,6 +3257,14 @@ final class SoftwarePlaybackHost { packet.pointee.pts = packet.pointee.dts } + // Audio on another axis than the video joins it before anything reads its timestamps. + if !applyLiveAudioAxis(&audioAxis, to: packet, videoIndex: videoStreamIndex, audioIndex: audioStreamIndex, + videoTimeBase: videoTimeBaseSeconds, audioTimeBase: audioTimeBaseSeconds) { + av_packet_unref(packet) + av_packet_free_safe(packet) + return true + } + // Live PTS-discontinuity: same accrual as combined loop. if streamIdx == videoStreamIndex, videoTimeBaseSeconds > 0, packet.pointee.pts != Int64.min { @@ -3338,6 +3347,34 @@ final class SoftwarePlaybackHost { } } + /// Moves a live audio packet onto the video's axis (`LiveAudioAxis`). False when it is to be dropped. + nonisolated private static func applyLiveAudioAxis( + _ axis: inout LiveAudioAxis, + to packet: UnsafeMutablePointer, + videoIndex: Int32, + audioIndex: Int32, + videoTimeBase: Double, + audioTimeBase: Double + ) -> Bool { + guard videoIndex >= 0, audioIndex >= 0, packet.pointee.pts != Int64.min else { return true } + if packet.pointee.stream_index == videoIndex { + if videoTimeBase > 0 { axis.noteVideo(ptsSeconds: Double(packet.pointee.pts) * videoTimeBase) } + return true + } + guard packet.pointee.stream_index == audioIndex, audioTimeBase > 0 else { return true } + switch axis.audio(ptsSeconds: Double(packet.pointee.pts) * audioTimeBase) { + case .drop: + return false + case .play(let shiftSeconds): + if shiftSeconds != 0 { + let ticks = SourceTimestampBounds.roundedTicks(shiftSeconds / audioTimeBase) + packet.pointee.pts = SourceTimestampBounds.shifted(packet.pointee.pts, back: ticks) + packet.pointee.dts = SourceTimestampBounds.shifted(packet.pointee.dts, back: ticks) + } + return true + } + } + // MARK: - Live feeder loop (DVR sessions) /// Ring cursor -> decoders -> renderer with back-pressure. Pause = timeshift (reader keeps filling); DVR seek = cursor move. @@ -3895,6 +3932,7 @@ final class SoftwarePlaybackHost { var frameIntervalSec = 0.0 var discontinuityOffsetSec = 0.0 var loggedSeamCount = 0 + var audioAxis = LiveAudioAxis() let replayReader = isLive && directLiveSource == nil ? LiveReplayPacketReader(videoIndex: videoStreamIndex, audioIndex: audioStreamIndex, videoTimeBase: videoTimeBaseSeconds, audioTimeBase: audioTimeBaseSeconds) : nil @@ -4330,6 +4368,14 @@ final class SoftwarePlaybackHost { let streamIdx = packet.pointee.stream_index + // Audio on another axis than the video joins it before anything reads its timestamps. + if isLive, !applyLiveAudioAxis(&audioAxis, to: packet, videoIndex: videoStreamIndex, audioIndex: audioStreamIndex, + videoTimeBase: videoTimeBaseSeconds, audioTimeBase: audioTimeBaseSeconds) { + av_packet_unref(packet) + av_packet_free_safe(packet) + return true + } + // Discontinuity runs before any timestamp read; offset applied to both streams. if foldTimeline, streamIdx == videoStreamIndex, videoTimeBaseSeconds > 0, packet.pointee.pts != Int64.min { diff --git a/Sources/AetherEngine/Video/LiveAudioAxis.swift b/Sources/AetherEngine/Video/LiveAudioAxis.swift new file mode 100644 index 000000000..ac9268922 --- /dev/null +++ b/Sources/AetherEngine/Video/LiveAudioAxis.swift @@ -0,0 +1,79 @@ +import Foundation + +/// A live source whose audio stamps sit far from its video stamps plays on one clock only once the +/// audio is moved onto the video's axis. The packets arrive interleaved, so the video packet last +/// read says where the audio belongs. +/// +/// An MPEG-TS source does this when its audio runs more than a minute behind its video: FFmpeg's +/// 33-bit wrap correction takes the audio for a stream that wrapped and adds 95,443 s to it. Left +/// there the audio sets the live edge, the rewind buffer discards every video packet as older than +/// its window, and the clock stands on an axis no picture is on. +struct LiveAudioAxis { + enum Decision: Equatable { + /// Plays, this many seconds earlier than stamped. + case play(shiftSeconds: Double) + case drop + } + + /// Further apart than this, audio and video are not one timeline. A muxer's interleave is seconds. + static let farApartSeconds = 30.0 + /// A source's clocks that disagree for this many audio packets (about 4 s of AAC) have moved + /// apart for good; fewer is one stream crossing a seam ahead of the other. + static let audioPacketsBeforeMoving = 200 + /// A picture this sparse is a still image over sound, which says nothing about the audio's axis. + static let videoPacketsBeforeMoving = 20 + /// With no video by this much audio there may be none coming. + static let audioSecondsWithoutVideo = 5.0 + + private var lastVideoSeconds = Double.nan + private var firstAudioSeconds = Double.nan + private var shiftSeconds = 0.0 + private var hasCompared = false + private var disagreeingAudioPackets = 0 + private var videoPacketsWhileDisagreeing = 0 + + mutating func noteVideo(ptsSeconds: Double) { + lastVideoSeconds = ptsSeconds + if disagreeingAudioPackets > 0 { + videoPacketsWhileDisagreeing += 1 + } + } + + mutating func audio(ptsSeconds: Double) -> Decision { + guard !lastVideoSeconds.isNaN else { + if firstAudioSeconds.isNaN { firstAudioSeconds = ptsSeconds } + return ptsSeconds - firstAudioSeconds < Self.audioSecondsWithoutVideo + ? .drop + : .play(shiftSeconds: 0) + } + guard abs(ptsSeconds - shiftSeconds - lastVideoSeconds) > Self.farApartSeconds else { + hasCompared = true + disagreeingAudioPackets = 0 + videoPacketsWhileDisagreeing = 0 + return .play(shiftSeconds: shiftSeconds) + } + guard hasCompared else { + hasCompared = true + moveOntoVideo(ptsSeconds) + return .play(shiftSeconds: shiftSeconds) + } + disagreeingAudioPackets += 1 + if disagreeingAudioPackets >= Self.audioPacketsBeforeMoving, + videoPacketsWhileDisagreeing >= Self.videoPacketsBeforeMoving { + moveOntoVideo(ptsSeconds) + return .play(shiftSeconds: shiftSeconds) + } + return videoPacketsWhileDisagreeing > 0 ? .drop : .play(shiftSeconds: shiftSeconds) + } + + private mutating func moveOntoVideo(_ ptsSeconds: Double) { + shiftSeconds = ptsSeconds - lastVideoSeconds + disagreeingAudioPackets = 0 + videoPacketsWhileDisagreeing = 0 + EngineLog.emit( + "[LiveAudioAxis] audio stamps are \(String(format: "%.2f", shiftSeconds))s from video; " + + "playing the audio on the video's axis", + category: .swPlayback + ) + } +} diff --git a/Tests/AetherEngineTests/LiveAudioAxisTests.swift b/Tests/AetherEngineTests/LiveAudioAxisTests.swift new file mode 100644 index 000000000..691cfcf1b --- /dev/null +++ b/Tests/AetherEngineTests/LiveAudioAxisTests.swift @@ -0,0 +1,78 @@ +import Testing +@testable import AetherEngine + +/// A live source's audio and video stamps can sit a day apart; the audio then has to play on the +/// video's axis, while audio and video that merely cross a seam at different moments stay as stamped. +@Suite("Live audio axis") +struct LiveAudioAxisTests { + /// What FFmpeg's wrap correction makes of audio 274 s behind its video. + private static let wrappedOffset = 95_169.6 + + @Test("audio a day from its video plays on the video's axis from the first packet") + func wrappedAudioMovesAtOnce() { + var axis = LiveAudioAxis() + axis.noteVideo(ptsSeconds: 13_914) + #expect(axis.audio(ptsSeconds: 13_914 + Self.wrappedOffset) == .play(shiftSeconds: Self.wrappedOffset)) + axis.noteVideo(ptsSeconds: 13_915) + #expect(axis.audio(ptsSeconds: 13_915 + Self.wrappedOffset) == .play(shiftSeconds: Self.wrappedOffset)) + } + + @Test("audio near its video keeps its stamps") + func nearbyAudioStays() { + var axis = LiveAudioAxis() + axis.noteVideo(ptsSeconds: 100) + #expect(axis.audio(ptsSeconds: 103) == .play(shiftSeconds: 0)) + } + + @Test("audio before the first video waits for it, for a while") + func audioBeforeVideo() { + var axis = LiveAudioAxis() + #expect(axis.audio(ptsSeconds: 500) == .drop) + #expect(axis.audio(ptsSeconds: 504.9) == .drop) + #expect(axis.audio(ptsSeconds: 505) == .play(shiftSeconds: 0)) + } + + @Test("a seam the audio crosses first drops only until the video follows") + func seamCrossedByAudioFirst() { + var axis = LiveAudioAxis() + axis.noteVideo(ptsSeconds: 100) + #expect(axis.audio(ptsSeconds: 100) == .play(shiftSeconds: 0)) + #expect(axis.audio(ptsSeconds: 5_000) == .play(shiftSeconds: 0), "no video has moved yet") + axis.noteVideo(ptsSeconds: 100.04) + #expect(axis.audio(ptsSeconds: 5_000.02) == .drop) + axis.noteVideo(ptsSeconds: 5_000.04) + #expect(axis.audio(ptsSeconds: 5_000.06) == .play(shiftSeconds: 0)) + } + + @Test("clocks that move apart mid-session move the audio once both have shown it for good") + func clocksMoveApartForGood() { + var axis = LiveAudioAxis() + axis.noteVideo(ptsSeconds: 100) + #expect(axis.audio(ptsSeconds: 100) == .play(shiftSeconds: 0)) + var video = 100.0 + var audio = 1_100.0 + var last = LiveAudioAxis.Decision.drop + for _ in 0..