diff --git a/apps/docs/content/api/canvas.mdx b/apps/docs/content/api/canvas.mdx index dc34e4594..28d5b98a8 100644 --- a/apps/docs/content/api/canvas.mdx +++ b/apps/docs/content/api/canvas.mdx @@ -154,7 +154,7 @@ On Android, `Canvas` can be backed by one of three native views, with different | Opaque by default | no | yes | never | | Status | default | opt-in | experimental, opt-in | -**`TextureView`** is the default: a regular view, so transforms, clipping, alpha and z-order all apply, and the canvas can be composited over other React Native views. The cost is an extra copy, typically a frame of latency, and occasional stalls during some view animations. +**`TextureView`** is the default: a regular view, so transforms, clipping, alpha and z-order all apply, and the canvas can be composited over other React Native views. The cost is an extra copy, typically a frame of latency, and occasional stalls during some view animations. A frame presented on the UI thread, from a worklet, reaches the `TextureView` behind the draw its window has pending, so that the window draws every frame. On Android 11 a `TextureView` takes no new frame before its window has drawn the last one, so a frame that comes sooner renders offscreen and is copied in once the window has drawn. **`SurfaceView`** is the fastest option. It is not regular view content, though: parent transforms, clipping, rounded corners and z-ordering with sibling views do not apply to it. Use it for a canvas that is a plain rectangle: a full-screen scene or a game. It is opaque unless asked otherwise: a translucent `SurfaceView` blends against the window background (usually black) instead of the React Native views beneath it, which are punched out. Pair it with `zOrderOnTop` to composite over React Native content, and note that `zOrderOnTop` draws above all React Native views in the window, including navigation screens, regardless of `zIndex`. diff --git a/apps/example/src/Tests.tsx b/apps/example/src/Tests.tsx index e62dfd551..e4142d730 100644 --- a/apps/example/src/Tests.tsx +++ b/apps/example/src/Tests.tsx @@ -7,7 +7,9 @@ import { PixelRatio, findNodeHandle, } from "react-native"; +import type { CanvasRef } from "react-native-webgpu"; import { + Canvas, GPUOffscreenCanvas, // The flags as the package exports them, not the globals of the same name // (see the `exportedFlags` spec in Constants.spec.ts). @@ -25,6 +27,7 @@ import { redFragWGSL, triangleVertWGSL } from "./Triangle/triangle"; import type { AssetProps } from "./components/useAssets"; import { Texture } from "./components/Texture"; import { workletDeviceStress } from "./components/workletDeviceStress"; +import { presentFrames } from "./components/presentFrames"; import { workletSerialization } from "./components/workletSerialization"; export const CI = process.env.CI === "true"; @@ -33,6 +36,7 @@ const { width } = Dimensions.get("window"); const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); const PROBE_SIZE = 64; const QUADRANT = PROBE_SIZE / 2; +const TEXTURE_CANVAS_SIZE = 32; export const Tests = ({ assets: { di3D, saturn, moon } }: AssetProps) => { const [texture, setTexture] = useState(null); @@ -42,6 +46,10 @@ export const Tests = ({ assets: { di3D, saturn, moon } }: AssetProps) => { // A fixed native view the drawElementImageToTexture spec snapshots: four // 32x32pt solid quadrants (red, lime, blue, white), 64x64pt in total. const probeRef = useRef(null); + // A canvas backed by a TextureView on Android, for the TextureView + // present spec, and the view around it, which that spec snapshots. + const textureCanvasRef = useRef(null); + const textureCanvasViewRef = useRef(null); useEffect(() => { let cancelled = false; (async () => { @@ -129,8 +137,14 @@ export const Tests = ({ assets: { di3D, saturn, moon } }: AssetProps) => { probe: findNodeHandle(probeRef.current), probeSize: PROBE_SIZE, pixelRatio: PixelRatio.get(), + textureCanvas: () => textureCanvasRef.current, + textureCanvasView: findNodeHandle( + textureCanvasViewRef.current, + ), + textureCanvasSize: TEXTURE_CANVAS_SIZE, }, workletDeviceStress, + presentFrames, workletSerialization, exportedFlags: { GPUBufferUsage: exportedBufferUsage, @@ -229,6 +243,17 @@ export const Tests = ({ assets: { di3D, saturn, moon } }: AssetProps) => { }} /> + + + void, +) => { + "worklet"; + const present = (color: GPUColor) => { + const encoder = device.createCommandEncoder(); + encoder + .beginRenderPass({ + colorAttachments: [ + { + view: context.getCurrentTexture().createView(), + clearValue: color, + loadOp: "clear", + storeOp: "store", + }, + ], + }) + .end(); + device.queue.submit([encoder.finish()]); + context.present(); + }; + let next = 0; + let previousFrameMs = -1; + let longestFrameGapMs = 0; + const onFrame = (callback: () => void) => () => { + const nowMs = performance.now(); + if (previousFrameMs >= 0) { + longestFrameGapMs = Math.max(longestFrameGapMs, nowMs - previousFrameMs); + } + previousFrameMs = nowMs; + callback(); + }; + const step = () => { + if (next === 0) { + present(colors[next++]); + } + if (next < colors.length) { + present(colors[next++]); + requestAnimationFrame(onFrame(step)); + return; + } + requestAnimationFrame( + onFrame(() => + requestAnimationFrame(onFrame(() => done(longestFrameGapMs))), + ), + ); + }; + requestAnimationFrame(onFrame(step)); +}; + +// Presents those frames from Reanimated's UI thread or from the JS thread. +export const presentFrames = ( + device: GPUDevice, + context: RNCanvasContext, + colors: GPUColor[], + thread: "UI" | "JS", +): Promise => + new Promise((resolve) => { + if (thread === "JS") { + presentColors(device, context, colors, resolve); + return; + } + scheduleOnUI(() => { + "worklet"; + presentColors(device, context, colors, (longestFrameGapMs) => + scheduleOnRN(resolve, longestFrameGapMs), + ); + }); + }); diff --git a/packages/webgpu/android/cpp/cpp-adapter.cpp b/packages/webgpu/android/cpp/cpp-adapter.cpp index b3768488c..f2ac19cfa 100644 --- a/packages/webgpu/android/cpp/cpp-adapter.cpp +++ b/packages/webgpu/android/cpp/cpp-adapter.cpp @@ -165,9 +165,11 @@ namespace { // Weak handle on a Java view and one of its no-argument methods, for native // threads to call into it: the fence waiter wakes a WebGPUHardwareBufferView // when a frame is ready, the presenting thread tells a SurfaceView it is -// animating (AnimationReporter). Holding the view weakly means a view that RN -// has dropped is not kept alive by a SurfaceInfo that outlives it (the -// registry keeps the SurfaceInfo until onDropViewInstance). +// animating (AnimationReporter), and a TextureView canvas presenting on the +// main thread posts its present behind the window's pending draw. Holding the +// view weakly means a view that RN has dropped is not kept alive by a +// SurfaceInfo that outlives it (the registry keeps the SurfaceInfo until +// onDropViewInstance). struct WeakViewCall { jweak view = nullptr; jmethodID method = nullptr; @@ -188,22 +190,25 @@ struct WeakViewCall { } } - // Called on a native thread, outside the SurfaceInfo locks. - void operator()() const { + // Returns whether the view was there to take the call. Called on a native + // thread, outside the SurfaceInfo locks. + bool operator()() const { facebook::jni::ThreadScope scope; JNIEnv *env = facebook::jni::Environment::current(); if (env == nullptr || method == nullptr) { - return; + return false; } jobject local = env->NewLocalRef(view); if (local == nullptr) { - return; // the view was garbage collected + return false; // the view was garbage collected } env->CallVoidMethod(local, method); - if (env->ExceptionCheck()) { + const bool called = !env->ExceptionCheck(); + if (!called) { env->ExceptionClear(); } env->DeleteLocalRef(local); + return called; } }; @@ -284,7 +289,8 @@ void setAnimationReporter(int contextId, extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_onSurfaceCreate( JNIEnv *env, jobject thiz, jobject jSurface, jint contextId, jfloat width, - jfloat height, jboolean reportAnimation) { + jfloat height, jboolean presentsBehindPendingDraw, + jboolean acquiresAfterDraw, jboolean reportAnimation) { if (manager == nullptr) { return; } @@ -298,13 +304,21 @@ extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_onSurfaceCreate( auto &runtime = rnwgpu::WebGPURuntime::shared(); auto gpu = runtime.instance(); auto surface = runtime.makeSurface(window); + rnwgpu::NativeSurfacePresentPoster presentPoster; + if (presentsBehindPendingDraw) { + auto postPresent = std::make_shared( + env, thiz, "postPresentBehindPendingDraw"); + presentPoster = [postPresent]() { return (*postPresent)(); }; + } // Find-or-create + attach runs atomically under the registry lock so a // concurrent destroyContext cannot orphan this surface. auto info = registry.attachSurface( contextId, gpu, static_cast(width), static_cast(height), window, - surface, [](void *nativeSurface) { + surface, + [](void *nativeSurface) { ANativeWindow_release(static_cast(nativeSurface)); - }); + }, + std::move(presentPoster), acquiresAfterDraw); // A SurfaceView asks to hear when it animates, for its refresh rate vote // (WebGPUView.onAnimating). if (reportAnimation) { @@ -322,6 +336,32 @@ extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_onSurfaceCreate( manager->flushPendingSurfaceTransition(info); } +// The present WebGPUView.postPresentBehindPendingDraw posted (UI thread). +extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_presentPostedFrame( + JNIEnv *env, jobject thiz, jint contextId) { + auto ®istry = rnwgpu::SurfaceRegistry::getInstance(); + if (auto info = registry.getSurfaceInfo(contextId)) { + info->presentPostedFrame(); + } +} + +// WebGPUView.surfaceUpdated, while the window draws (UI thread). +extern "C" JNIEXPORT jlong JNICALL Java_com_webgpu_WebGPUView_onSurfaceDrawing( + JNIEnv *env, jobject thiz, jint contextId) { + auto ®istry = rnwgpu::SurfaceRegistry::getInstance(); + auto info = registry.getSurfaceInfo(contextId); + return info ? static_cast(info->presentCount()) : 0; +} + +// Posted by WebGPUView.surfaceUpdated behind the window's draw (UI thread). +extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_onSurfaceDrawn( + JNIEnv *env, jobject thiz, jint contextId, jlong presentCount) { + auto ®istry = rnwgpu::SurfaceRegistry::getInstance(); + if (auto info = registry.getSurfaceInfo(contextId)) { + info->surfaceDrawn(static_cast(presentCount)); + } +} + extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_switchToOffscreenSurface(JNIEnv *env, jobject thiz, jint contextId) { diff --git a/packages/webgpu/android/src/main/java/com/webgpu/WebGPUAPI.java b/packages/webgpu/android/src/main/java/com/webgpu/WebGPUAPI.java index 487128285..b9c673458 100644 --- a/packages/webgpu/android/src/main/java/com/webgpu/WebGPUAPI.java +++ b/packages/webgpu/android/src/main/java/com/webgpu/WebGPUAPI.java @@ -23,5 +23,8 @@ void surfaceChanged( void surfaceOffscreen(); + /** A TextureView's window is drawing the newest frame of its surface. */ + void surfaceUpdated(); + int getContextId(); } diff --git a/packages/webgpu/android/src/main/java/com/webgpu/WebGPUTextureView.java b/packages/webgpu/android/src/main/java/com/webgpu/WebGPUTextureView.java index e84f73806..91f121d5c 100644 --- a/packages/webgpu/android/src/main/java/com/webgpu/WebGPUTextureView.java +++ b/packages/webgpu/android/src/main/java/com/webgpu/WebGPUTextureView.java @@ -45,6 +45,6 @@ public boolean onSurfaceTextureDestroyed(@NonNull SurfaceTexture surfaceTexture) @Override public void onSurfaceTextureUpdated(@NonNull SurfaceTexture surfaceTexture) { - // No implementation needed + mApi.surfaceUpdated(); } } diff --git a/packages/webgpu/android/src/main/java/com/webgpu/WebGPUView.java b/packages/webgpu/android/src/main/java/com/webgpu/WebGPUView.java index 04663c300..1fa62fb3a 100644 --- a/packages/webgpu/android/src/main/java/com/webgpu/WebGPUView.java +++ b/packages/webgpu/android/src/main/java/com/webgpu/WebGPUView.java @@ -2,6 +2,8 @@ import android.content.Context; import android.os.Build; +import android.os.Handler; +import android.os.Looper; import android.util.Log; import android.view.Surface; import android.view.View; @@ -36,6 +38,10 @@ public class WebGPUView extends ReactViewGroup implements WebGPUAPI { private View mView = null; @Nullable private BackingViewKind mKind = null; + // Native posts a frame presented on the main thread here, to present it + // behind the draw the window has pending (see SurfaceRegistry.h). + private final Handler mMainHandler = new Handler(Looper.getMainLooper()); + private final Runnable mPostedPresent = () -> presentPostedFrame(mContextId); // Only a SurfaceView votes: a TextureView or HardwareBufferView is // composited into the window, which votes for it. private final SurfaceFrameRateVote mFrameRateVote = new SurfaceFrameRateVote( @@ -194,7 +200,15 @@ public void surfaceCreated(Surface surface) { if (votesFrameRate) { mFrameRateVote.attach(surface); } - onSurfaceCreate(surface, mContextId, width, height, votesFrameRate); + onSurfaceCreate(surface, mContextId, width, height, presentsBehindPendingDraw(), + acquiresAfterDraw(), votesFrameRate); + } + + // A TextureView shows a new frame only when its window draws. On Android 11 + // the frame is acquired only after that draw (acquiresAfterDraw), which keeps + // the held present clear of the loader's 4.3 s dequeue timeout. + private boolean presentsBehindPendingDraw() { + return mKind == BackingViewKind.TEXTURE_VIEW; } @Override @@ -221,11 +235,37 @@ void onAnimating() { mFrameRateVote.onAnimating(); } + @Override + public void surfaceUpdated() { + if (!acquiresAfterDraw()) { + return; + } + long presentCount = onSurfaceDrawing(mContextId); + // The window takes the frame when this draw syncs with its render thread, + // and a message posted now runs after that sync. + mMainHandler.post(() -> onSurfaceDrawn(mContextId, presentCount)); + } + + // Android 11's Vulkan loader hands a TextureView's buffer queue its dequeue + // timeout as an int where the window reads an int64_t (frameworks/native + // android-11.0.0_r48, vulkan/libvulkan/swapchain.cpp), so the queue waits + // up to 4.3 s for a free buffer: a frame is acquired only once the window + // has drawn the one presented last. + private boolean acquiresAfterDraw() { + return mKind == BackingViewKind.TEXTURE_VIEW && Build.VERSION.SDK_INT == Build.VERSION_CODES.R; + } + @Override public int getContextId() { return mContextId; } + /** Called by native on the main thread, see mPostedPresent. */ + @DoNotStrip + private void postPresentBehindPendingDraw() { + mMainHandler.post(mPostedPresent); + } + /** * Called from WebGPUViewManager.onDropViewInstance when React removes this * view: the view dies with its Canvas, so it retires the registry entry. @@ -235,6 +275,9 @@ public void destroy() { onViewDestroyed(mContextId); } + // presentsBehindPendingDraw has native post a present made on the main + // thread through postPresentBehindPendingDraw(); acquiresAfterDraw has it + // acquire a frame only once the window drew the last one (surfaceUpdated); // reportAnimation asks native to call onAnimating() when frames start being // presented to this surface back to back. @DoNotStrip @@ -243,6 +286,8 @@ private native void onSurfaceCreate( int contextId, float width, float height, + boolean presentsBehindPendingDraw, + boolean acquiresAfterDraw, boolean reportAnimation ); @@ -254,6 +299,15 @@ private native void onSurfaceChanged( float height ); + @DoNotStrip + private native void presentPostedFrame(int contextId); + + @DoNotStrip + private native long onSurfaceDrawing(int contextId); + + @DoNotStrip + private native void onSurfaceDrawn(int contextId, long presentCount); + @DoNotStrip private native void switchToOffscreenSurface(int contextId); diff --git a/packages/webgpu/cpp/rnwgpu/SurfaceRegistry.h b/packages/webgpu/cpp/rnwgpu/SurfaceRegistry.h index e734e65f6..89fd2772b 100644 --- a/packages/webgpu/cpp/rnwgpu/SurfaceRegistry.h +++ b/packages/webgpu/cpp/rnwgpu/SurfaceRegistry.h @@ -58,6 +58,13 @@ struct Size { // CAMetalLayer on Apple platforms). May run on any thread. using NativeSurfaceReleaser = std::function; +// Installed with a surface whose window shows a new frame only when the +// window draws (Android's TextureView): has the main looper call +// SurfaceInfo::presentPostedFrame() behind the draw the window has pending. +// Returns false when it could not post. Called on the main thread, outside +// SurfaceInfo's lock. +using NativeSurfacePresentPoster = std::function; + #if defined(__ANDROID__) // --- AHB-pool presentation mode (WebGPUNativeBufferView) // ----------------------------- @@ -145,6 +152,12 @@ struct AHBPool { // content appears without waiting for the next render — the same mechanism // that gives a fast time-to-first-frame when rendering starts before the // native surface exists. +// - A surface whose buffer queue makes an acquire wait until its window has +// drawn the frame presented last (an Android 11 TextureView, see +// WebGPUView.acquiresAfterDraw) defers a frame that comes sooner: it +// renders offscreen and is copied onto the surface once the window has +// drawn, by the frame boundary or the posted surfaceDrawn that sees it +// first. class SurfaceInfo { public: SurfaceInfo(wgpu::Instance gpu, int width, int height) @@ -184,20 +197,26 @@ class SurfaceInfo { // RNWebGPUManager::flushPendingSurfaceTransition so contexts that are not // currently rendering also pick it up. void attachSurface(void *nativeSurface, wgpu::Surface surface, - NativeSurfaceReleaser releaser) { + NativeSurfaceReleaser releaser, + NativeSurfacePresentPoster presentPoster = nullptr, + bool acquiresAfterDraw = false) { void *replacedSurface = nullptr; NativeSurfaceReleaser replacedReleaser; + NativeSurfacePresentPoster replacedPresentPoster; { std::unique_lock lock(_mutex); if (_hasPendingAttach) { // Replaced before it was ever adopted. replacedSurface = _pendingNativeSurface; replacedReleaser = std::move(_pendingReleaser); + replacedPresentPoster = std::move(_pendingPresentPoster); } _hasPendingAttach = true; _pendingNativeSurface = nativeSurface; _pendingSurface = std::move(surface); _pendingReleaser = std::move(releaser); + _pendingPresentPoster = std::move(presentPoster); + _pendingAcquiresAfterDraw = acquiresAfterDraw; } if (replacedReleaser && replacedSurface) { replacedReleaser(replacedSurface); @@ -214,6 +233,22 @@ class SurfaceInfo { // being destroyed and nothing will consume further frames. void detachSurface() { detach(/* createFallbackTexture = */ false); } + // The frames presented so far, read while the window draws: that draw + // syncs with every one of them queued. + uint64_t presentCount() { + std::shared_lock lock(_mutex); + return _presentCount; + } + + // The window's draw that read presentCount() has synced, taking the + // frames presented by then: a deferred frame waiting for it is copied + // onto the surface. + void surfaceDrawn(uint64_t presentCount) { + std::unique_lock lock(_mutex); + _drawnPresentCount = std::max(_drawnPresentCount, presentCount); + deliverDeferredFrameLocked(); + } + // Reflects native view layout changes. Does not resize the drawing buffer: // that tracks canvas.width/height (like on the web), see // GPUCanvasContext::getCurrentTexture. @@ -241,6 +276,7 @@ class SurfaceInfo { wgpu::Device device = nullptr; void *replacedSurface = nullptr; NativeSurfaceReleaser replacedReleaser; + NativeSurfacePresentPoster replacedPresentPoster; { std::unique_lock lock(_mutex); if (supersedeInFlightFrame) { @@ -255,6 +291,11 @@ class SurfaceInfo { // can be null if surface creation failed). replacedSurface = _nativeSurface; replacedReleaser = std::move(_releaser); + replacedPresentPoster = std::move(_presentPoster); + // A frame waiting for its posted present, or for the window's draw, + // belongs to the replaced surface; the blit below shows the latter. + _presentPending = false; + _deferredFrame = DeferredFrame::None; _surface = std::move(_pendingSurface); #if defined(__ANDROID__) // DAWN_WORKAROUND_DEVICE_DESTROY_BEFORE_SURFACE_RELEASE @@ -262,6 +303,8 @@ class SurfaceInfo { #endif _nativeSurface = _pendingNativeSurface; _releaser = std::move(_pendingReleaser); + _presentPoster = std::move(_pendingPresentPoster); + _acquiresAfterDraw = _pendingAcquiresAfterDraw; _hasPendingAttach = false; _pendingNativeSurface = nullptr; _frameEpoch++; @@ -307,7 +350,7 @@ class SurfaceInfo { // (their newer content stands; presenting here would be a Dawn // present-without-acquire error). if (_surface && !_frameInFlight && _frameEpoch == blitEpoch) { - _surface.Present(); + presentSurfaceLocked(); } } } @@ -319,6 +362,7 @@ class SurfaceInfo { std::vector viewFormats) { applyPendingAttach(/* supersedeInFlightFrame = */ true); std::unique_lock lock(_mutex); + presentPendingFrameLocked(); _viewFormats = std::move(viewFormats); _config = newConfig; // The caller's viewFormats storage dies with the call; point the stored @@ -330,7 +374,7 @@ class SurfaceInfo { _config.width = std::max(1, _width); _config.height = std::max(1, _height); _config.presentMode = wgpu::PresentMode::Fifo; - _texture = nullptr; + dropOffscreenTextureLocked(); _frameEpoch++; #if defined(__ANDROID__) { @@ -355,19 +399,21 @@ class SurfaceInfo { if (_config.device == nullptr) { return; } + presentPendingFrameLocked(); _config.width = std::max(1, newWidth); _config.height = std::max(1, newHeight); - _texture = nullptr; + dropOffscreenTextureLocked(); _frameEpoch++; _configureLocked(); } void unconfigure() { std::unique_lock lock(_mutex); + presentPendingFrameLocked(); if (_surface) { _surface.Unconfigure(); } - _texture = nullptr; + dropOffscreenTextureLocked(); _config = {}; _viewFormats.clear(); _acquiredFromSurface = false; @@ -490,13 +536,20 @@ class SurfaceInfo { "[WebGPU] getCurrentTexture() called on a canvas context that is " "not configured; call context.configure() first"); } + presentPendingFrameLocked(); _frameInFlight = true; _acquiredFromSurface = false; + // This frame supersedes one waiting for the window's draw. + _deferredFrame = DeferredFrame::None; _frameEpoch++; if (frameEpoch) { *frameEpoch = _frameEpoch; } - if (_surface) { + if (_surface && awaitsWindowDrawLocked()) { + // The acquire would wait for the window's draw, which runs on the main + // thread, maybe this one. + _deferredFrame = DeferredFrame::Rendering; + } else if (_surface) { auto texture = acquireSurfaceTextureLocked(); if (texture) { _acquiredFromSurface = true; @@ -525,10 +578,13 @@ class SurfaceInfo { // Present the current frame. Runs synchronously on the thread that did // getCurrentTexture/submit (main JS, Reanimated UI, or a worklet runtime), - // preserving Dawn surface thread-affinity. Frames whose texture was not - // acquired from the attached surface (offscreen, detached mid-frame, or - // acquire failure) are dropped. This is also the end-of-frame boundary: it - // adopts a surface that attached while the frame was in flight. + // preserving Dawn surface thread-affinity, except on a surface with a + // present poster (see presentsBehindPendingDrawLocked). Frames whose + // texture was not acquired from the attached surface (offscreen, detached + // mid-frame, or acquire failure) are dropped, except a deferred frame, + // which reaches the surface once the window has drawn. This is also the + // end-of-frame boundary: it adopts a surface that attached while the frame + // was in flight. void presentFrame() { #if defined(__ANDROID__) if (_poolMode.load()) { @@ -549,21 +605,36 @@ class SurfaceInfo { dawn::native::metal::WaitForCommandsToBeScheduled(device.Get()); } #endif + NativeSurfacePresentPoster presentPoster; #if defined(__ANDROID__) std::function framePresented; #endif { std::unique_lock lock(_mutex); if (_surface && _acquiredFromSurface) { - _surface.Present(); + if (presentsBehindPendingDrawLocked()) { + _presentPending = true; + if (!_presentPosted) { + _presentPosted = true; + presentPoster = _presentPoster; + } + } else { + presentSurfaceLocked(); #if defined(__ANDROID__) - framePresented = _framePresented; + framePresented = _framePresented; #endif + } + } else if (_deferredFrame == DeferredFrame::Rendering) { + _deferredFrame = DeferredFrame::Waiting; + deliverDeferredFrameLocked(); } _acquiredFromSurface = false; _frameInFlight = false; _frameEpoch++; } + if (presentPoster && !presentPoster()) { + presentPostedFrame(); + } #if defined(__ANDROID__) if (framePresented) { framePresented(); @@ -572,6 +643,15 @@ class SurfaceInfo { applyPendingAttach(); } + // Main thread, from the call a present poster posted: presents the frame + // presentFrame() left waiting, unless a frame boundary since has presented + // or dropped it. + void presentPostedFrame() { + std::unique_lock lock(_mutex); + _presentPosted = false; + presentPendingFrameLocked(); + } + NativeInfo getNativeInfo() { std::shared_lock lock(_mutex); // A surface that is still pending adoption is the one callers should see. @@ -718,6 +798,8 @@ class SurfaceInfo { void detach(bool createFallbackTexture) { void *releasedSurfaces[2] = {nullptr, nullptr}; NativeSurfaceReleaser releasers[2]; + // Destroyed outside the lock: they release JNI references. + NativeSurfacePresentPoster releasedPresentPosters[2]; #if defined(__ANDROID__) // Destroyed outside the lock: it releases a JNI reference. std::function releasedFrameReady; @@ -761,6 +843,7 @@ class SurfaceInfo { _pendingSurface = nullptr; releasedSurfaces[0] = _pendingNativeSurface; releasers[0] = std::move(_pendingReleaser); + releasedPresentPosters[0] = std::move(_pendingPresentPoster); _pendingNativeSurface = nullptr; } if (_surface) { @@ -773,14 +856,19 @@ class SurfaceInfo { _surfaceDevice = nullptr; #endif // The in-flight frame (if any) rendered into the destroyed surface; - // presentFrame() must not present it. + // presentFrame() must not present it, nor presentPostedFrame() one + // waiting for its posted present. _acquiredFromSurface = false; + _presentPending = false; + _acquiresAfterDraw = false; + _deferredFrame = DeferredFrame::None; } _frameEpoch++; // Window ownership is independent of the Dawn surface handle (which can // be null if surface creation failed): always return the window. releasedSurfaces[1] = _nativeSurface; releasers[1] = std::move(_releaser); + releasedPresentPosters[1] = std::move(_presentPoster); _nativeSurface = nullptr; } // Release outside the lock: the platform may do real work here. @@ -793,6 +881,59 @@ class SurfaceInfo { // All *Locked helpers below require _mutex to be held exclusively. + // A frame presented on the main thread can be inside a Choreographer + // frame, ahead of the window's draw, and a TextureView's buffer queue + // replaces a frame its window has not drawn yet: every other frame would + // never show. Such a frame is presented behind the draw the window has + // pending instead. Frames presented on another thread land between the + // main thread's frames and are presented straight away. + bool presentsBehindPendingDrawLocked() const { +#if defined(__ANDROID__) + return _presentPoster != nullptr && gettid() == getpid(); +#else + return false; +#endif + } + + // A frame waiting for its posted present is presented before the next + // frame boundary changes the surface, so no frame is acquired over it. + void presentPendingFrameLocked() { + if (_presentPending) { + _presentPending = false; + presentSurfaceLocked(); + } + } + + // Every present goes through here, counted for awaitsWindowDrawLocked. + void presentSurfaceLocked() { + _surface.Present(); + _presentCount++; + } + + // Whether the attached surface makes an acquire wait until its window has + // drawn the frame presented last, and the window has yet to draw it. + bool awaitsWindowDrawLocked() const { + return _acquiresAfterDraw && _drawnPresentCount != _presentCount; + } + + // Copies a deferred frame waiting in the offscreen texture onto the surface + // once the window has drawn the frame presented last. + void deliverDeferredFrameLocked() { + if (_deferredFrame != DeferredFrame::Waiting || awaitsWindowDrawLocked()) { + return; + } + _deferredFrame = DeferredFrame::None; + if (blitOffscreenToSurfaceLocked()) { + presentSurfaceLocked(); + } + } + + // A deferred frame waiting in the offscreen texture goes with it. + void dropOffscreenTextureLocked() { + _texture = nullptr; + _deferredFrame = DeferredFrame::None; + } + wgpu::Texture createOffscreenTextureLocked() { wgpu::TextureDescriptor descriptor; // Union with the user's usage so offscreen frames stay compatible with @@ -842,8 +983,9 @@ class SurfaceInfo { surfaceTexture.texture != nullptr; } - // Copy the last offscreen frame onto the freshly attached surface. Returns - // true when the copy was submitted and the surface should be presented. + // Copy the last offscreen frame onto the surface, freshly attached or + // deferred to. Returns true when the copy was submitted and the surface + // should be presented. bool blitOffscreenToSurfaceLocked() { wgpu::SurfaceTexture surfaceTexture; _surface.GetCurrentTexture(&surfaceTexture); @@ -882,9 +1024,7 @@ class SurfaceInfo { } } - // Every Surface::Configure goes through here. Once the - // DAWN_WORKAROUND_DEVICE_DESTROY_BEFORE_SURFACE_RELEASE lines are gone, this - // is a plain wrapper around _surface.Configure. + // Every Surface::Configure goes through here. void configureSurfaceLocked(const wgpu::SurfaceConfiguration &config) { #if defined(__ANDROID__) // DAWN_WORKAROUND_DEVICE_DESTROY_BEFORE_SURFACE_RELEASE @@ -899,7 +1039,14 @@ class SurfaceInfo { } } #endif - _surface.Configure(&config); + wgpu::SurfaceConfiguration surfaceConfig = config; + if (_acquiresAfterDraw) { + // A deferred frame reaches the surface by a copy. + surfaceConfig.usage |= wgpu::TextureUsage::CopyDst; + } + _surface.Configure(&surfaceConfig); + // A new swapchain holds no frame for the window to draw. + _drawnPresentCount = _presentCount; #if defined(__ANDROID__) // DAWN_WORKAROUND_DEVICE_DESTROY_BEFORE_SURFACE_RELEASE _surfaceDevice = config.device; @@ -914,8 +1061,11 @@ class SurfaceInfo { void recreateSurfaceLocked() { _surface = nullptr; _surfaceDevice = nullptr; - // A frame acquired from the old surface can no longer be presented. + // A frame acquired from, or deferred for, the old surface can no longer + // be presented. _acquiredFromSurface = false; + _presentPending = false; + _deferredFrame = DeferredFrame::None; _frameEpoch++; if (_nativeSurface) { wgpu::SurfaceSourceAndroidNativeWindow source; @@ -1229,9 +1379,27 @@ class SurfaceInfo { void *_pendingNativeSurface = nullptr; wgpu::Surface _pendingSurface = nullptr; NativeSurfaceReleaser _pendingReleaser; + NativeSurfacePresentPoster _pendingPresentPoster; // Frame state: set by getCurrentTexture, cleared by presentFrame. bool _frameInFlight = false; bool _acquiredFromSurface = false; + // The attached surface's present poster (null when it presents straight + // away), whether its last frame waits for the posted present, and whether + // a posted present is on its way. + NativeSurfacePresentPoster _presentPoster; + bool _presentPending = false; + bool _presentPosted = false; + // Whether the attached surface makes an acquire wait until its window has + // drawn the frame presented last, and the pending surface's flag. + bool _acquiresAfterDraw = false; + bool _pendingAcquiresAfterDraw = false; + // Frames presented, and the ones the attached surface's window has drawn. + uint64_t _presentCount = 0; + uint64_t _drawnPresentCount = 0; + // A frame rendering offscreen because the window had yet to draw the one + // presented last, or finished and waiting in the offscreen texture. + enum class DeferredFrame : uint8_t { None, Rendering, Waiting }; + DeferredFrame _deferredFrame = DeferredFrame::None; // Bumped on every acquire, present, configure/reconfigure/unconfigure, // adoption, and detach. The deferred blit-present in applyPendingAttach // revalidates against it so it never presents a texture that stopped being @@ -1322,14 +1490,16 @@ class SurfaceRegistry { // an entry that a concurrent destroyContext is erasing (it either marks the // entry attached before the check, or re-creates the entry after the // erase). Lock order is registry -> SurfaceInfo, matching every other path. - std::shared_ptr attachSurface(int id, wgpu::Instance gpu, - int width, int height, - void *nativeSurface, - wgpu::Surface surface, - NativeSurfaceReleaser releaser) { + std::shared_ptr + attachSurface(int id, wgpu::Instance gpu, int width, int height, + void *nativeSurface, wgpu::Surface surface, + NativeSurfaceReleaser releaser, + NativeSurfacePresentPoster presentPoster = nullptr, + bool acquiresAfterDraw = false) { std::unique_lock lock(_mutex); auto info = getSurfaceInfoOrCreateLocked(id, gpu, width, height); - info->attachSurface(nativeSurface, std::move(surface), std::move(releaser)); + info->attachSurface(nativeSurface, std::move(surface), std::move(releaser), + std::move(presentPoster), acquiresAfterDraw); return info; } diff --git a/packages/webgpu/src/__tests__/TextureViewPresent.spec.ts b/packages/webgpu/src/__tests__/TextureViewPresent.spec.ts new file mode 100644 index 000000000..cac5b98dd --- /dev/null +++ b/packages/webgpu/src/__tests__/TextureViewPresent.spec.ts @@ -0,0 +1,95 @@ +import { client } from "./setup"; + +// The harness mounts a 32x32pt canvas backed by a TextureView on Android +// (apps/example/src/Tests.tsx). A frame presented on the UI thread reaches it +// behind the window's pending draw, and on Android 11 a frame presented before +// the window drew the previous one is copied in once it has; this checks that +// the window shows the last frame presented from either thread, also after a +// frame acquired while the previous one still waited for its present, and that +// presenting never stalls the thread. + +type RGBA = [number, number, number, number]; + +const LIME: RGBA = [0, 255, 0, 255]; + +// An acquire that waits for the window's draw on the thread that draws it +// stalls that thread for Android 11's 4.29 s dequeue timeout. +const FRAME_GAP_BUDGET_MS = 2000; + +const expectColor = (actual: number[], expected: RGBA, tolerance = 3) => { + expected.forEach((channel, i) => { + expect(Math.abs(actual[i] - channel)).toBeLessThanOrEqual(tolerance); + }); +}; + +describe("TextureView canvas", () => { + it.each(["UI", "JS"] as const)( + "shows the last frame presented on the %s thread without stalling it", + async (presentingThread) => { + if (client.OS !== "android") { + return; + } + const result = await client.eval( + ({ device, views, presentFrames, thread }) => { + const context = views?.textureCanvas()?.getContext("webgpu"); + const source = views?.textureCanvasView; + if (!views || !context || !source || !presentFrames) { + throw new Error("The Tests screen has no TextureView canvas"); + } + context.configure({ + device, + format: navigator.gpu.getPreferredCanvasFormat(), + alphaMode: "opaque", + }); + const size = Math.round(views.textureCanvasSize * views.pixelRatio); + const texture = device.createTexture({ + size: [size, size], + format: "rgba8unorm", + usage: GPUTextureUsage.COPY_DST | GPUTextureUsage.COPY_SRC, + }); + const bytesPerRow = Math.ceil((size * 4) / 256) * 256; + const output = device.createBuffer({ + size: bytesPerRow * size, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const colors: GPUColor[] = [ + [1, 0, 0, 1], + [1, 1, 0, 1], + [0, 1, 1, 1], + [1, 0, 1, 1], + [0, 1, 0, 1], + ]; + let longestFrameGapMs = 0; + return presentFrames(device, context, colors, thread) + .then((frameGapMs) => { + longestFrameGapMs = frameGapMs; + return device.queue.drawElementImageToTexture( + { source }, + { texture }, + ); + }) + .then(() => { + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer( + { texture }, + { buffer: output, bytesPerRow }, + [size, size], + ); + device.queue.submit([encoder.finish()]); + return output.mapAsync(GPUMapMode.READ); + }) + .then(() => { + const data = new Uint8Array(output.getMappedRange()); + const centre = (size / 2) * bytesPerRow + (size / 2) * 4; + const pixel = Array.from(data.slice(centre, centre + 4)); + output.unmap(); + return { pixel, longestFrameGapMs }; + }); + }, + { thread: presentingThread }, + ); + expectColor(result.pixel, LIME); + expect(result.longestFrameGapMs).toBeLessThan(FRAME_GAP_BUDGET_MS); + }, + ); +}); diff --git a/packages/webgpu/src/__tests__/setup.ts b/packages/webgpu/src/__tests__/setup.ts index c639e139f..c7746c89b 100644 --- a/packages/webgpu/src/__tests__/setup.ts +++ b/packages/webgpu/src/__tests__/setup.ts @@ -11,6 +11,7 @@ import { mat4, vec3, mat3 } from "wgpu-matrix"; import type { Server, WebSocket } from "ws"; import type { Browser, Page } from "puppeteer"; +import type { CanvasRef, RNCanvasContext } from "../Canvas"; import type { GPUOffscreenCanvas } from "../Offscreen"; import { cubeVertexArray } from "./components/cube"; @@ -86,13 +87,27 @@ interface GPUTestingContext { GPUShaderStage: typeof GPUShaderStage; GPUTextureUsage: typeof GPUTextureUsage; }; + // Native app harness only: clears a canvas to each color in turn from + // Reanimated's UI thread or the JS thread and resolves with the longest + // time between two frames, see apps/example/src/components/presentFrames.ts. + presentFrames?: ( + device: GPUDevice, + context: RNCanvasContext, + colors: GPUColor[], + thread: "UI" | "JS", + ) => Promise; // Native app harness only: a mounted probe view for the // drawElementImageToTexture spec (its React tag, its size in points and the - // device pixel ratio). See apps/example/src/Tests.tsx. + // device pixel ratio), and for the TextureView present spec a canvas + // backed by a TextureView on Android, the React tag of the view around it + // and its size in points. See apps/example/src/Tests.tsx. views?: { probe: number | null; probeSize: number; pixelRatio: number; + textureCanvas: () => CanvasRef | null; + textureCanvasView: number | null; + textureCanvasSize: number; }; mat4: typeof mat4; vec3: typeof vec3;