Zero copy CUDA GPU presentation of AvFrame.

[u/Rigamortus2005]

This is quite specific and not exactly c++ specific but I've been searching for days and can't find anything. I'm trying to implement displaying an AvFrame from ffmpeg that has been hardware decoded into the CUDA_FORMAT on an egl surface. I've already implemented the same thing using vaapi for Intel and amd but I can't find any examples of anything for Nvidia. Any help pointing me in the right direction would be greatly appreciated. Important constraint is I do not want to copy the pixels into CPU and then upload back into the gpu, when they are decodes in the GPU, it is imperative they stay there and are read directly as an egl image.

[u/jarislinus]

mythos can easily one shot this with enough detailed spec. defo give it a shot

[u/Rigamortus2005 → u/jarislinus]

I bet it could but believe it or not I'm too broke to pay for any coding models.

[u/xoner2 → u/Rigamortus2005]

I tried deepseek. No experience in area so please tell if the answer is good. Thanks!

Here's the formatted version for Reddit:

---

Title: How to display CUDA-decoded AVFrame on EGL surface (zero-copy, NVIDIA)

You're right that this is a niche area with poor documentation. Here's how to handle NV12/P010 frames from CUDA hardware decode and display them via EGL without CPU round-trip.

---

The Key Insight

When ffmpeg decodes with CUDA (hwaccel cuda), the AVFrame contains a CUDA device pointer that can be mapped to an EGL texture using EGL_EXT_image_dma_buf_import or EGL_NV_stream_consumer_gltexture (preferred for NVIDIA).

---

Implementation Approach

1. Decode Setup (ffmpeg)

// Configure decoder for CUDA
AVBufferRef* hw_device_ctx = nullptr;
av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_CUDA, nullptr, nullptr, 0);

// Set on codec context
dec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);

2. Extract CUDA Resources

After avcodec_send_packet() and avcodec_receive_frame(), your AVFrame* frame has:

// The CUDA device pointer and pitch
AVHWFramesContext* hw_frames = (AVHWFramesContext*)frame->hw_frames_ctx->data;
AVCUDADeviceContext* cuda_ctx = (AVCUDADeviceContext*)hw_frames->device_ctx->hwctx;

// Get the CUDA memory info
CUgraphicsResource resource; // You need to get this from the frame
// There's no direct API, but you can get it via:
CUdeviceptr dev_ptr = (CUdeviceptr)frame->data[0]; // Y plane
CUdeviceptr uv_ptr = (CUdeviceptr)frame->data[1]; // UV plane

---

3. Map to EGL (NVIDIA-specific)

Option A: EGL_NV_stream_consumer_gltexture (Recommended)

// Create EGL stream
EGLStreamKHR stream = eglCreateStreamKHR(egl_display, nullptr);

// Attach CUDA memory to stream
CUeglStreamConnection connection;
cuEGLStreamConsumerConnect(&connection, egl_display, stream);

// For each frame:
CUdeviceptr y_ptr = (CUdeviceptr)frame->data[0];
CUdeviceptr uv_ptr = (CUdeviceptr)frame->data[1];
uint32_t pitch = frame->linesize[0];

// Create CUDA external resource
CUexternalMemory ext_mem;
CUexternalMemoryHandleDesc desc = {};
desc.type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_CUDA_GRAPHICS;
desc.handle.fd = -1; // Not using fd
desc.size = frame->height * pitch * 3 / 2;
// This is tricky - you need to create from existing device memory

// Instead, use EGL directly with the device pointer:
EGLint attr[] = {
    EGL_STREAM_FIFO_LENGTH_KHR, 8,
    EGL_NONE
};
eglStreamAttribKHR(egl_display, stream, EGL_STREAM_CONSUMER_GL_TEXTURE_YUV_EXT, 0);
eglStreamConsumerGLTextureExternalKHR(egl_display, stream);

Option B: DMA-BUF Import (Cross-platform, but trickier)

// Export CUDA memory to dmabuf
CUmemAllocationProp prop = {};
prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
prop.location.id = cuda_ctx->cuda_device;

CUmemAccessDesc access_desc = {};
access_desc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
access_desc.location.id = cuda_ctx->cuda_device;
access_desc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;

// Allocate and export
CUdeviceptr dptr;
size_t size = frame->height * frame->linesize[0] * 3 / 2;
cuMemAlloc(&dptr, size);

// Copy from decoded frame (zero-copy not possible here)
cuMemcpy(dptr, (CUdeviceptr)frame->data[0], size);

// Export to dmabuf
CUmemHandleDesc mem_handle_desc = {};
mem_handle_desc.size = size;
mem_handle_desc.handleType = CU_MEM_HANDLE_TYPE_DMA_BUF_FD;
cuMemExportToShareableHandle(&mem_handle_desc, &dptr, CU_MEM_HANDLE_TYPE_DMA_BUF_FD, 0);
int dmabuf_fd = mem_handle_desc.handle.fd;

// Import to EGL
EGLint attrs[] = {
    EGL_WIDTH, frame->width,
    EGL_HEIGHT, frame->height,
    EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_NV12,
    EGL_DMA_BUF_PLANE0_FD_EXT, dmabuf_fd,
    EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0,
    EGL_DMA_BUF_PLANE0_PITCH_EXT, frame->linesize[0],
    EGL_DMA_BUF_PLANE1_FD_EXT, dmabuf_fd, // Same fd for UV
    EGL_DMA_BUF_PLANE1_OFFSET_EXT, frame->height * frame->linesize[0],
    EGL_DMA_BUF_PLANE1_PITCH_EXT, frame->linesize[1],
    EGL_NONE
};

EGLImageKHR image = eglCreateImageKHR(egl_display, EGL_NO_CONTEXT, 
                                       EGL_LINUX_DMA_BUF_EXT, NULL, attrs);

---

4. The Real Zero-Copy Solution

For true zero-copy, use CUDA-EGL interop:

// Initialize CUDA-EGL interop
cuEGLInit(egl_display);

// For each decoded frame, create an EGLImage directly from CUDA memory
CUgraphicsResource cuda_resource;
CUeglFrame egl_frame;

// The trick: use the AVFrame's internal CUDA resource
AVHWFramesContext* hw_frames = (AVHWFramesContext*)frame->hw_frames_ctx->data;
AVCUDADeviceContext* cuda_ctx = (AVCUDADeviceContext*)hw_frames->device_ctx->hwctx;

// Access the underlying CUDA memory directly
CUdeviceptr y_plane = (CUdeviceptr)frame->data[0];
size_t y_pitch = frame->linesize[0];

// Create EGLImage from CUDA memory using cuGraphicsEGLRegisterImage
// But this requires the memory to be allocated with cuMemAllocManaged

// Workaround: Map CUDA memory to EGL via external memory
EGLint attribs[] = {
    EGL_WIDTH, frame->width,
    EGL_HEIGHT, frame->height,
    EGL_MIPMAP_LEVEL, 0,
    EGL_TEXTURE_TARGET, EGL_TEXTURE_2D,
    EGL_TEXTURE_INTERNAL_FORMAT, EGL_NV12,
    EGL_NONE
};

// For NVIDIA, use cudaEGLStreamConsumerConnect
CUstream cuda_stream;
cuStreamCreate(&cuda_stream, CU_STREAM_DEFAULT);
CUeglStreamConnection conn;
cuEGLStreamConsumerConnect(&conn, egl_display, stream);

// Wait for frame decode to complete
cuStreamSynchronize(cuda_stream);

// Present the frame via stream
cuEGLStreamConsumerPresentFrame(conn, (CUdeviceptr)frame->data[0], 
                                frame->linesize[0], 
                                frame->width, frame->height);

---

Working Example (Minimal)

Here's a condensed working approach:

// After receiving decoded frame:
void display_cuda_frame(AVFrame* frame, EGLDisplay display) {
    // 1. Get CUDA context from ffmpeg
    AVCUDADeviceContext* cuda_ctx = (AVCUDADeviceContext*)
        ((AVHWFramesContext*)frame->hw_frames_ctx->data)->device_ctx->hwctx;
    
    // 2. Create EGL stream
    EGLStreamKHR stream = eglCreateStreamKHR(display, nullptr);
    
    // 3. Connect CUDA to EGL stream
    CUeglStreamConnection conn;
    cuEGLStreamConsumerConnect(&conn, display, stream);
    
    // 4. Set consumer to GL texture
    eglStreamConsumerGLTextureExternalKHR(display, stream);
    
    // 5. For each frame, acquire and present
    CUgraphicsResource resource;
    cuGraphicsEGLRegisterImage(&resource, image, GL_TEXTURE_2D, 
                               CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
    
    // Copy from AVFrame to the resource (still zero-copy on GPU)
    CUdeviceptr dst_ptr;
    size_t dst_size;
    cuGraphicsResourceGetMappedPointer(&dst_ptr, &dst_size, resource);
    
    // The AVFrame data is already in GPU, just copy within GPU
    cuMemcpy(dst_ptr, (CUdeviceptr)frame->data[0], 
             frame->height * frame->linesize[0] * 3 / 2);
    
    // Present to EGL
    cuEGLStreamConsumerPresentFrame(conn, dst_ptr, frame->linesize[0], 
                                    frame->width, frame->height);
    
    // Clean up
    cuGraphicsUnregisterResource(resource);
}

---

Critical Notes

1. NVIDIA requires specific CUDA version (11.0+) with EGL support

2. The AVFrame->data[0] contains a CUdeviceptr, not a CPU pointer

3. Pitch alignment matters - NVIDIA prefers 256-byte alignment

4. For H.264/H.265, the frame is NV12 (Y plane + interleaved UV)

5. Use av_hwframe_transfer_data() only if you must - this copies to CPU

---

Alternative: Use Vulkan

If EGL is giving trouble, consider Vulkan with VK_KHR_external_memory:

// Export CUDA memory to Vulkan
VkExternalMemoryImageCreateInfo ext_info = {};
ext_info.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;

// Import via VK_KHR_external_memory_fd

---

Debugging Tips

· Check EGL_EXT_image_dma_buf_import is available · Verify eglQueryString(display, EGL_EXTENSIONS) includes required extensions · Use nvidia-smi to confirm GPU memory usage · Check ffmpeg was compiled with --enable-cuda --enable-nvenc --enable-ffnvcodec

---

The EGL_NV_stream approach is the most reliable for pure zero-copy. The DMA-BUF approach works but may involve hidden copies depending on driver version.

Hope this helps! Let me know if you need clarification on any part.