From a793b272b89bc65244578f019d17bc49bd321777 Mon Sep 17 00:00:00 2001 From: mohammadmseet-hue Date: Sat, 4 Apr 2026 21:49:26 +0200 Subject: [PATCH] Fix integer overflow in Vulkan/Metal buffer size and pitch calculations The Terracotta initiative (commits 88ff9f4, a7d24e6) fixed integer overflow in D3D11 backend buffer allocation and Metal sizeInBytes calculation, but several equivalent overflow patterns in the Vulkan and Metal backends were missed. When processing large textures (e.g. RGBA32F at GL_MAX_TEXTURE_SIZE=16384), multiplication of pixelBytes * width * height exceeds UINT32_MAX, wrapping to a small value. This causes undersized buffer allocation followed by writes of the full image data, resulting in heap buffer overflow. Fixed locations: - vk_helpers.cpp readPixelsImpl: allocationSize computed in 32-bit - TextureVk.cpp reformatStagedUpdate: GLuint pitch variables overflow - SurfaceVk.cpp: GLuint rowStride overflow before VkDeviceSize widening - UtilsVk.cpp: GLuint sliceTexels/sliceSize/texBufferSize chain overflow - FrameBufferMtl.mm: uint32_t bytesPerRow not fixed by Terracotta All fixed by widening to size_t/VkDeviceSize before multiplication. --- src/libANGLE/renderer/metal/FrameBufferMtl.mm | 2 +- src/libANGLE/renderer/vulkan/SurfaceVk.cpp | 5 ++--- src/libANGLE/renderer/vulkan/TextureVk.cpp | 12 ++++++------ src/libANGLE/renderer/vulkan/UtilsVk.cpp | 6 +++--- src/libANGLE/renderer/vulkan/vk_helpers.cpp | 4 +++- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/libANGLE/renderer/metal/FrameBufferMtl.mm b/src/libANGLE/renderer/metal/FrameBufferMtl.mm index 9ebc41a209c..4c6d87f31a0 100644 --- a/src/libANGLE/renderer/metal/FrameBufferMtl.mm +++ b/src/libANGLE/renderer/metal/FrameBufferMtl.mm @@ -92,7 +92,7 @@ void OverrideMTLClearColor(const mtl::TextureRef &texture, sizeInBytes.ValueOrDie(), &tempBuffer)); gl::Rectangle region(0, 0, width, height); - uint32_t bytesPerRow = angleFormat.pixelBytes * width; + size_t bytesPerRow = static_cast(angleFormat.pixelBytes) * width; uint32_t destOffset = 0; ANGLE_TRY(mtl::ReadTexturePerSliceBytesToBuffer(context, srcTexture, bytesPerRow, region, mipNativeLevel, layerIndex, destOffset, diff --git a/src/libANGLE/renderer/vulkan/SurfaceVk.cpp b/src/libANGLE/renderer/vulkan/SurfaceVk.cpp index 24c7a3a32aa..1f30156ff7f 100644 --- a/src/libANGLE/renderer/vulkan/SurfaceVk.cpp +++ b/src/libANGLE/renderer/vulkan/SurfaceVk.cpp @@ -283,9 +283,8 @@ angle::Result LockSurfaceImpl(DisplayVk *displayVk, { const gl::InternalFormat &internalFormat = gl::GetSizedInternalFormatInfo(image->getActualFormat().glInternalFormat); - GLuint rowStride = image->getActualFormat().pixelBytes * width; - VkDeviceSize bufferSize = - (static_cast(rowStride) * static_cast(height)); + VkDeviceSize rowStride = static_cast(image->getActualFormat().pixelBytes) * width; + VkDeviceSize bufferSize = rowStride * static_cast(height); if (!lockBufferHelper.valid() || (lockBufferHelper.getSize() != bufferSize)) { diff --git a/src/libANGLE/renderer/vulkan/TextureVk.cpp b/src/libANGLE/renderer/vulkan/TextureVk.cpp index 3de8b877985..04ea9e6f5f2 100644 --- a/src/libANGLE/renderer/vulkan/TextureVk.cpp +++ b/src/libANGLE/renderer/vulkan/TextureVk.cpp @@ -3173,14 +3173,14 @@ angle::Result TextureVk::reinitImageAsRenderable(ContextVk *contextVk, const vk: &dstData, dstFormat.id)); // Source and destination data is tightly packed - GLuint srcDataRowPitch = sourceBox.width * srcFormat.pixelBytes; - GLuint dstDataRowPitch = sourceBox.width * dstFormat.pixelBytes; + size_t srcDataRowPitch = static_cast(sourceBox.width) * srcFormat.pixelBytes; + size_t dstDataRowPitch = static_cast(sourceBox.width) * dstFormat.pixelBytes; - GLuint srcDataDepthPitch = srcDataRowPitch * sourceBox.height; - GLuint dstDataDepthPitch = dstDataRowPitch * sourceBox.height; + size_t srcDataDepthPitch = srcDataRowPitch * sourceBox.height; + size_t dstDataDepthPitch = dstDataRowPitch * sourceBox.height; - GLuint srcDataLayerPitch = srcDataDepthPitch * sourceBox.depth; - GLuint dstDataLayerPitch = dstDataDepthPitch * sourceBox.depth; + size_t srcDataLayerPitch = srcDataDepthPitch * sourceBox.depth; + size_t dstDataLayerPitch = dstDataDepthPitch * sourceBox.depth; rx::PixelReadFunction pixelReadFunction = srcFormat.pixelReadFunction; rx::PixelWriteFunction pixelWriteFunction = dstFormat.pixelWriteFunction; diff --git a/src/libANGLE/renderer/vulkan/UtilsVk.cpp b/src/libANGLE/renderer/vulkan/UtilsVk.cpp index 7c369d35062..15fc26a8beb 100644 --- a/src/libANGLE/renderer/vulkan/UtilsVk.cpp +++ b/src/libANGLE/renderer/vulkan/UtilsVk.cpp @@ -4284,10 +4284,10 @@ angle::Result UtilsVk::transCodeEtcToBc(ContextVk *contextVk, ASSERT(dstImage->getType() != VK_IMAGE_TYPE_1D && dstImage->getType() != VK_IMAGE_TYPE_3D); - GLuint sliceTexels = (copyRegion->bufferRowLength / info.compressedBlockWidth) * + size_t sliceTexels = static_cast(copyRegion->bufferRowLength / info.compressedBlockWidth) * (copyRegion->bufferImageHeight / info.compressedBlockHeight); - GLuint sliceSize = sliceTexels * intendedFormat.pixelBytes; - GLuint texBufferSize = sliceSize * copyRegion->imageSubresource.layerCount; + size_t sliceSize = sliceTexels * intendedFormat.pixelBytes; + size_t texBufferSize = sliceSize * copyRegion->imageSubresource.layerCount; // Make sure the texture buffer size not out of limit. // Usually the limit is more than 128M. diff --git a/src/libANGLE/renderer/vulkan/vk_helpers.cpp b/src/libANGLE/renderer/vulkan/vk_helpers.cpp index 2d012de8d6f..973ca369558 100644 --- a/src/libANGLE/renderer/vulkan/vk_helpers.cpp +++ b/src/libANGLE/renderer/vulkan/vk_helpers.cpp @@ -11535,7 +11535,9 @@ angle::Result ImageHelper::readPixelsImpl(ContextVk *contextVk, uint8_t *readPixelBuffer = nullptr; VkDeviceSize stagingOffset = 0; - size_t allocationSize = readFormat->pixelBytes * area.width * area.height; + size_t allocationSize = static_cast(readFormat->pixelBytes) * + static_cast(area.width) * + static_cast(area.height); ANGLE_TRY(contextVk->initBufferForImageCopy(stagingBuffer, allocationSize, MemoryCoherency::CachedPreferCoherent,