fix(psd): fixes against corrupt files with better validation (#5089)

lgritz · web-flow · commit d3aa1658fc4a · 2026-03-17T16:39:17.000-07:00
1. Validate channel count against color mode to prevent heap buffer
   overflow.

2. Corrupted PSD files can declare a color mode (e.g. RGB, needing 3
   channels) with transparency (needing +1) but report fewer channels
   in the header.  This caused an out-of-bounds read in
   read_native_scanline when setup() built channel pointer arrays
   using mode_channel_count, which exceeded the actual
   m_image_data.channel_info size.

3. Additional buffer overflow protection, error checking and
propagation.

4. Avoid leak by properly freeing z_stream resources in decompress_zip.

Used Claude Code / Opus 4.6 to help narrow down and confirm the bugs 1
and 2.

Assisted-by: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;

Signed-off-by: Larry Gritz &lt;lg@larrygritz.com&gt;
diff --git a/src/psd.imageio/psdinput.cpp b/src/psd.imageio/psdinput.cpp
@@ -294,7 +294,7 @@ class PSDInput final : public ImageInput {
 
     void set_type_desc();
     //Setup m_specs and m_channels
-    void setup();
+    bool setup();
     void fill_channel_names(ImageSpec& spec, bool transparency);
 
     //Read a row of channel data
@@ -619,9 +619,11 @@ PSDInput::open(const std::string& name, ImageSpec& newspec)
     // Set m_type_desc to the appropriate TypeDesc
     set_type_desc();
     // Setup ImageSpecs and m_channels
-    setup();
+    bool ok = true;
+    ok &= setup();
 
-    bool ok = seek_subimage(0, 0);
+    if (ok)
+        ok &= seek_subimage(0, 0);
     if (ok)
         newspec = spec();
     else
@@ -1468,7 +1470,9 @@ PSDInput::load_layers()
     ok &= read_bige<int16_t>(layer_info.layer_count);
     if (layer_info.layer_count < 0) {
         m_image_data.transparency = true;
-        layer_info.layer_count    = -layer_info.layer_count;
+        if (layer_info.layer_count == -32768)
+            return false;  // will overflow when negated
+        layer_info.layer_count = -layer_info.layer_count;
     }
     m_layers.resize(layer_info.layer_count);
     for (int16_t layer_nbr = 0; layer_nbr < layer_info.layer_count;
@@ -1639,6 +1643,7 @@ PSDInput::load_layer_channel(Layer& layer, ChannelInfo& channel_info)
     channel_info.row_pos.resize(height);
     channel_info.row_length = (width * m_header.depth + 7) / 8;
 
+    bool ok = true;
     switch (channel_info.compression) {
     case Compression_Raw:
         if (height) {
@@ -1687,7 +1692,7 @@ PSDInput::load_layer_channel(Layer& layer, ChannelInfo& channel_info)
         if (!ioread(compressed_data.data(), channel_info.data_length))
             return false;
 
-        decompress_zip(compressed_data, channel_info.decompressed_data);
+        ok = decompress_zip(compressed_data, channel_info.decompressed_data);
     } break;
     case Compression_ZIP_Predict: {
         // We subtract the compression marker from the data length
@@ -1704,16 +1709,18 @@ PSDInput::load_layer_channel(Layer& layer, ChannelInfo& channel_info)
         if (!ioread(compressed_data.data(), channel_info.data_length))
             return false;
 
-        decompress_zip_prediction(compressed_data,
-                                  channel_info.decompressed_data, width,
-                                  height);
+        ok = decompress_zip_prediction(compressed_data,
+                                       channel_info.decompressed_data, width,
+                                       height);
     } break;
     default:
         errorfmt("[Layer Channel] unsupported compression {}",
                  channel_info.compression);
         return false;
     }
-    return true;
+    if (!ok)
+        errorfmt("Error during layer decompression. Possible corrupt file?");
+    return ok;
 }
 
 
@@ -1853,7 +1860,9 @@ PSDInput::load_layers_16_32(uint64_t length)
     ok &= read_bige<int16_t>(layer_info.layer_count);
     if (layer_info.layer_count < 0) {
         m_image_data.transparency = true;
-        layer_info.layer_count    = -layer_info.layer_count;
+        if (layer_info.layer_count == -32768)
+            return false;  // will overflow when negated
+        layer_info.layer_count = -layer_info.layer_count;
     }
     m_layers.resize(layer_info.layer_count);
     for (int16_t layer_nbr = 0; layer_nbr < layer_info.layer_count;
@@ -1894,6 +1903,22 @@ PSDInput::load_image_data()
                  compression);
         return false;
     }
+    // Validate that the file has enough channels for its color mode.
+    // mode_channel_count gives the minimum channels required; if the layer
+    // info section indicated transparency, we need one more channel.
+    {
+        int required = (m_header.color_mode <= ColorMode_Lab)
+                           ? (int)mode_channel_count[m_header.color_mode]
+                           : 0;
+        if (m_image_data.transparency)
+            required++;
+        if (m_header.channel_count < required) {
+            errorfmt(
+                "[Image Data Section] channel count {} is too few for color mode {}",
+                m_header.channel_count, m_header.color_mode);
+            return false;
+        }
+    }
     m_image_data.channel_info.resize(m_header.channel_count);
     // setup some generic properties and read any RLE lengths
     // Image Data Section has RLE lengths for all channels stored first
@@ -1938,7 +1963,7 @@ PSDInput::load_image_data()
 
 
 
-void
+bool
 PSDInput::setup()
 {
     // raw_channel_count is the number of channels in the file
@@ -2012,6 +2037,8 @@ PSDInput::setup()
         if (layer.name.size())
             spec.attribute("oiio:subimagename", layer.name);
     }
+
+    return true;
 }
 
 
@@ -2297,6 +2324,7 @@ bool
 PSDInput::decompress_zip(span<char> src, span<char> dest)
 {
     z_stream stream {};
+    stream.zalloc    = Z_NULL;
     stream.zfree     = Z_NULL;
     stream.opaque    = Z_NULL;
     stream.avail_in  = src.size();
@@ -2306,26 +2334,29 @@ PSDInput::decompress_zip(span<char> src, span<char> dest)
 
     if (inflateInit(&stream) != Z_OK) {
         errorfmt(
-            "zip compression inflate init failed with: src_size={}, dst_size={}",
-            src.size(), dest.size());
+            "zip compression inflate init failed with: src_size={}, dst_size={} {}",
+            src.size(), dest.size(), stream.msg ? stream.msg : "");
         return false;
     }
 
+    bool ok = true;
     if (inflate(&stream, Z_FINISH) != Z_STREAM_END) {
         errorfmt(
-            "unable to decode zip compressed data: src_size={}, dst_size={}",
-            src.size(), dest.size());
-        return false;
+            "unable to decode zip compressed data: src_size={}, dst_size={} {}",
+            src.size(), dest.size(), stream.msg ? stream.msg : "");
+        ok = false;
     }
 
+    // Note: call inflateEnd even if ok == false, because we need to clean up.
     if (inflateEnd(&stream) != Z_OK) {
-        errorfmt(
-            "zip compression inflate cleanup failed with: src_size={}, dst_size={}",
-            src.size(), dest.size());
-        return false;
+        if (ok)  // message only if this was the first error
+            errorfmt(
+                "zip compression inflate cleanup failed with: src_size={}, dst_size={} {}",
+                src.size(), dest.size(), stream.msg ? stream.msg : "");
+        ok = false;
     }
 
-    return true;
+    return ok;
 }
 
 
@@ -2342,6 +2373,8 @@ PSDInput::decompress_zip_prediction(span<char> src, span<char> dest,
 
     switch (m_header.depth) {
     case 8:
+        if ((height - 1) * width + (width - 1) >= dest.size())
+            return false;  // going to exceed the dest bounds
         for (uint64_t y = 0; y < height; ++y) {
             // Index x beginning at one since we look behind to calculate
             // the offset
@@ -2355,6 +2388,8 @@ PSDInput::decompress_zip_prediction(span<char> src, span<char> dest,
         // prediction decoding to work correctly
         span<uint16_t> destView(reinterpret_cast<uint16_t*>(dest.data()),
                                 dest.size() / 2);
+        if ((height - 1) * width + (width - 1) >= destView.size())
+            return false;  // going to exceed the dest bounds
         if (!bigendian())
             byteswap_span(destView);
 
@@ -2373,6 +2408,8 @@ PSDInput::decompress_zip_prediction(span<char> src, span<char> dest,
         for (uint64_t y = 0; y < height; ++y) {
             ++index;
             for (uint64_t x = 1; x < (width * sizeof(float)); ++x) {
+                if (index >= dest.size())
+                    return false;  // going to exceed the dest bounds
                 uint8_t value = dest[index] + dest[index - 1];
                 dest[index]   = value;
                 ++index;
diff --git a/testsuite/psd/ref/out-linuxarm.txt b/testsuite/psd/ref/out-linuxarm.txt
@@ -1467,14 +1467,6 @@ src/layer-mask.psd   :   10 x   10, 4 channel, uint8 psd
     stEvt:instanceID: "xmp.iid:a64763c8-be7b-ff48-b857-0f1ee8e5da2b; xmp.iid:9847e4c5-ca7e-fa42-9c7f-5fe6373d31de; xmp.iid:ddf40b95-b12c-744e-a1a9-5e7724fe4ca9"
     stEvt:softwareAgent: "Adobe Photoshop CC 2017 (Windows)"
     stEvt:when: "2017-07-13T10:26:10+09:00; 2017-07-13T10:32:41+09:00; 2017-07-13T11:42:54+09:00"
-oiiotool ERROR: read : Failed to decode Exif data
-failed to open "src/crash-psd-exif-1632.psd": failed load_resources
-Full command line was:
-> oiiotool --info -v -a --hash src/crash-psd-exif-1632.psd
-oiiotool ERROR: read : Corrupt thumbnail: 262304w * 24bpp does not match 480 width bytes
-failed to open "src/crash-thumb-1626.psd": failed load_resources
-Full command line was:
-> oiiotool --info -v -a --hash src/crash-thumb-1626.psd
 Reading src/Layers_8bit_RGB.psd
 src/Layers_8bit_RGB.psd :   48 x   27, 3 channel, uint8 psd
     4 subimages: 48x27 [u8,u8,u8], 48x27 [u8,u8,u8,u8], 48x27 [u8,u8,u8,u8], 48x27 [u8,u8,u8,u8]
@@ -2098,3 +2090,23 @@ src/Layers_32bit_RGB.psd :   48 x   27, 3 channel, float psd
     stEvt:instanceID: "xmp.iid:68fcb000-4377-c148-974d-bdd193ca024d; xmp.iid:cbd904c5-53b4-0d4e-9ff8-37ac35429f46"
     stEvt:softwareAgent: "Adobe Photoshop 23.3 (Windows)"
     stEvt:when: "2024-04-01T19:35:16+02:00"
+oiiotool ERROR: read : Failed to decode Exif data
+failed to open "src/crash-psd-exif-1632.psd": failed load_resources
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-psd-exif-1632.psd
+oiiotool ERROR: read : Corrupt thumbnail: 262304w * 24bpp does not match 480 width bytes
+failed to open "src/crash-thumb-1626.psd": failed load_resources
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-thumb-1626.psd
+oiiotool ERROR: read : failed to open "src/crash-005c.psd": failed load_layers
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-005c.psd
+oiiotool ERROR: read : [Image Data Section] channel count 3 is too few for color mode 3
+failed to open "src/crash-8a15.psd": failed load_image_data
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-8a15.psd
+oiiotool ERROR: read : unable to decode zip compressed data: src_size=39, dst_size=1296 
+Error during layer decompression. Possible corrupt file?
+failed to open "../oiio-images/psd/corrupt_20260312a.psd": failed load_global_additional
+Full command line was:
+> oiiotool --info -v -a --hash ../oiio-images/psd/corrupt_20260312a.psd
diff --git a/testsuite/psd/ref/out.txt b/testsuite/psd/ref/out.txt
@@ -1467,14 +1467,6 @@ src/layer-mask.psd   :   10 x   10, 4 channel, uint8 psd
     stEvt:instanceID: "xmp.iid:a64763c8-be7b-ff48-b857-0f1ee8e5da2b; xmp.iid:9847e4c5-ca7e-fa42-9c7f-5fe6373d31de; xmp.iid:ddf40b95-b12c-744e-a1a9-5e7724fe4ca9"
     stEvt:softwareAgent: "Adobe Photoshop CC 2017 (Windows)"
     stEvt:when: "2017-07-13T10:26:10+09:00; 2017-07-13T10:32:41+09:00; 2017-07-13T11:42:54+09:00"
-oiiotool ERROR: read : Failed to decode Exif data
-failed to open "src/crash-psd-exif-1632.psd": failed load_resources
-Full command line was:
-> oiiotool --info -v -a --hash src/crash-psd-exif-1632.psd
-oiiotool ERROR: read : Corrupt thumbnail: 262304w * 24bpp does not match 480 width bytes
-failed to open "src/crash-thumb-1626.psd": failed load_resources
-Full command line was:
-> oiiotool --info -v -a --hash src/crash-thumb-1626.psd
 Reading src/Layers_8bit_RGB.psd
 src/Layers_8bit_RGB.psd :   48 x   27, 3 channel, uint8 psd
     4 subimages: 48x27 [u8,u8,u8], 48x27 [u8,u8,u8,u8], 48x27 [u8,u8,u8,u8], 48x27 [u8,u8,u8,u8]
@@ -2098,3 +2090,23 @@ src/Layers_32bit_RGB.psd :   48 x   27, 3 channel, float psd
     stEvt:instanceID: "xmp.iid:68fcb000-4377-c148-974d-bdd193ca024d; xmp.iid:cbd904c5-53b4-0d4e-9ff8-37ac35429f46"
     stEvt:softwareAgent: "Adobe Photoshop 23.3 (Windows)"
     stEvt:when: "2024-04-01T19:35:16+02:00"
+oiiotool ERROR: read : Failed to decode Exif data
+failed to open "src/crash-psd-exif-1632.psd": failed load_resources
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-psd-exif-1632.psd
+oiiotool ERROR: read : Corrupt thumbnail: 262304w * 24bpp does not match 480 width bytes
+failed to open "src/crash-thumb-1626.psd": failed load_resources
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-thumb-1626.psd
+oiiotool ERROR: read : failed to open "src/crash-005c.psd": failed load_layers
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-005c.psd
+oiiotool ERROR: read : [Image Data Section] channel count 3 is too few for color mode 3
+failed to open "src/crash-8a15.psd": failed load_image_data
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-8a15.psd
+oiiotool ERROR: read : unable to decode zip compressed data: src_size=39, dst_size=1296 
+Error during layer decompression. Possible corrupt file?
+failed to open "../oiio-images/psd/corrupt_20260312a.psd": failed load_global_additional
+Full command line was:
+> oiiotool --info -v -a --hash ../oiio-images/psd/corrupt_20260312a.psd
diff --git a/testsuite/psd/run.py b/testsuite/psd/run.py
@@ -19,12 +19,19 @@
 command += info_command ("src/different-mask-size.psd")
 command += info_command ("src/layer-mask.psd")
 
+# Test more modern (Photoshop 2023 files) with 16- and 32-bit files containing multiple sublayers
+command += info_command ("src/Layers_8bit_RGB.psd")
+command += info_command ("src/Layers_16bit_RGB.psd")
+command += info_command ("src/Layers_32bit_RGB.psd")
+
 # This file has a corrupted Exif block
 command += info_command ("src/crash-psd-exif-1632.psd", failureok = 1)
 # Corrupted thumbnail clobbered memory
 command += info_command ("src/crash-thumb-1626.psd", failureok = 1)
+# Corruption caused an integer overflow
+command += info_command ("src/crash-005c.psd", failureok=True)
+# Corruption where the file didn't have enough channels for its color mode
+command += info_command ("src/crash-8a15.psd", failureok=True)
+# Corruption where bad zip compression data caused a buffer overrun
+command += info_command (OIIO_TESTSUITE_IMAGEDIR + "/corrupt_20260312a.psd", failureok=True)
 
-# Test more modern (Photoshop 2023 files) with 16- and 32-bit files containing multiple sublayers
-command += info_command ("src/Layers_8bit_RGB.psd")
-command += info_command ("src/Layers_16bit_RGB.psd")
-command += info_command ("src/Layers_32bit_RGB.psd")
diff --git a/testsuite/psd/src/crash-005c.psd b/testsuite/psd/src/crash-005c.psd
diff --git a/testsuite/psd/src/crash-8a15.psd b/testsuite/psd/src/crash-8a15.psd
