fix(psd): fixes against corrupt files with better validation

* Validate channel count against color mode to prevent heap buffer
  overflow.

* Corrupted PSD files can declare a color mode (e.g. RGB, needing 3
  channels) with transparency (needing +1) but report fewer channels
  in the header.  This caused an out-of-bounds read in
  read_native_scanline when setup() built channel pointer arrays using
  mode_channel_count, which exceeded the actual
  m_image_data.channel_info size.

Used Claude Code / Opus 4.6 to help narrow down and confirm the bug.

Signed-off-by: Larry Gritz <lg@larrygritz.com>
Assisted-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: lgritz

src/psd.imageio/psdinput.cpp

@@ -1468,7 +1468,9 @@ PSDInput::load_layers()
     ok &= read_bige<int16_t>(layer_info.layer_count);
     if (layer_info.layer_count < 0) {
         m_image_data.transparency = true;
-        layer_info.layer_count    = -layer_info.layer_count;
+        if (layer_info.layer_count == -32768)
+            return false;  // will overflow when negated
+        layer_info.layer_count = -layer_info.layer_count;
     }
     m_layers.resize(layer_info.layer_count);
     for (int16_t layer_nbr = 0; layer_nbr < layer_info.layer_count;
@@ -1853,7 +1855,9 @@ PSDInput::load_layers_16_32(uint64_t length)
     ok &= read_bige<int16_t>(layer_info.layer_count);
     if (layer_info.layer_count < 0) {
         m_image_data.transparency = true;
-        layer_info.layer_count    = -layer_info.layer_count;
+        if (layer_info.layer_count == -32768)
+            return false;  // will overflow when negated
+        layer_info.layer_count = -layer_info.layer_count;
     }
     m_layers.resize(layer_info.layer_count);
     for (int16_t layer_nbr = 0; layer_nbr < layer_info.layer_count;
@@ -1894,6 +1898,22 @@ PSDInput::load_image_data()
                  compression);
         return false;
     }
+    // Validate that the file has enough channels for its color mode.
+    // mode_channel_count gives the minimum channels required; if the layer
+    // info section indicated transparency, we need one more channel.
+    {
+        int required = (m_header.color_mode <= ColorMode_Lab)
+                           ? (int)mode_channel_count[m_header.color_mode]
+                           : 0;
+        if (m_image_data.transparency)
+            required++;
+        if (m_header.channel_count < required) {
+            errorfmt(
+                "[Image Data Section] channel count {} is too few for color mode {}",
+                m_header.channel_count, m_header.color_mode);
+            return false;
+        }
+    }
     m_image_data.channel_info.resize(m_header.channel_count);
     // setup some generic properties and read any RLE lengths
     // Image Data Section has RLE lengths for all channels stored first

testsuite/psd/ref/out-linuxarm.txt

@@ -1467,14 +1467,6 @@ src/layer-mask.psd   :   10 x   10, 4 channel, uint8 psd
     stEvt:instanceID: "xmp.iid:a64763c8-be7b-ff48-b857-0f1ee8e5da2b; xmp.iid:9847e4c5-ca7e-fa42-9c7f-5fe6373d31de; xmp.iid:ddf40b95-b12c-744e-a1a9-5e7724fe4ca9"
     stEvt:softwareAgent: "Adobe Photoshop CC 2017 (Windows)"
     stEvt:when: "2017-07-13T10:26:10+09:00; 2017-07-13T10:32:41+09:00; 2017-07-13T11:42:54+09:00"
-oiiotool ERROR: read : Failed to decode Exif data
-failed to open "src/crash-psd-exif-1632.psd": failed load_resources
-Full command line was:
-> oiiotool --info -v -a --hash src/crash-psd-exif-1632.psd
-oiiotool ERROR: read : Corrupt thumbnail: 262304w * 24bpp does not match 480 width bytes
-failed to open "src/crash-thumb-1626.psd": failed load_resources
-Full command line was:
-> oiiotool --info -v -a --hash src/crash-thumb-1626.psd
 Reading src/Layers_8bit_RGB.psd
 src/Layers_8bit_RGB.psd :   48 x   27, 3 channel, uint8 psd
     4 subimages: 48x27 [u8,u8,u8], 48x27 [u8,u8,u8,u8], 48x27 [u8,u8,u8,u8], 48x27 [u8,u8,u8,u8]
@@ -2098,3 +2090,18 @@ src/Layers_32bit_RGB.psd :   48 x   27, 3 channel, float psd
     stEvt:instanceID: "xmp.iid:68fcb000-4377-c148-974d-bdd193ca024d; xmp.iid:cbd904c5-53b4-0d4e-9ff8-37ac35429f46"
     stEvt:softwareAgent: "Adobe Photoshop 23.3 (Windows)"
     stEvt:when: "2024-04-01T19:35:16+02:00"
+oiiotool ERROR: read : Failed to decode Exif data
+failed to open "src/crash-psd-exif-1632.psd": failed load_resources
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-psd-exif-1632.psd
+oiiotool ERROR: read : Corrupt thumbnail: 262304w * 24bpp does not match 480 width bytes
+failed to open "src/crash-thumb-1626.psd": failed load_resources
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-thumb-1626.psd
+oiiotool ERROR: read : failed to open "src/crash-005c.psd": failed load_layers
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-005c.psd
+oiiotool ERROR: read : [Image Data Section] channel count 3 is too few for color mode 3
+failed to open "src/crash-8a15.psd": failed load_image_data
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-8a15.psd

testsuite/psd/ref/out.txt

@@ -1467,14 +1467,6 @@ src/layer-mask.psd   :   10 x   10, 4 channel, uint8 psd
     stEvt:instanceID: "xmp.iid:a64763c8-be7b-ff48-b857-0f1ee8e5da2b; xmp.iid:9847e4c5-ca7e-fa42-9c7f-5fe6373d31de; xmp.iid:ddf40b95-b12c-744e-a1a9-5e7724fe4ca9"
     stEvt:softwareAgent: "Adobe Photoshop CC 2017 (Windows)"
     stEvt:when: "2017-07-13T10:26:10+09:00; 2017-07-13T10:32:41+09:00; 2017-07-13T11:42:54+09:00"
-oiiotool ERROR: read : Failed to decode Exif data
-failed to open "src/crash-psd-exif-1632.psd": failed load_resources
-Full command line was:
-> oiiotool --info -v -a --hash src/crash-psd-exif-1632.psd
-oiiotool ERROR: read : Corrupt thumbnail: 262304w * 24bpp does not match 480 width bytes
-failed to open "src/crash-thumb-1626.psd": failed load_resources
-Full command line was:
-> oiiotool --info -v -a --hash src/crash-thumb-1626.psd
 Reading src/Layers_8bit_RGB.psd
 src/Layers_8bit_RGB.psd :   48 x   27, 3 channel, uint8 psd
     4 subimages: 48x27 [u8,u8,u8], 48x27 [u8,u8,u8,u8], 48x27 [u8,u8,u8,u8], 48x27 [u8,u8,u8,u8]
@@ -2098,3 +2090,18 @@ src/Layers_32bit_RGB.psd :   48 x   27, 3 channel, float psd
     stEvt:instanceID: "xmp.iid:68fcb000-4377-c148-974d-bdd193ca024d; xmp.iid:cbd904c5-53b4-0d4e-9ff8-37ac35429f46"
     stEvt:softwareAgent: "Adobe Photoshop 23.3 (Windows)"
     stEvt:when: "2024-04-01T19:35:16+02:00"
+oiiotool ERROR: read : Failed to decode Exif data
+failed to open "src/crash-psd-exif-1632.psd": failed load_resources
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-psd-exif-1632.psd
+oiiotool ERROR: read : Corrupt thumbnail: 262304w * 24bpp does not match 480 width bytes
+failed to open "src/crash-thumb-1626.psd": failed load_resources
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-thumb-1626.psd
+oiiotool ERROR: read : failed to open "src/crash-005c.psd": failed load_layers
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-005c.psd
+oiiotool ERROR: read : [Image Data Section] channel count 3 is too few for color mode 3
+failed to open "src/crash-8a15.psd": failed load_image_data
+Full command line was:
+> oiiotool --info -v -a --hash src/crash-8a15.psd

testsuite/psd/run.py

@@ -19,12 +19,16 @@
 command += info_command ("src/different-mask-size.psd")
 command += info_command ("src/layer-mask.psd")
 
-# This file has a corrupted Exif block
-command += info_command ("src/crash-psd-exif-1632.psd", failureok = 1)
-# Corrupted thumbnail clobbered memory
-command += info_command ("src/crash-thumb-1626.psd", failureok = 1)
-
 # Test more modern (Photoshop 2023 files) with 16- and 32-bit files containing multiple sublayers
 command += info_command ("src/Layers_8bit_RGB.psd")
 command += info_command ("src/Layers_16bit_RGB.psd")
 command += info_command ("src/Layers_32bit_RGB.psd")
+
+# This file has a corrupted Exif block
+command += info_command ("src/crash-psd-exif-1632.psd", failureok = 1)
+# Corrupted thumbnail clobbered memory
+command += info_command ("src/crash-thumb-1626.psd", failureok = 1)
+# Corruption caused an integer overflow
+command += info_command ("src/crash-005c.psd", failureok=True)
+# Corruption where the file didn't have enough channels for its color mode
+command += info_command ("src/crash-8a15.psd", failureok=True)