fix(tiff): check for invalid bit depth for palette images (#5091)

Reading the TIFF 6.0 spec, on p. 23, it sure looks like palette images
can only be 4 or 8 bits per sample. But the libtiff set of test images
indeed has 2, 4, 8, and yes, 16 bps palette images among the examples.
So I think we should support up to 16, since we might encounter it in
the wild, but no higher.

We didn't previously check for this, though, and that could lead to
accepting invalid TIFF files, and then potentially have problems with
`(1 << m_bitspersample)` which appears several times. That of course is
safe if m_bitspersample can't be > 16.

Also check and proper error for a file that has a bitspersample that is
not one of the valid choices (1, 2, 4, 6, 8, 10, 12, 14, 16, 24, 32,
64), and proper errors for if the sampleformat was an unknown value or
was not a valid value for the number of bits per sample.

Note that this changes the reference output of some tests because the
new checks can alter the first error discovered in a corrupt file.

Signed-off-by: Larry Gritz <lg@larrygritz.com>

Author: lgritz

src/tiff.imageio/tiffinput.cpp

@@ -238,8 +238,10 @@ class TIFFInput final : public ImageInput {
     // like we think the file is hopelessly corrupted.
     bool readspec(bool read_meta = true);
 
-    // Figure out all the photometric-related aspects of the header
-    void readspec_photometric();
+    // Figure out all the photometric-related aspects of the header.
+    // Return true if all is fine, false if something really bad happens,
+    // like we think the file is invalid or hopelessly corrupted.
+    bool readspec_photometric();
 
     // Convert planar separate to contiguous data format
     void separate_to_contig(size_t nplanes, size_t nvals,
@@ -1205,6 +1207,7 @@ TIFFInput::readspec(bool read_meta)
     TIFFGetFieldDefaulted(m_tif, TIFFTAG_BITSPERSAMPLE, &m_bitspersample);
     m_spec.attribute("oiio:BitsPerSample", (int)m_bitspersample);
 
+    m_spec.set_format(TypeDesc::UNKNOWN);
     unsigned short sampleformat = SAMPLEFORMAT_UINT;
     TIFFGetFieldDefaulted(m_tif, TIFFTAG_SAMPLEFORMAT, &sampleformat);
     switch (m_bitspersample) {
@@ -1233,8 +1236,7 @@ TIFFInput::readspec(bool read_meta)
         else if (sampleformat == SAMPLEFORMAT_IEEEFP) {
             m_spec.set_format(TypeDesc::HALF);
             // Adobe extension, see http://chriscox.org/TIFFTN3d1.pdf
-        } else
-            m_spec.set_format(TypeDesc::UNKNOWN);
+        }
         break;
     case 24:
         // Make 24 bit look like 32 bit
@@ -1245,16 +1247,22 @@ TIFFInput::readspec(bool read_meta)
             m_spec.set_format(TypeDesc::UINT32);
         else if (sampleformat == SAMPLEFORMAT_INT)
             m_spec.set_format(TypeDesc::INT32);
-        else
-            m_spec.set_format(TypeDesc::UNKNOWN);
         break;
     case 64:
         if (sampleformat == SAMPLEFORMAT_IEEEFP)
             m_spec.set_format(TypeDesc::DOUBLE);
         else
             m_spec.set_format(TypeDesc::UNKNOWN);
         break;
-    default: m_spec.set_format(TypeDesc::UNKNOWN); break;
+    default:
+        errorfmt("Invalid bits per sample ({})", m_bitspersample);
+        return false;
+    }
+
+    if (m_spec.format == TypeUnknown) {
+        errorfmt("Invalid sampleformat value ({}) for {} bits per sample",
+                 sampleformat, m_bitspersample);
+        return false;
     }
 
     // Use the table for all the obvious things that can be mindlessly
@@ -1278,7 +1286,8 @@ TIFFInput::readspec(bool read_meta)
     TIFFGetField(m_tif, TIFFTAG_PHOTOMETRIC, &m_photometric);
     m_spec.attribute("tiff:PhotometricInterpretation", (int)m_photometric);
 
-    readspec_photometric();
+    if (!readspec_photometric())
+        return false;
 
     TIFFGetFieldDefaulted(m_tif, TIFFTAG_PLANARCONFIG, &m_planarconfig);
     m_separate = (m_planarconfig == PLANARCONFIG_SEPARATE
@@ -1579,7 +1588,7 @@ TIFFInput::readspec(bool read_meta)
 
 
 
-void
+bool
 TIFFInput::readspec_photometric()
 {
     switch (m_photometric) {
@@ -1651,6 +1660,12 @@ TIFFInput::readspec_photometric()
         m_spec.attribute("tiff:ColorSpace", "LOGLUV");
         break;
     case PHOTOMETRIC_PALETTE: {
+        if (m_bitspersample > 16) {
+            errorfmt(
+                "Palette images only support <= 16 bits per sample (was {})",
+                m_bitspersample);
+            return false;
+        }
         m_spec.attribute("tiff:ColorSpace", "palette");
         // Read the color map
         unsigned short *r = NULL, *g = NULL, *b = NULL;
@@ -1710,6 +1725,8 @@ TIFFInput::readspec_photometric()
         m_spec.attribute("oiio:ColorSpace",
                          m_spec.get_string_attribute("tiff:ColorSpace"));
     }
+
+    return true;
 }
 
 

testsuite/tiff-misc/ref/out-libtiff409.txt

@@ -14,7 +14,7 @@ src/separate.tif     :  128 x  128, 3 channel, uint8 tiff
     tiff:RowsPerStrip: 7
 Comparing "src/separate.tif" and "separate.tif"
 PASS
-oiiotool ERROR: read : No support for data format of "src/corrupt1.tif"
+oiiotool ERROR: read : "src/corrupt1.tif": Invalid bits per sample (5)
 Full command line was:
 > oiiotool --oiioattrib try_all_readers 0 --info -v src/corrupt1.tif
 oiiotool ERROR: read : File does not exist: "src/crash-1633.tif"

testsuite/tiff-misc/ref/out-libtiff410.txt

@@ -14,7 +14,7 @@ src/separate.tif     :  128 x  128, 3 channel, uint8 tiff
     tiff:RowsPerStrip: 7
 Comparing "src/separate.tif" and "separate.tif"
 PASS
-oiiotool ERROR: read : No support for data format of "src/corrupt1.tif"
+oiiotool ERROR: read : "src/corrupt1.tif": Invalid bits per sample (5)
 Full command line was:
 > oiiotool --oiioattrib try_all_readers 0 --info -v src/corrupt1.tif
 oiiotool ERROR: read : File does not exist: "src/crash-1633.tif"

testsuite/tiff-misc/ref/out-libtiff430.txt

@@ -14,7 +14,7 @@ src/separate.tif     :  128 x  128, 3 channel, uint8 tiff
     tiff:RowsPerStrip: 7
 Comparing "src/separate.tif" and "separate.tif"
 PASS
-oiiotool ERROR: read : No support for data format of "src/corrupt1.tif"
+oiiotool ERROR: read : "src/corrupt1.tif": Invalid bits per sample (5)
 Full command line was:
 > oiiotool --oiioattrib try_all_readers 0 --info -v src/corrupt1.tif
 oiiotool ERROR: read : File does not exist: "src/crash-1633.tif"

testsuite/tiff-misc/ref/out-libtiff470-b.txt

@@ -14,7 +14,7 @@ src/separate.tif     :  128 x  128, 3 channel, uint8 tiff
     tiff:RowsPerStrip: 7
 Comparing "src/separate.tif" and "separate.tif"
 PASS
-oiiotool ERROR: read : No support for data format of "src/corrupt1.tif"
+oiiotool ERROR: read : "src/corrupt1.tif": Invalid bits per sample (5)
 Full command line was:
 > oiiotool --oiioattrib try_all_readers 0 --info -v src/corrupt1.tif
 oiiotool ERROR: read : File does not exist: "src/crash-1633.tif"

testsuite/tiff-misc/ref/out-libtiff470.txt

@@ -14,7 +14,7 @@ src/separate.tif     :  128 x  128, 3 channel, uint8 tiff
     tiff:RowsPerStrip: 7
 Comparing "src/separate.tif" and "separate.tif"
 PASS
-oiiotool ERROR: read : No support for data format of "src/corrupt1.tif"
+oiiotool ERROR: read : "src/corrupt1.tif": Invalid bits per sample (5)
 Full command line was:
 > oiiotool --oiioattrib try_all_readers 0 --info -v src/corrupt1.tif
 oiiotool ERROR: read : File does not exist: "src/crash-1633.tif"

testsuite/tiff-misc/ref/out.txt

@@ -14,7 +14,7 @@ src/separate.tif     :  128 x  128, 3 channel, uint8 tiff
     tiff:RowsPerStrip: 7
 Comparing "src/separate.tif" and "separate.tif"
 PASS
-oiiotool ERROR: read : No support for data format of "src/corrupt1.tif"
+oiiotool ERROR: read : "src/corrupt1.tif": Invalid bits per sample (5)
 Full command line was:
 > oiiotool --oiioattrib try_all_readers 0 --info -v src/corrupt1.tif
 oiiotool ERROR: read : File does not exist: "src/crash-1633.tif"