fix(bmp): use check_open to guard against corrupt resolutions (#5086)

Also change the x and y counters to uint64_t to prevent any possible
overflow as it's computing offsets.

---------

Signed-off-by: Larry Gritz <lg@larrygritz.com>

Author: lgritz

src/bmp.imageio/bmpinput.cpp

@@ -186,20 +186,14 @@ BmpInput::open(const std::string& name, ImageSpec& newspec,
     const int height    = (m_dib_header.height >= 0) ? m_dib_header.height
                                                      : -m_dib_header.height;
     m_spec = ImageSpec(m_dib_header.width, height, nchannels, TypeDesc::UINT8);
+    if (!check_open(m_spec, { 0, 1 << 16, 0, 1 << 16, 0, 1, 0, 4 }))
+        return false;
+
     if (m_dib_header.hres > 0 && m_dib_header.vres > 0) {
         m_spec.attribute("XResolution", (int)m_dib_header.hres);
         m_spec.attribute("YResolution", (int)m_dib_header.vres);
         m_spec.attribute("ResolutionUnit", "m");
     }
-    if (m_spec.width < 1 || m_spec.height < 1 || m_spec.nchannels < 1
-        || m_spec.image_bytes() < 1
-        || m_spec.image_pixels() > std::numeric_limits<uint32_t>::max()) {
-        errorfmt(
-            "Invalid image size {} x {} ({} chans, {}), is likely corrupted",
-            m_spec.width, m_spec.height, m_spec.nchannels, m_spec.format);
-        close();
-        return false;
-    }
 
     // Compute channel shifts & masks (only relevant for 16bpp case)
     if (m_dib_header.red_mask == 0 || m_dib_header.green_mask == 0
@@ -309,8 +303,8 @@ BmpInput::read_rle_image()
     m_uncompressed.clear();
     m_uncompressed.resize(m_spec.image_pixels());
     // Note: the clear+resize zeroes out the buffer
-    bool ok = true;
-    int y = 0, x = 0;
+    bool ok   = true;
+    int64_t y = 0, x = 0;
     while (ok) {
         // Strutil::print("currently at {},{}\n", x, y);
         unsigned char rle_pair[2];
@@ -402,7 +396,8 @@ BmpInput::read_native_scanline(int subimage, int miplevel, int y, int /*z*/,
          || m_dib_header.compression == RLE8_COMPRESSION)
         && m_colortable.size()) {
         for (int x = 0; x < m_spec.width; ++x) {
-            int p = m_uncompressed[(m_spec.height - 1 - y) * m_spec.width + x];
+            int p = m_uncompressed[int64_t(m_spec.height - 1 - y) * m_spec.width
+                                   + x];
             auto& c              = colortable(p);
             mscanline[3 * x]     = c.r;
             mscanline[3 * x + 1] = c.g;

testsuite/bmp/ref/out.txt

@@ -301,3 +301,6 @@ Full command line was:
 oiiotool ERROR: read : "src/palette32bit-corrupt.bmp": BMP error: bad BPP (32) for palette image -- presumed corrupt file
 Full command line was:
 > oiiotool --info -v -a --hash src/palette32bit-corrupt.bmp
+oiiotool ERROR: read : "src/huge-width.bmp": bmp image resolution may not exceed 65536x65536, but the file appears to be 536870911x8. Possible corrupt input?
+Full command line was:
+> oiiotool --info -v -a --hash src/huge-width.bmp

testsuite/bmp/run.py

@@ -34,3 +34,4 @@
 command += info_command ("src/decodecolormap-corrupt.bmp")
 command += info_command ("src/bad-y.bmp")
 command += info_command ("src/palette32bit-corrupt.bmp")
+command += info_command ("src/huge-width.bmp")