Skip to content

Commit ba68899

Browse files
wantehchangwantehchang
committed
Fix overflows in two functions in src/reformat.c
Fix overflows when multiplying with rowBytes in avifImageRGBToYUV() and avifImageYUVAnyToRGBAnySlow(), by storing the various uint32_t rowBytes fields in local variables of the size_t type. Then multiplications with the size_t rowBytes local variables will be done in size_t. Part of the fix to #2271.
1 parent 1f86eb6 commit ba68899

1 file changed

Lines changed: 37 additions & 38 deletions

File tree

‎src/reformat.c‎

Lines changed: 37 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -271,8 +271,13 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb)
271271
struct YUVBlock yuvBlock[2][2];
272272
float rgbPixel[3];
273273
const float rgbMaxChannelF = state.rgb.maxChannelF;
274-
uint8_t ** yuvPlanes = image->yuvPlanes;
275-
uint32_t * yuvRowBytes = image->yuvRowBytes;
274+
const size_t rgbRowBytes = rgb->rowBytes;
275+
uint8_t * yPlane = image->yuvPlanes[AVIF_CHAN_Y];
276+
uint8_t * uPlane = image->yuvPlanes[AVIF_CHAN_U];
277+
uint8_t * vPlane = image->yuvPlanes[AVIF_CHAN_V];
278+
const size_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y];
279+
const size_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U];
280+
const size_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V];
276281
for (uint32_t outerJ = 0; outerJ < image->height; outerJ += 2) {
277282
for (uint32_t outerI = 0; outerI < image->width; outerI += 2) {
278283
int blockW = 2, blockH = 2;
@@ -292,30 +297,30 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb)
292297
// Unpack RGB into normalized float
293298
if (state.rgb.channelBytes > 1) {
294299
rgbPixel[0] =
295-
*((uint16_t *)(&rgb->pixels[state.rgb.offsetBytesR + (i * state.rgb.pixelBytes) + (j * rgb->rowBytes)])) /
300+
*((uint16_t *)(&rgb->pixels[state.rgb.offsetBytesR + (i * state.rgb.pixelBytes) + (j * rgbRowBytes)])) /
296301
rgbMaxChannelF;
297302
rgbPixel[1] =
298-
*((uint16_t *)(&rgb->pixels[state.rgb.offsetBytesG + (i * state.rgb.pixelBytes) + (j * rgb->rowBytes)])) /
303+
*((uint16_t *)(&rgb->pixels[state.rgb.offsetBytesG + (i * state.rgb.pixelBytes) + (j * rgbRowBytes)])) /
299304
rgbMaxChannelF;
300305
rgbPixel[2] =
301-
*((uint16_t *)(&rgb->pixels[state.rgb.offsetBytesB + (i * state.rgb.pixelBytes) + (j * rgb->rowBytes)])) /
306+
*((uint16_t *)(&rgb->pixels[state.rgb.offsetBytesB + (i * state.rgb.pixelBytes) + (j * rgbRowBytes)])) /
302307
rgbMaxChannelF;
303308
} else {
304-
rgbPixel[0] = rgb->pixels[state.rgb.offsetBytesR + (i * state.rgb.pixelBytes) + (j * rgb->rowBytes)] /
309+
rgbPixel[0] = rgb->pixels[state.rgb.offsetBytesR + (i * state.rgb.pixelBytes) + (j * rgbRowBytes)] /
305310
rgbMaxChannelF;
306-
rgbPixel[1] = rgb->pixels[state.rgb.offsetBytesG + (i * state.rgb.pixelBytes) + (j * rgb->rowBytes)] /
311+
rgbPixel[1] = rgb->pixels[state.rgb.offsetBytesG + (i * state.rgb.pixelBytes) + (j * rgbRowBytes)] /
307312
rgbMaxChannelF;
308-
rgbPixel[2] = rgb->pixels[state.rgb.offsetBytesB + (i * state.rgb.pixelBytes) + (j * rgb->rowBytes)] /
313+
rgbPixel[2] = rgb->pixels[state.rgb.offsetBytesB + (i * state.rgb.pixelBytes) + (j * rgbRowBytes)] /
309314
rgbMaxChannelF;
310315
}
311316

312317
if (alphaMode != AVIF_ALPHA_MULTIPLY_MODE_NO_OP) {
313318
float a;
314319
if (state.rgb.channelBytes > 1) {
315-
a = *((uint16_t *)(&rgb->pixels[state.rgb.offsetBytesA + (i * state.rgb.pixelBytes) + (j * rgb->rowBytes)])) /
320+
a = *((uint16_t *)(&rgb->pixels[state.rgb.offsetBytesA + (i * state.rgb.pixelBytes) + (j * rgbRowBytes)])) /
316321
rgbMaxChannelF;
317322
} else {
318-
a = rgb->pixels[state.rgb.offsetBytesA + (i * state.rgb.pixelBytes) + (j * rgb->rowBytes)] / rgbMaxChannelF;
323+
a = rgb->pixels[state.rgb.offsetBytesA + (i * state.rgb.pixelBytes) + (j * rgbRowBytes)] / rgbMaxChannelF;
319324
}
320325

321326
if (alphaMode == AVIF_ALPHA_MULTIPLY_MODE_MULTIPLY) {
@@ -377,24 +382,21 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb)
377382
}
378383

379384
if (state.yuv.channelBytes > 1) {
380-
uint16_t * pY = (uint16_t *)&yuvPlanes[AVIF_CHAN_Y][(i * 2) + (j * yuvRowBytes[AVIF_CHAN_Y])];
385+
uint16_t * pY = (uint16_t *)&yPlane[(i * 2) + (j * yRowBytes)];
381386
*pY = (uint16_t)avifYUVColorSpaceInfoYToUNorm(&state.yuv, yuvBlock[bI][bJ].y);
382387
if (image->yuvFormat == AVIF_PIXEL_FORMAT_YUV444) {
383388
// YUV444, full chroma
384-
uint16_t * pU = (uint16_t *)&yuvPlanes[AVIF_CHAN_U][(i * 2) + (j * yuvRowBytes[AVIF_CHAN_U])];
389+
uint16_t * pU = (uint16_t *)&uPlane[(i * 2) + (j * uRowBytes)];
385390
*pU = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, yuvBlock[bI][bJ].u);
386-
uint16_t * pV = (uint16_t *)&yuvPlanes[AVIF_CHAN_V][(i * 2) + (j * yuvRowBytes[AVIF_CHAN_V])];
391+
uint16_t * pV = (uint16_t *)&vPlane[(i * 2) + (j * vRowBytes)];
387392
*pV = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, yuvBlock[bI][bJ].v);
388393
}
389394
} else {
390-
yuvPlanes[AVIF_CHAN_Y][i + (j * yuvRowBytes[AVIF_CHAN_Y])] =
391-
(uint8_t)avifYUVColorSpaceInfoYToUNorm(&state.yuv, yuvBlock[bI][bJ].y);
395+
yPlane[i + (j * yRowBytes)] = (uint8_t)avifYUVColorSpaceInfoYToUNorm(&state.yuv, yuvBlock[bI][bJ].y);
392396
if (image->yuvFormat == AVIF_PIXEL_FORMAT_YUV444) {
393397
// YUV444, full chroma
394-
yuvPlanes[AVIF_CHAN_U][i + (j * yuvRowBytes[AVIF_CHAN_U])] =
395-
(uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, yuvBlock[bI][bJ].u);
396-
yuvPlanes[AVIF_CHAN_V][i + (j * yuvRowBytes[AVIF_CHAN_V])] =
397-
(uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, yuvBlock[bI][bJ].v);
398+
uPlane[i + (j * uRowBytes)] = (uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, yuvBlock[bI][bJ].u);
399+
vPlane[i + (j * vRowBytes)] = (uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, yuvBlock[bI][bJ].v);
398400
}
399401
}
400402
}
@@ -423,15 +425,13 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb)
423425
int uvI = outerI >> chromaShiftX;
424426
int uvJ = outerJ >> chromaShiftY;
425427
if (state.yuv.channelBytes > 1) {
426-
uint16_t * pU = (uint16_t *)&yuvPlanes[AVIF_CHAN_U][(uvI * 2) + (uvJ * yuvRowBytes[AVIF_CHAN_U])];
428+
uint16_t * pU = (uint16_t *)&uPlane[(uvI * 2) + (uvJ * uRowBytes)];
427429
*pU = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);
428-
uint16_t * pV = (uint16_t *)&yuvPlanes[AVIF_CHAN_V][(uvI * 2) + (uvJ * yuvRowBytes[AVIF_CHAN_V])];
430+
uint16_t * pV = (uint16_t *)&vPlane[(uvI * 2) + (uvJ * vRowBytes)];
429431
*pV = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgV);
430432
} else {
431-
yuvPlanes[AVIF_CHAN_U][uvI + (uvJ * yuvRowBytes[AVIF_CHAN_U])] =
432-
(uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);
433-
yuvPlanes[AVIF_CHAN_V][uvI + (uvJ * yuvRowBytes[AVIF_CHAN_V])] =
434-
(uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgV);
433+
uPlane[uvI + (uvJ * uRowBytes)] = (uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);
434+
vPlane[uvI + (uvJ * vRowBytes)] = (uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgV);
435435
}
436436
} else if (image->yuvFormat == AVIF_PIXEL_FORMAT_YUV422) {
437437
// YUV422, average 2 samples (1x2), twice
@@ -451,15 +451,13 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb)
451451
int uvI = outerI >> chromaShiftX;
452452
int uvJ = outerJ + bJ;
453453
if (state.yuv.channelBytes > 1) {
454-
uint16_t * pU = (uint16_t *)&yuvPlanes[AVIF_CHAN_U][(uvI * 2) + (uvJ * yuvRowBytes[AVIF_CHAN_U])];
454+
uint16_t * pU = (uint16_t *)&uPlane[(uvI * 2) + (uvJ * uRowBytes)];
455455
*pU = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);
456-
uint16_t * pV = (uint16_t *)&yuvPlanes[AVIF_CHAN_V][(uvI * 2) + (uvJ * yuvRowBytes[AVIF_CHAN_V])];
456+
uint16_t * pV = (uint16_t *)&vPlane[(uvI * 2) + (uvJ * vRowBytes)];
457457
*pV = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgV);
458458
} else {
459-
yuvPlanes[AVIF_CHAN_U][uvI + (uvJ * yuvRowBytes[AVIF_CHAN_U])] =
460-
(uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);
461-
yuvPlanes[AVIF_CHAN_V][uvI + (uvJ * yuvRowBytes[AVIF_CHAN_V])] =
462-
(uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgV);
459+
uPlane[uvI + (uvJ * uRowBytes)] = (uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);
460+
vPlane[uvI + (uvJ * vRowBytes)] = (uint8_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgV);
463461
}
464462
}
465463
}
@@ -592,10 +590,11 @@ static avifResult avifImageYUVAnyToRGBAnySlow(const avifImage * image,
592590
const uint8_t * uPlane = image->yuvPlanes[AVIF_CHAN_U];
593591
const uint8_t * vPlane = image->yuvPlanes[AVIF_CHAN_V];
594592
const uint8_t * aPlane = image->alphaPlane;
595-
const uint32_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y];
596-
const uint32_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U];
597-
const uint32_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V];
598-
const uint32_t aRowBytes = image->alphaRowBytes;
593+
const size_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y];
594+
const size_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U];
595+
const size_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V];
596+
const size_t aRowBytes = image->alphaRowBytes;
597+
const size_t rgbRowBytes = rgb->rowBytes;
599598

600599
// Various observations and limits
601600
const avifBool hasColor = (uPlane && vPlane && (image->yuvFormat != AVIF_PIXEL_FORMAT_YUV400));
@@ -619,9 +618,9 @@ static avifResult avifImageYUVAnyToRGBAnySlow(const avifImage * image,
619618
const uint16_t * ptrV16 = (const uint16_t *)ptrV8;
620619
const uint16_t * ptrA16 = (const uint16_t *)ptrA8;
621620

622-
uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)];
623-
uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)];
624-
uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)];
621+
uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgbRowBytes)];
622+
uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgbRowBytes)];
623+
uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgbRowBytes)];
625624

626625
for (uint32_t i = 0; i < image->width; ++i) {
627626
float Y, Cb = 0.5f, Cr = 0.5f;

0 commit comments

Comments
 (0)