Fix phpGH-16559: UBSan abort in ext/gd/libgd/gd_interpolation.c:1007

The `uchar_clamp` function was backported from old code, this backports
it from new code.

Closes phpGH-16562.

Author: ndossche

NEWS

@@ -55,6 +55,8 @@ PHP                                                                        NEWS
   . Fixed bug GH-16334 (imageaffine overflow on matrix elements).
     (David Carlier)
   . Fixed bug GH-16427 (Unchecked libavif return values). (cmb)
+  . Fixed bug GH-16559 (UBSan abort in ext/gd/libgd/gd_interpolation.c:1007).
+    (nielsdos)
 
 - GMP:
   . Fixed floating point exception bug with gmp_pow when using

ext/gd/libgd/gd_interpolation.c

@@ -929,21 +929,29 @@ static inline LineContribType *_gdContributionsCalc(unsigned int line_size, unsi
 	return res;
 }
 
+/* Convert a double to an unsigned char, rounding to the nearest
+ * integer and clamping the result between 0 and max.  The absolute
+ * value of clr must be less than the maximum value of an unsigned
+ * short. */
 static inline unsigned char
-uchar_clamp(double clr) {
+uchar_clamp(double clr, unsigned char max) {
 	unsigned short result;
-	assert(fabs(clr) <= SHRT_MAX);
+
+	//assert(fabs(clr) <= SHRT_MAX);
+
 	/* Casting a negative float to an unsigned short is undefined.
 	 * However, casting a float to a signed truncates toward zero and
 	 * casting a negative signed value to an unsigned of the same size
 	 * results in a bit-identical value (assuming twos-complement
 	 * arithmetic).	 This is what we want: all legal negative values
 	 * for clr will be greater than 255. */
+
 	/* Convert and clamp. */
 	result = (unsigned short)(short)(clr + 0.5);
-	if (result > 255) {
-		result = (clr < 0) ? 0 : 255;
+	if (result > max) {
+		result = (clr < 0) ? 0 : max;
 	}/* if */
+
 	return result;
 }/* uchar_clamp*/
 
@@ -967,7 +975,9 @@ static inline void _gdScaleRow(gdImagePtr pSrc,  unsigned int src_width, gdImage
 		b += contrib->ContribRow[x].Weights[left_channel] * (double)(gdTrueColorGetBlue(p_src_row[i]));
 		a += contrib->ContribRow[x].Weights[left_channel] * (double)(gdTrueColorGetAlpha(p_src_row[i]));
 	}
-	p_dst_row[x] = gdTrueColorAlpha(uchar_clamp(r), uchar_clamp(g), uchar_clamp(b), uchar_clamp(a));
+	p_dst_row[x] = gdTrueColorAlpha(uchar_clamp(r, 0xFF), uchar_clamp(g, 0xFF),
+									uchar_clamp(b, 0xFF),
+									uchar_clamp(a, 0x7F)); /* alpha is 0..127 */
     }
 }
 
@@ -1014,7 +1024,9 @@ static inline void _gdScaleCol (gdImagePtr pSrc,  unsigned int src_width, gdImag
 			b += contrib->ContribRow[y].Weights[i_iLeft] * (double)(gdTrueColorGetBlue(pCurSrc));
 			a += contrib->ContribRow[y].Weights[i_iLeft] * (double)(gdTrueColorGetAlpha(pCurSrc));
 		}
-		pRes->tpixels[y][uCol] = gdTrueColorAlpha(uchar_clamp(r), uchar_clamp(g), uchar_clamp(b), uchar_clamp(a));
+		pRes->tpixels[y][uCol] = gdTrueColorAlpha(uchar_clamp(r, 0xFF), uchar_clamp(g, 0xFF),
+												  uchar_clamp(b, 0xFF),
+												  uchar_clamp(a, 0x7F)); /* alpha is 0..127 */
 	}
 }
 

ext/gd/tests/gh16559.phpt

@@ -0,0 +1,15 @@
+--TEST--
+GH-16559 (UBSan abort in ext/gd/libgd/gd_interpolation.c:1007)
+--EXTENSIONS--
+gd
+--FILE--
+<?php
+$input = imagecreatefrompng(__DIR__ . '/gh10614.png');
+for ($angle = 0; $angle <= 270; $angle += 90) {
+	$output = imagerotate($input, $angle, 0);
+}
+var_dump(imagescale($output, -1, 64, IMG_BICUBIC));
+?>
+--EXPECT--
+object(GdImage)#2 (0) {
+}