Check for integer overflow on mult

Fixes an issue that the fuzzer found.

Bug: 849664
Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel;luci.chromium.try:linux_optional_gpu_tests_rel;luci.chromium.try:mac_optional_gpu_tests_rel;luci.chromium.try:win_optional_gpu_tests_rel
Change-Id: I1180b9d0eced6709b1a3245c2f96deca123a71b3
Reviewed-on: https://chromium-review.googlesource.com/1087530
Commit-Queue: Jonathan Backer <backer@chromium.org>
Reviewed-by: Zhenyao Mo <zmo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#564628}

gpu/command_buffer/service/raster_decoder.cc

@@ -2491,14 +2491,6 @@ void RasterDecoderImpl::DoTexStorage2D(GLuint client_id,
     return;
   }
 
-  // For testing only. Allows us to stress the ability to respond to OOM errors.
-  if (workarounds().simulate_out_of_memory_on_large_textures &&
-      (width * height >= 4096 * 4096)) {
-    LOCAL_SET_GL_ERROR(GL_OUT_OF_MEMORY, "glTexStorage2D",
-                       "synthetic out of memory");
-    return;
-  }
-
   // Check if we have enough memory.
   unsigned int internal_format =
       viz::GLInternalFormat(texture_metadata->format());
@@ -2516,6 +2508,15 @@ void RasterDecoderImpl::DoTexStorage2D(GLuint client_id,
                        "dimensions too large");
     return;
   }
+
+  // For testing only. Allows us to stress the ability to respond to OOM errors.
+  if (workarounds().simulate_out_of_memory_on_large_textures &&
+      (width * height >= 4096 * 4096)) {
+    LOCAL_SET_GL_ERROR(GL_OUT_OF_MEMORY, "glTexStorage2D",
+                       "synthetic out of memory");
+    return;
+  }
+
   if (!EnsureGPUMemoryAvailable(size)) {
     LOCAL_SET_GL_ERROR(GL_OUT_OF_MEMORY, "glTexStorage2D", "out of memory");
     return;

gpu/command_buffer/service/raster_decoder_unittest.cc

@@ -296,6 +296,11 @@ TEST_P(RasterDecoderTest, TexStorage2DInvalid) {
   cmd.Init(client_texture_id_, kWidth, 0);
   EXPECT_EQ(error::kNoError, ExecuteCmd(cmd));
   EXPECT_EQ(GL_INVALID_VALUE, GetGLError());
+
+  // Too large: integer overflow on width * height.
+  cmd.Init(client_texture_id_, std::numeric_limits<GLsizei>::max(), 2);
+  EXPECT_EQ(error::kNoError, ExecuteCmd(cmd));
+  EXPECT_EQ(GL_OUT_OF_MEMORY, GetGLError());
 }
 
 TEST_P(RasterDecoderTest, ProduceAndConsumeTexture) {