gpu_test_expectations_parser_unittest.cc

// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <stdint.h>

#include "base/format_macros.h"
#include "base/logging.h"
#include "base/strings/stringprintf.h"
#include "gpu/config/gpu_test_expectations_parser.h"
#include "testing/gtest/include/gtest/gtest.h"

namespace gpu {

struct TestOSEntry {
  const char* name;
  GPUTestConfig::OS os;
};

static const TestOSEntry kOsFamilyWin = { "WIN", GPUTestConfig::kOsWin };
static const TestOSEntry kOsFamilyMac = { "MAC", GPUTestConfig::kOsMac };

static const struct TestOsWithFamily {
  TestOSEntry version;
  TestOSEntry family;
} kOSVersionsWithFamily[] = {
    {{"XP", GPUTestConfig::kOsWinXP}, kOsFamilyWin},
    {{"VISTA", GPUTestConfig::kOsWinVista}, kOsFamilyWin},
    {{"WIN7", GPUTestConfig::kOsWin7}, kOsFamilyWin},
    {{"WIN8", GPUTestConfig::kOsWin8}, kOsFamilyWin},
    {{"WIN10", GPUTestConfig::kOsWin10}, kOsFamilyWin},
    {{"LEOPARD", GPUTestConfig::kOsMacLeopard}, kOsFamilyMac},
    {{"SNOWLEOPARD", GPUTestConfig::kOsMacSnowLeopard}, kOsFamilyMac},
    {{"LION", GPUTestConfig::kOsMacLion}, kOsFamilyMac},
    {{"MOUNTAINLION", GPUTestConfig::kOsMacMountainLion}, kOsFamilyMac},
    {{"MAVERICKS", GPUTestConfig::kOsMacMavericks}, kOsFamilyMac},
    {{"YOSEMITE", GPUTestConfig::kOsMacYosemite}, kOsFamilyMac},
    {{"ELCAPITAN", GPUTestConfig::kOsMacElCapitan}, kOsFamilyMac},
    {{"SIERRA", GPUTestConfig::kOsMacSierra}, kOsFamilyMac},
    {{"HIGHSIERRA", GPUTestConfig::kOsMacHighSierra}, kOsFamilyMac},
    {{"LINUX", GPUTestConfig::kOsLinux}, {"LINUX", GPUTestConfig::kOsLinux}},
    {{"CHROMEOS", GPUTestConfig::kOsChromeOS},
     {"CHROMEOS", GPUTestConfig::kOsChromeOS}},
    {{"ANDROID", GPUTestConfig::kOsAndroid},
     {"ANDROID", GPUTestConfig::kOsAndroid}}};

TestOSEntry GetUnrelatedOS(const TestOSEntry& os) {
  return (os.os & kOsFamilyWin.os) ? kOsFamilyMac : kOsFamilyWin;
}

// Prints test parameter details.
std::ostream& operator << (std::ostream& out, const TestOsWithFamily& os) {
  out << "{ os_name: \"" << os.version.name
      << "\", os_flag: " << os.version.os
      << ", os_family: \"" << os.family.name
      << "\", os_family_flag: " << os.family.os
      << " }";
  return out;
}

class GPUTestExpectationsParserTest : public testing::Test {
 public:
  GPUTestExpectationsParserTest() = default;

  ~GPUTestExpectationsParserTest() override = default;

  const GPUTestBotConfig& bot_config() const {
    return bot_config_;
  }

 protected:
  void SetUp() override {
    bot_config_.set_os(GPUTestConfig::kOsWin7);
    bot_config_.set_build_type(GPUTestConfig::kBuildTypeRelease);
    bot_config_.AddGPUVendor(0x10de);
    bot_config_.set_gpu_device_id(0x0640);
    bot_config_.set_api(GPUTestConfig::kAPID3D11);
    ASSERT_TRUE(bot_config_.IsValid());
  }

  void TearDown() override {}

  void set_os(int32_t os) {
    bot_config_.set_os(os);
    ASSERT_TRUE(bot_config_.IsValid());
  }

 private:
  GPUTestBotConfig bot_config_;
};

class GPUTestExpectationsParserParamTest
    : public GPUTestExpectationsParserTest,
      public testing::WithParamInterface<TestOsWithFamily> {
 public:
  GPUTestExpectationsParserParamTest() = default;

  ~GPUTestExpectationsParserParamTest() override = default;

 protected:
  const GPUTestBotConfig& GetBotConfig() {
    set_os(GetParam().version.os);
    return bot_config();
  }

 private:
  // Restrict access to bot_config() function.
  // GetBotConfig() should be used instead.
  using GPUTestExpectationsParserTest::bot_config;
};

TEST_F(GPUTestExpectationsParserTest, CommentOnly) {
  const std::string text =
      "  \n"
      "// This is just some comment\n"
      "";
  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
            parser.GetTestExpectation("some_test", bot_config()));
}

TEST_P(GPUTestExpectationsParserParamTest, ValidFullEntry) {
  const std::string text =
      base::StringPrintf("BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
                         GetParam().version.name);

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
            parser.GetTestExpectation("MyTest", GetBotConfig()));
}

TEST_P(GPUTestExpectationsParserParamTest, ValidPartialEntry) {
  const std::string text =
      base::StringPrintf("BUG12345 %s NVIDIA : MyTest = TIMEOUT",
                         GetParam().family.name);

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestTimeout,
            parser.GetTestExpectation("MyTest", GetBotConfig()));
}

TEST_P(GPUTestExpectationsParserParamTest, ValidUnrelatedOsEntry) {
  const std::string text = base::StringPrintf(
      "BUG12345 %s : MyTest = TIMEOUT",
      GetUnrelatedOS(GetParam().version).name);

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
            parser.GetTestExpectation("MyTest", GetBotConfig()));
}

TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedTestEntry) {
  const std::string text =
      "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : AnotherTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
            parser.GetTestExpectation("MyTest", bot_config()));
}

TEST_F(GPUTestExpectationsParserTest, AllModifiers) {
  const std::string text =
      "BUG12345 XP VISTA WIN7 WIN8 WIN10 LEOPARD SNOWLEOPARD LION MOUNTAINLION "
      "MAVERICKS LINUX CHROMEOS ANDROID "
      "NVIDIA INTEL AMD VMWARE RELEASE DEBUG : MyTest = "
      "PASS FAIL FLAKY TIMEOUT SKIP";

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
            GPUTestExpectationsParser::kGpuTestFail |
            GPUTestExpectationsParser::kGpuTestFlaky |
            GPUTestExpectationsParser::kGpuTestTimeout |
            GPUTestExpectationsParser::kGpuTestSkip,
            parser.GetTestExpectation("MyTest", bot_config()));
}

TEST_P(GPUTestExpectationsParserParamTest, DuplicateModifiers) {
  const std::string text = base::StringPrintf(
      "BUG12345 %s %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
      GetParam().version.name,
      GetParam().version.name);

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_F(GPUTestExpectationsParserTest, AllModifiersLowerCase) {
  const std::string text =
      "BUG12345 xp vista win7 win8 win10 leopard snowleopard lion linux "
      "chromeos android nvidia intel amd vmware release debug : MyTest = "
      "pass fail flaky timeout skip";

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
            GPUTestExpectationsParser::kGpuTestFail |
            GPUTestExpectationsParser::kGpuTestFlaky |
            GPUTestExpectationsParser::kGpuTestTimeout |
            GPUTestExpectationsParser::kGpuTestSkip,
            parser.GetTestExpectation("MyTest", bot_config()));
}

TEST_F(GPUTestExpectationsParserTest, MissingColon) {
  const std::string text =
      "BUG12345 XP MyTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_F(GPUTestExpectationsParserTest, MissingEqual) {
  const std::string text =
      "BUG12345 XP : MyTest FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_F(GPUTestExpectationsParserTest, IllegalModifier) {
  const std::string text =
      "BUG12345 XP XXX : MyTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_P(GPUTestExpectationsParserParamTest, OsConflicts) {
  const std::string text = base::StringPrintf("BUG12345 %s %s : MyTest = FAIL",
                                              GetParam().version.name,
                                              GetParam().family.name);

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_F(GPUTestExpectationsParserTest, InvalidModifierCombination) {
  const std::string text =
      "BUG12345 XP NVIDIA INTEL 0x0640 : MyTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_F(GPUTestExpectationsParserTest, BadGpuDeviceID) {
  const std::string text =
      "BUG12345 XP NVIDIA 0xU07X : MyTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_F(GPUTestExpectationsParserTest, MoreThanOneGpuDeviceID) {
  const std::string text =
      "BUG12345 XP NVIDIA 0x0640 0x0641 : MyTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_P(GPUTestExpectationsParserParamTest, MultipleEntriesConflicts) {
  const std::string text = base::StringPrintf(
      "BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
      "BUG12345 %s : MyTest = FAIL",
      GetParam().version.name,
      GetParam().family.name);

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

TEST_F(GPUTestExpectationsParserTest, MultipleTests) {
  const std::string text =
      "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
      "BUG12345 WIN : AnotherTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
}

TEST_F(GPUTestExpectationsParserTest, ValidMultipleEntries) {
  const std::string text =
      "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
      "BUG12345 LINUX : MyTest = TIMEOUT";

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
            parser.GetTestExpectation("MyTest", bot_config()));
}

TEST_F(GPUTestExpectationsParserTest, StarMatching) {
  const std::string text =
      "BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest* = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
            parser.GetTestExpectation("MyTest0", bot_config()));
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
            parser.GetTestExpectation("OtherTest", bot_config()));
}

TEST_F(GPUTestExpectationsParserTest, ValidAPI) {
  const std::string text =
      "BUG12345 WIN7 NVIDIA D3D9 D3D11 OPENGL GLES : MyTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_TRUE(parser.LoadTestExpectations(text));
  EXPECT_EQ(0u, parser.GetErrorMessages().size());
  EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
            parser.GetTestExpectation("MyTest", bot_config()));
}

TEST_F(GPUTestExpectationsParserTest, MultipleAPIsConflict) {
  const std::string text = "BUG12345 WIN7 NVIDIA D3D9 D3D9 : MyTest = FAIL";

  GPUTestExpectationsParser parser;
  EXPECT_FALSE(parser.LoadTestExpectations(text));
  EXPECT_NE(0u, parser.GetErrorMessages().size());
}

INSTANTIATE_TEST_CASE_P(GPUTestExpectationsParser,
                        GPUTestExpectationsParserParamTest,
                        ::testing::ValuesIn(kOSVersionsWithFamily));

}  // namespace gpu