[CUDA] Add fair regression objective for cuda_exp (microsoft#5469)

* change percentile in CUDARegressionL1loss::BoostFromScore to 0.5

src/objective/cuda/cuda_regression_objective.cpp

@@ -95,6 +95,16 @@ CUDARegressionL2loss(strs) {}
 
 CUDARegressionHuberLoss::~CUDARegressionHuberLoss() {}
 
+
+CUDARegressionFairLoss::CUDARegressionFairLoss(const Config& config):
+CUDARegressionL2loss(config), c_(config.fair_c) {}
+
+CUDARegressionFairLoss::CUDARegressionFairLoss(const std::vector<std::string>& strs):
+CUDARegressionL2loss(strs) {}
+
+CUDARegressionFairLoss::~CUDARegressionFairLoss() {}
+
+
 }  // namespace LightGBM
 
 #endif  // USE_CUDA_EXP

src/objective/cuda/cuda_regression_objective.cu

@@ -70,7 +70,7 @@ void CUDARegressionL2loss::LaunchGetGradientsKernel(const double* score, score_t
 
 
 double CUDARegressionL1loss::LaunchCalcInitScoreKernel() const {
-  const double alpha = 0.9f;
+  const double alpha = 0.5f;
   if (cuda_weights_ == nullptr) {
     PercentileGlobal<label_t, data_size_t, label_t, double, false, false>(
       cuda_labels_, nullptr, cuda_data_indices_buffer_.RawData(), nullptr, nullptr, alpha, num_data_, cuda_percentile_result_.RawData());
@@ -226,6 +226,35 @@ void CUDARegressionHuberLoss::LaunchGetGradientsKernel(const double* score, scor
   }
 }
 
+
+template <bool USE_WEIGHT>
+__global__ void GetGradientsKernel_Fair(const double* cuda_scores, const label_t* cuda_labels, const label_t* cuda_weights, const data_size_t num_data,
+  const double c, score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
+  const data_size_t data_index = static_cast<data_size_t>(blockDim.x * blockIdx.x + threadIdx.x);
+  if (data_index < num_data) {
+    if (!USE_WEIGHT) {
+      const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
+      cuda_out_gradients[data_index] = static_cast<score_t>(c * diff / (fabs(diff) + c));
+      cuda_out_hessians[data_index] = static_cast<score_t>(c * c / ((fabs(diff) + c) * (fabs(diff) + c)));
+    } else {
+      const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
+      const score_t weight = static_cast<score_t>(cuda_weights[data_index]);
+      cuda_out_gradients[data_index] = static_cast<score_t>(c * diff / (fabs(diff) + c) * weight);
+      cuda_out_hessians[data_index] = static_cast<score_t>(c * c / ((fabs(diff) + c) * (fabs(diff) + c)) * weight);
+    }
+  }
+}
+
+void CUDARegressionFairLoss::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
+  const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
+  if (cuda_weights_ == nullptr) {
+    GetGradientsKernel_Fair<false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, nullptr, num_data_, c_, gradients, hessians);
+  } else {
+    GetGradientsKernel_Fair<true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, cuda_weights_, num_data_, c_, gradients, hessians);
+  }
+}
+
+
 }  // namespace LightGBM
 
 #endif  // USE_CUDA_EXP

src/objective/cuda/cuda_regression_objective.hpp

@@ -118,6 +118,26 @@ class CUDARegressionHuberLoss : public CUDARegressionL2loss {
 };
 
 
+// http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html
+class CUDARegressionFairLoss : public CUDARegressionL2loss {
+ public:
+  explicit CUDARegressionFairLoss(const Config& config);
+
+  explicit CUDARegressionFairLoss(const std::vector<std::string>& strs);
+
+  ~CUDARegressionFairLoss();
+
+  bool IsConstantHessian() const override {
+    return false;
+  }
+
+ private:
+  void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const override;
+
+  const double c_ = 0.0f;
+};
+
+
 }  // namespace LightGBM
 
 #endif  // USE_CUDA_EXP

src/objective/objective_function.cpp

@@ -22,17 +22,14 @@ ObjectiveFunction* ObjectiveFunction::CreateObjectiveFunction(const std::string&
     if (type == std::string("regression")) {
       return new CUDARegressionL2loss(config);
     } else if (type == std::string("regression_l1")) {
-      Log::Warning("Objective regression_l1 is not implemented in cuda_exp version. Fall back to boosting on CPU.");
-      return new RegressionL1loss(config);
+      return new CUDARegressionL1loss(config);
     } else if (type == std::string("quantile")) {
       Log::Warning("Objective quantile is not implemented in cuda_exp version. Fall back to boosting on CPU.");
       return new RegressionQuantileloss(config);
     } else if (type == std::string("huber")) {
-      Log::Warning("Objective huber is not implemented in cuda_exp version. Fall back to boosting on CPU.");
-      return new RegressionHuberLoss(config);
+      return new CUDARegressionHuberLoss(config);
     } else if (type == std::string("fair")) {
-      Log::Warning("Objective fair is not implemented in cuda_exp version. Fall back to boosting on CPU.");
-      return new RegressionFairLoss(config);
+      return new CUDARegressionFairLoss(config);
     } else if (type == std::string("poisson")) {
       Log::Warning("Objective poisson is not implemented in cuda_exp version. Fall back to boosting on CPU.");
       return new RegressionPoissonLoss(config);

tests/python_package_test/test_engine.py

@@ -111,7 +111,7 @@ def test_rf():
     assert evals_result['valid_0']['binary_logloss'][-1] == pytest.approx(ret)
 
 
-@pytest.mark.parametrize('objective', ['regression', 'regression_l1', 'huber'])
+@pytest.mark.parametrize('objective', ['regression', 'regression_l1', 'huber', 'fair'])
 def test_regression(objective):
     X, y = load_boston(return_X_y=True)
     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
@@ -133,6 +133,8 @@ def test_regression(objective):
     ret = mean_squared_error(y_test, gbm.predict(X_test))
     if objective == 'huber':
         assert ret < 35
+    elif objective == 'fair':
+        assert ret < 17
     else:
         assert ret < 7
     assert evals_result['valid_0']['l2'][-1] == pytest.approx(ret)