Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SW-2646] Calculate Metrics on Arbitrary Dataset #2745

Open
wants to merge 38 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
753562d
[SW-2646] Calculate Metrics on Arbitrary Dataset
mn-mikke Nov 15, 2021
d8af501
Update api
mn-mikke Mar 18, 2022
5ed9b34
Remove unrelated tests
mn-mikke Mar 21, 2022
c50fffe
Remove unrelated logic
mn-mikke Mar 21, 2022
aeb48eb
Remove unrelated logic
mn-mikke Mar 21, 2022
e25801a
Remove extra dependency
mn-mikke Mar 21, 2022
32db73d
Remove extra dependency and return test back
mn-mikke Mar 21, 2022
a777a7d
Remove extra tests
mn-mikke Mar 21, 2022
66f0451
Update tests
mn-mikke Mar 21, 2022
38e5ab4
Fix tests
mn-mikke Mar 22, 2022
d118f74
Fix tests
mn-mikke Mar 25, 2022
3f4cf56
Python wrappers
mn-mikke Mar 28, 2022
3f2dabd
Fix multinomial tests
mn-mikke Mar 29, 2022
ef0670c
Remove prior distribution
mn-mikke Mar 29, 2022
6a4a15c
remove distribution option
mn-mikke Mar 29, 2022
624a627
spotless Apply
mn-mikke Mar 29, 2022
cefff34
dataType checks
mn-mikke Mar 29, 2022
75d1274
revert test changes in python
mn-mikke Mar 30, 2022
e6887bc
revert change in R
mn-mikke Mar 30, 2022
1708d9b
Add R classes
mn-mikke Mar 30, 2022
5240eba
Remove offset column from multinomial metrics
mn-mikke Mar 30, 2022
0dfc38e
Fix metric factory generatino
mn-mikke Mar 30, 2022
afd7676
Use original builders instead
mn-mikke Apr 4, 2022
756e169
Add python smoke tests
mn-mikke Apr 4, 2022
d1de562
Use master version
mn-mikke Apr 5, 2022
df87eb3
Update python API
mn-mikke Apr 5, 2022
87d434c
fix formatting
mn-mikke Apr 7, 2022
3002e21
fix python test
mn-mikke Apr 7, 2022
93fd203
Add R tests
mn-mikke Apr 8, 2022
c939266
add more conditions to python tests
mn-mikke Apr 8, 2022
da2c335
spotless apply
mn-mikke Apr 8, 2022
da8417c
Update metric calculation to work with just probabilities
mn-mikke Apr 12, 2022
45bc54e
Fix Python test
mn-mikke Apr 12, 2022
1965292
Fix R test
mn-mikke Apr 13, 2022
7e8f954
Address review comments from Bartosz
mn-mikke Apr 14, 2022
2be8b85
spotless apply
mn-mikke Apr 19, 2022
ea696fe
dataframe.isEmpty is not present in spark 2.2
mn-mikke Apr 20, 2022
e5457e8
Merge remote-tracking branch 'origin/master' into mn/SW-2646b
krasinski Jun 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use original builders instead
  • Loading branch information
mn-mikke committed Apr 12, 2022
commit afd76768e89921c9c407fef1686b69b119492a5e
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@

public class MetricsCalculationTypeExtensions implements TypeMapExtension {
public static final String[] MODEL_BUILDER_CLASSES = {
ModelMetrics.IndependentMetricBuilder.class.getName(),
ModelMetricsSupervised.IndependentMetricBuilderSupervised.class.getName(),
ModelMetricsBinomial.IndependentMetricBuilderBinomial.class.getName(),
ModelMetrics.MetricBuilder.class.getName(),
ModelMetricsSupervised.MetricBuilderSupervised.class.getName(),
ModelMetricsBinomial.MetricBuilderBinomial.class.getName(),
AUC2.AUCBuilder.class.getName(),
ModelMetricsRegression.IndependentMetricBuilderRegression.class.getName(),
ModelMetricsRegression.MetricBuilderRegression.class.getName(),
Distribution.class.getName(),
GaussianDistribution.class.getName(),
BernoulliDistribution.class.getName(),
Expand All @@ -33,7 +33,7 @@ public class MetricsCalculationTypeExtensions implements TypeMapExtension {
OlogitFunction.class.getName(),
OloglogFunction.class.getName(),
OprobitFunction.class.getName(),
ModelMetricsMultinomial.IndependentMetricBuilderMultinomial.class.getName()
ModelMetricsMultinomial.MetricBuilderMultinomial.class.getName()
};

public static final String[] SCHEMA_CLASSES = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@

package ai.h2o.sparkling.ml.metrics

import hex.ModelMetricsBinomial.IndependentMetricBuilderBinomial
import hex.genmodel.utils.DistributionFamily
import hex.ModelMetricsBinomial.MetricBuilderBinomial
import org.apache.spark.{ExposeUtils, ml, mllib}
import org.apache.spark.ml.util.Identifiable
import org.apache.spark.sql.types._
Expand Down Expand Up @@ -59,7 +58,7 @@ object H2OBinomialMetrics extends MetricCalculation {
weightColOption: Option[String] = None,
offsetColOption: Option[String] = None): H2OBinomialMetrics = {
validateDataFrameForMetricCalculation(dataFrame, predictionCol, labelCol, offsetColOption, weightColOption)
val getMetricBuilder = () => new IndependentMetricBuilderBinomial(domain, DistributionFamily.bernoulli)
val getMetricBuilder = () => new MetricBuilderBinomial(domain)
val castedLabelDF = dataFrame.withColumn(labelCol, col(labelCol) cast StringType)

val gson =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package ai.h2o.sparkling.ml.metrics

import hex.ModelMetricsMultinomial.IndependentMetricBuilderMultinomial
import hex.ModelMetricsMultinomial.MetricBuilderMultinomial
import hex.MultinomialAucType
import org.apache.spark.{ExposeUtils, ml, mllib}
import org.apache.spark.ml.util.Identifiable
Expand Down Expand Up @@ -67,7 +67,7 @@ object H2OMultinomialMetrics extends MetricCalculation {
val aucTypeEnum = MultinomialAucType.valueOf(aucType)
val nclasses = domain.length
val getMetricBuilder =
() => new IndependentMetricBuilderMultinomial(nclasses, domain, aucTypeEnum, null)
() => new MetricBuilderMultinomial(nclasses, domain, aucTypeEnum)
val castedLabelDF = dataFrame.withColumn(labelCol, col(labelCol) cast StringType)

val gson =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package ai.h2o.sparkling.ml.metrics

import hex.DistributionFactory
import hex.ModelMetricsRegression.IndependentMetricBuilderRegression
import hex.ModelMetricsRegression.MetricBuilderRegression
import hex.genmodel.utils.DistributionFamily
import org.apache.spark.ml.util.Identifiable
import org.apache.spark.sql.functions.col
Expand Down Expand Up @@ -54,7 +54,7 @@ object H2ORegressionMetrics extends MetricCalculation {
offsetColOption: Option[String] = None): H2ORegressionMetrics = {
validateDataFrameForMetricCalculation(dataFrame, predictionCol, labelCol, offsetColOption, weightColOption)
val getMetricBuilder =
() => new IndependentMetricBuilderRegression(DistributionFactory.getDistribution(DistributionFamily.AUTO))
() => new MetricBuilderRegression(DistributionFactory.getDistribution(DistributionFamily.AUTO))
val castedLabelDF = dataFrame.withColumn(labelCol, col(labelCol) cast DoubleType)
val gson =
getMetricGson(getMetricBuilder, castedLabelDF, predictionCol, labelCol, offsetColOption, weightColOption, null)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package ai.h2o.sparkling.ml.metrics

import com.google.gson.{GsonBuilder, JsonObject}
import hex._
import hex.ModelMetrics.IndependentMetricBuilder
import hex.ModelMetrics.MetricBuilder
import org.apache.spark.sql.{DataFrame, Row}
import water.api.{Schema, SchemaServer}
import water.api.schemas3._
Expand Down Expand Up @@ -91,7 +91,7 @@ trait MetricCalculation {
protected def getActualValue(dataType: DataType, domain: Array[String], row: Row): Double

protected def getMetricGson(
createMetricBuilder: () => IndependentMetricBuilder[_],
createMetricBuilder: () => MetricBuilder[_],
dataFrame: DataFrame,
predictionCol: String,
labelCol: String,
Expand All @@ -108,22 +108,22 @@ trait MetricCalculation {
val predictionType = flatDF.schema.fields(0).dataType
val actualType = flatDF.schema.fields(1).dataType
val filledMetricsBuilder = flatDF.rdd
.mapPartitions[IndependentMetricBuilder[_]] { rows =>
.mapPartitions[MetricBuilder[_]] { rows =>
val metricBuilder = createMetricBuilder()
while (rows.hasNext) {
val row = rows.next()
val prediction = getPredictionValues(predictionType, domain, row)
val actualValue: Double = getActualValue(actualType, domain, row)
val weight = row.getDouble(2)
val offset = row.getDouble(3)
metricBuilder.perRow(prediction, Array(actualValue), weight, offset)
metricBuilder.perRow(prediction, Array(actualValue.toFloat), weight, offset, null)
}
Iterator.single(metricBuilder)
}
.reduce((f, s) => { f.reduce(s); f })

filledMetricsBuilder.postGlobal()
val metrics = filledMetricsBuilder.makeModelMetrics()
val metrics = filledMetricsBuilder.makeModelMetrics(null, null, null, null)
val schema = metricsToSchema(metrics)
val json = schema.toJsonString
new GsonBuilder().create().fromJson(json, classOf[JsonObject])
Expand Down