|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | + * contributor license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright ownership. |
| 5 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | + * (the "License"); you may not use this file except in compliance with |
| 7 | + * the License. You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | + |
| 18 | +package org.apache.spark.mllib.evaluation.binary |
| 19 | + |
| 20 | +import org.apache.spark.rdd.{UnionRDD, RDD} |
| 21 | +import org.apache.spark.SparkContext._ |
| 22 | +import org.apache.spark.mllib.evaluation.AreaUnderCurve |
| 23 | +import org.apache.spark.Logging |
| 24 | + |
| 25 | +/** |
| 26 | + * Implementation of [[org.apache.spark.mllib.evaluation.binary.BinaryConfusionMatrix]]. |
| 27 | + * |
| 28 | + * @param count label counter for labels with scores greater than or equal to the current score |
| 29 | + * @param totalCount label counter for all labels |
| 30 | + */ |
| 31 | +private case class BinaryConfusionMatrixImpl( |
| 32 | + count: LabelCounter, |
| 33 | + totalCount: LabelCounter) extends BinaryConfusionMatrix with Serializable { |
| 34 | + |
| 35 | + /** number of true positives */ |
| 36 | + override def numTruePositives: Long = count.numPositives |
| 37 | + |
| 38 | + /** number of false positives */ |
| 39 | + override def numFalsePositives: Long = count.numNegatives |
| 40 | + |
| 41 | + /** number of false negatives */ |
| 42 | + override def numFalseNegatives: Long = totalCount.numPositives - count.numPositives |
| 43 | + |
| 44 | + /** number of true negatives */ |
| 45 | + override def numTrueNegatives: Long = totalCount.numNegatives - count.numNegatives |
| 46 | + |
| 47 | + /** number of positives */ |
| 48 | + override def numPositives: Long = totalCount.numPositives |
| 49 | + |
| 50 | + /** number of negatives */ |
| 51 | + override def numNegatives: Long = totalCount.numNegatives |
| 52 | +} |
| 53 | + |
| 54 | +/** |
| 55 | + * Evaluator for binary classification. |
| 56 | + * |
| 57 | + * @param scoreAndLabels an RDD of (score, label) pairs. |
| 58 | + */ |
| 59 | +class BinaryClassificationMetrics(scoreAndLabels: RDD[(Double, Double)]) |
| 60 | + extends Serializable with Logging { |
| 61 | + |
| 62 | + private lazy val ( |
| 63 | + cumulativeCounts: RDD[(Double, LabelCounter)], |
| 64 | + confusions: RDD[(Double, BinaryConfusionMatrix)]) = { |
| 65 | + // Create a bin for each distinct score value, count positives and negatives within each bin, |
| 66 | + // and then sort by score values in descending order. |
| 67 | + val counts = scoreAndLabels.combineByKey( |
| 68 | + createCombiner = (label: Double) => new LabelCounter(0L, 0L) += label, |
| 69 | + mergeValue = (c: LabelCounter, label: Double) => c += label, |
| 70 | + mergeCombiners = (c1: LabelCounter, c2: LabelCounter) => c1 += c2 |
| 71 | + ).sortByKey(ascending = false) |
| 72 | + val agg = counts.values.mapPartitions({ iter => |
| 73 | + val agg = new LabelCounter() |
| 74 | + iter.foreach(agg += _) |
| 75 | + Iterator(agg) |
| 76 | + }, preservesPartitioning = true).collect() |
| 77 | + val partitionwiseCumulativeCounts = |
| 78 | + agg.scanLeft(new LabelCounter())((agg: LabelCounter, c: LabelCounter) => agg.clone() += c) |
| 79 | + val totalCount = partitionwiseCumulativeCounts.last |
| 80 | + logInfo(s"Total counts: $totalCount") |
| 81 | + val cumulativeCounts = counts.mapPartitionsWithIndex( |
| 82 | + (index: Int, iter: Iterator[(Double, LabelCounter)]) => { |
| 83 | + val cumCount = partitionwiseCumulativeCounts(index) |
| 84 | + iter.map { case (score, c) => |
| 85 | + cumCount += c |
| 86 | + (score, cumCount.clone()) |
| 87 | + } |
| 88 | + }, preservesPartitioning = true) |
| 89 | + cumulativeCounts.persist() |
| 90 | + val confusions = cumulativeCounts.map { case (score, cumCount) => |
| 91 | + (score, BinaryConfusionMatrixImpl(cumCount, totalCount).asInstanceOf[BinaryConfusionMatrix]) |
| 92 | + } |
| 93 | + (cumulativeCounts, confusions) |
| 94 | + } |
| 95 | + |
| 96 | + /** Unpersist intermediate RDDs used in the computation. */ |
| 97 | + def unpersist() { |
| 98 | + cumulativeCounts.unpersist() |
| 99 | + } |
| 100 | + |
| 101 | + /** Returns thresholds in descending order. */ |
| 102 | + def thresholds(): RDD[Double] = cumulativeCounts.map(_._1) |
| 103 | + |
| 104 | + /** |
| 105 | + * Returns the receiver operating characteristic (ROC) curve, |
| 106 | + * which is an RDD of (false positive rate, true positive rate) |
| 107 | + * with (0.0, 0.0) prepended and (1.0, 1.0) appended to it. |
| 108 | + * @see http://en.wikipedia.org/wiki/Receiver_operating_characteristic |
| 109 | + */ |
| 110 | + def roc(): RDD[(Double, Double)] = { |
| 111 | + val rocCurve = createCurve(FalsePositiveRate, Recall) |
| 112 | + val sc = confusions.context |
| 113 | + val first = sc.makeRDD(Seq((0.0, 0.0)), 1) |
| 114 | + val last = sc.makeRDD(Seq((1.0, 1.0)), 1) |
| 115 | + new UnionRDD[(Double, Double)](sc, Seq(first, rocCurve, last)) |
| 116 | + } |
| 117 | + |
| 118 | + /** |
| 119 | + * Computes the area under the receiver operating characteristic (ROC) curve. |
| 120 | + */ |
| 121 | + def areaUnderROC(): Double = AreaUnderCurve.of(roc()) |
| 122 | + |
| 123 | + /** |
| 124 | + * Returns the precision-recall curve, which is an RDD of (recall, precision), |
| 125 | + * NOT (precision, recall), with (0.0, 1.0) prepended to it. |
| 126 | + * @see http://en.wikipedia.org/wiki/Precision_and_recall |
| 127 | + */ |
| 128 | + def pr(): RDD[(Double, Double)] = { |
| 129 | + val prCurve = createCurve(Recall, Precision) |
| 130 | + val sc = confusions.context |
| 131 | + val first = sc.makeRDD(Seq((0.0, 1.0)), 1) |
| 132 | + first.union(prCurve) |
| 133 | + } |
| 134 | + |
| 135 | + /** |
| 136 | + * Computes the area under the precision-recall curve. |
| 137 | + */ |
| 138 | + def areaUnderPR(): Double = AreaUnderCurve.of(pr()) |
| 139 | + |
| 140 | + /** |
| 141 | + * Returns the (threshold, F-Measure) curve. |
| 142 | + * @param beta the beta factor in F-Measure computation. |
| 143 | + * @return an RDD of (threshold, F-Measure) pairs. |
| 144 | + * @see http://en.wikipedia.org/wiki/F1_score |
| 145 | + */ |
| 146 | + def fMeasureByThreshold(beta: Double): RDD[(Double, Double)] = createCurve(FMeasure(beta)) |
| 147 | + |
| 148 | + /** Returns the (threshold, F-Measure) curve with beta = 1.0. */ |
| 149 | + def fMeasureByThreshold(): RDD[(Double, Double)] = fMeasureByThreshold(1.0) |
| 150 | + |
| 151 | + /** Returns the (threshold, precision) curve. */ |
| 152 | + def precisionByThreshold(): RDD[(Double, Double)] = createCurve(Precision) |
| 153 | + |
| 154 | + /** Returns the (threshold, recall) curve. */ |
| 155 | + def recallByThreshold(): RDD[(Double, Double)] = createCurve(Recall) |
| 156 | + |
| 157 | + /** Creates a curve of (threshold, metric). */ |
| 158 | + private def createCurve(y: BinaryClassificationMetricComputer): RDD[(Double, Double)] = { |
| 159 | + confusions.map { case (s, c) => |
| 160 | + (s, y(c)) |
| 161 | + } |
| 162 | + } |
| 163 | + |
| 164 | + /** Creates a curve of (metricX, metricY). */ |
| 165 | + private def createCurve( |
| 166 | + x: BinaryClassificationMetricComputer, |
| 167 | + y: BinaryClassificationMetricComputer): RDD[(Double, Double)] = { |
| 168 | + confusions.map { case (_, c) => |
| 169 | + (x(c), y(c)) |
| 170 | + } |
| 171 | + } |
| 172 | +} |
| 173 | + |
| 174 | +/** |
| 175 | + * A counter for positives and negatives. |
| 176 | + * |
| 177 | + * @param numPositives number of positive labels |
| 178 | + * @param numNegatives number of negative labels |
| 179 | + */ |
| 180 | +private class LabelCounter( |
| 181 | + var numPositives: Long = 0L, |
| 182 | + var numNegatives: Long = 0L) extends Serializable { |
| 183 | + |
| 184 | + /** Processes a label. */ |
| 185 | + def +=(label: Double): LabelCounter = { |
| 186 | + // Though we assume 1.0 for positive and 0.0 for negative, the following check will handle |
| 187 | + // -1.0 for negative as well. |
| 188 | + if (label > 0.5) numPositives += 1L else numNegatives += 1L |
| 189 | + this |
| 190 | + } |
| 191 | + |
| 192 | + /** Merges another counter. */ |
| 193 | + def +=(other: LabelCounter): LabelCounter = { |
| 194 | + numPositives += other.numPositives |
| 195 | + numNegatives += other.numNegatives |
| 196 | + this |
| 197 | + } |
| 198 | + |
| 199 | + override def clone: LabelCounter = { |
| 200 | + new LabelCounter(numPositives, numNegatives) |
| 201 | + } |
| 202 | + |
| 203 | + override def toString: String = s"{numPos: $numPositives, numNeg: $numNegatives}" |
| 204 | +} |
0 commit comments