Skip to content

Commit

Permalink
Merge pull request #5 from dan-blanchard/patch-1
Browse files Browse the repository at this point in the history
Fixed TabError and PEP8 compliance issues.
  • Loading branch information
benhamner committed Jul 21, 2013
2 parents 5ae41fc + d570521 commit 7460b01
Showing 1 changed file with 15 additions and 10 deletions.
25 changes: 15 additions & 10 deletions Python/ml_metrics/quadratic_weighted_kappa.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#! /usr/bin/env python2.7

import numpy as np
import pandas as pd
import pdb


def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
"""
Expand All @@ -20,6 +19,7 @@ def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat


def histogram(ratings, min_rating=None, max_rating=None):
"""
Returns the counts of each type of rating that a rater made
Expand All @@ -34,7 +34,8 @@ def histogram(ratings, min_rating=None, max_rating=None):
hist_ratings[r - min_rating] += 1
return hist_ratings

def quadratic_weighted_kappa(rater_a, rater_b, min_rating = None, max_rating = None):

def quadratic_weighted_kappa(rater_a, rater_b, min_rating=None, max_rating=None):
"""
Calculates the quadratic weighted kappa
quadratic_weighted_kappa calculates the quadratic weighted kappa
Expand Down Expand Up @@ -63,7 +64,7 @@ def quadratic_weighted_kappa(rater_a, rater_b, min_rating = None, max_rating = N
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
min_rating, max_rating)
num_ratings = len(conf_mat)
num_scored_items = float(len(rater_a))

Expand All @@ -83,7 +84,8 @@ def quadratic_weighted_kappa(rater_a, rater_b, min_rating = None, max_rating = N

return 1.0 - numerator / denominator

def linear_weighted_kappa(rater_a, rater_b, min_rating = None, max_rating = None):

def linear_weighted_kappa(rater_a, rater_b, min_rating=None, max_rating=None):
"""
Calculates the linear weighted kappa
linear_weighted_kappa calculates the linear weighted kappa
Expand All @@ -110,7 +112,7 @@ def linear_weighted_kappa(rater_a, rater_b, min_rating = None, max_rating = None
if max_rating is None:
max_rating = max(rater_a + rater_b)
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
min_rating, max_rating)
num_ratings = len(conf_mat)
num_scored_items = float(len(rater_a))

Expand All @@ -130,7 +132,8 @@ def linear_weighted_kappa(rater_a, rater_b, min_rating = None, max_rating = None

return 1.0 - numerator / denominator

def kappa(rater_a, rater_b, min_rating = None, max_rating = None):

def kappa(rater_a, rater_b, min_rating=None, max_rating=None):
"""
Calculates the kappa
kappa calculates the kappa
Expand All @@ -157,7 +160,7 @@ def kappa(rater_a, rater_b, min_rating = None, max_rating = None):
if max_rating is None:
max_rating = max(rater_a + rater_b)
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
min_rating, max_rating)
num_ratings = len(conf_mat)
num_scored_items = float(len(rater_a))

Expand All @@ -180,6 +183,7 @@ def kappa(rater_a, rater_b, min_rating = None, max_rating = None):

return 1.0 - numerator / denominator


def mean_quadratic_weighted_kappa(kappas, weights=None):
"""
Calculates the mean of the quadratic
Expand Down Expand Up @@ -210,14 +214,15 @@ def mean_quadratic_weighted_kappa(kappas, weights=None):
z = np.mean(z)
return (np.exp(2 * z) - 1) / (np.exp(2 * z) + 1)


def weighted_mean_quadratic_weighted_kappa(solution, submission):
predicted_score = submission[submission.columns[-1]].copy()
predicted_score.name = "predicted_score"
if predicted_score.index[0]==0:
if predicted_score.index[0] == 0:
predicted_score = predicted_score[:len(solution)]
predicted_score.index = solution.index
combined = solution.join(predicted_score, how="left")
groups = combined.groupby(by="essay_set")
kappas = [quadratic_weighted_kappa(group[1]["essay_score"], group[1]["predicted_score"]) for group in groups]
weights = [group[1]["essay_weight"].irow(0) for group in groups]
return mean_quadratic_weighted_kappa(kappas, weights=weights)
return mean_quadratic_weighted_kappa(kappas, weights=weights)

1 comment on commit 7460b01

@dan-blanchard
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for merging this, but you could please also update the package on the python package index?

Please sign in to comment.