-
Notifications
You must be signed in to change notification settings - Fork 1.1k
/
Copy pathXGB_encoding(LB1106.33084).py
executable file
·201 lines (165 loc) · 7.41 KB
/
XGB_encoding(LB1106.33084).py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
import numpy as np
import pandas as pd
import xgboost as xgb
from datetime import datetime
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
from scipy.stats import skew, boxcox
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
import itertools
shift = 200
COMB_FEATURE = 'cat80,cat87,cat57,cat12,cat79,cat10,cat7,cat89,cat2,cat72,' \
'cat81,cat11,cat1,cat13,cat9,cat3,cat16,cat90,cat23,cat36,' \
'cat73,cat103,cat40,cat28,cat111,cat6,cat76,cat50,cat5,' \
'cat4,cat14,cat38,cat24,cat82,cat25'.split(',')
def encode(charcode):
r = 0
ln = len(str(charcode))
for i in range(ln):
r += (ord(str(charcode)[i]) - ord('A') + 1) * 26 ** (ln - i - 1)
return r
fair_constant = 0.7
def fair_obj(preds, dtrain):
labels = dtrain.get_label()
x = (preds - labels)
den = abs(x) + fair_constant
grad = fair_constant * x / (den)
hess = fair_constant * fair_constant / (den * den)
return grad, hess
def xg_eval_mae(yhat, dtrain):
y = dtrain.get_label()
return 'mae', mean_absolute_error(np.exp(y)-shift,
np.exp(yhat)-shift)
def mungeskewed(train, test, numeric_feats):
ntrain = train.shape[0]
test['loss'] = 0
train_test = pd.concat((train, test)).reset_index(drop=True)
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.25]
skewed_feats = skewed_feats.index
for feats in skewed_feats:
train_test[feats] = train_test[feats] + 1
train_test[feats], lam = boxcox(train_test[feats])
return train_test, ntrain
if __name__ == "__main__":
print('\nStarted')
directory = '../input/'
train = pd.read_csv(directory + 'train.csv')
test = pd.read_csv(directory + 'test.csv')
# 20161203
# drop_id = np.load('./drop_id.npy')
# train = train[~train['id'].isin(drop_id)]
# 20161201
# train.drop(['cat62', 'cat63', 'cat64', 'cat55'], axis = 1, inplace = True)
# test.drop(['cat62', 'cat63', 'cat64', 'cat55'], axis = 1, inplace = True)
numeric_feats = [x for x in train.columns[1:-1] if 'cont' in x]
categorical_feats = [x for x in train.columns[1:-1] if 'cat' in x]
train_test, ntrain = mungeskewed(train, test, numeric_feats)
# taken from Vladimir's script (https://www.kaggle.com/iglovikov/allstate-claims-severity/xgb-1114)
for column in list(train.select_dtypes(include=['object']).columns):
if train[column].nunique() != test[column].nunique():
set_train = set(train[column].unique())
set_test = set(test[column].unique())
remove_train = set_train - set_test
remove_test = set_test - set_train
remove = remove_train.union(remove_test)
def filter_cat(x):
if x in remove:
return np.nan
return x
train_test[column] = train_test[column].apply(lambda x: filter_cat(x), 1)
# taken from Ali's script (https://www.kaggle.com/aliajouz/allstate-claims-severity/singel-model-lb-1117)
train_test["cont1"] = np.sqrt(preprocessing.minmax_scale(train_test["cont1"]))
train_test["cont4"] = np.sqrt(preprocessing.minmax_scale(train_test["cont4"]))
train_test["cont5"] = np.sqrt(preprocessing.minmax_scale(train_test["cont5"]))
train_test["cont8"] = np.sqrt(preprocessing.minmax_scale(train_test["cont8"]))
train_test["cont10"] = np.sqrt(preprocessing.minmax_scale(train_test["cont10"]))
train_test["cont11"] = np.sqrt(preprocessing.minmax_scale(train_test["cont11"]))
train_test["cont12"] = np.sqrt(preprocessing.minmax_scale(train_test["cont12"]))
train_test["cont6"] = np.log(preprocessing.minmax_scale(train_test["cont6"]) + 0000.1)
train_test["cont7"] = np.log(preprocessing.minmax_scale(train_test["cont7"]) + 0000.1)
train_test["cont9"] = np.log(preprocessing.minmax_scale(train_test["cont9"]) + 0000.1)
train_test["cont13"] = np.log(preprocessing.minmax_scale(train_test["cont13"]) + 0000.1)
train_test["cont14"] = (np.maximum(train_test["cont14"] - 0.179722, 0) / 0.665122) ** 0.25
print('')
for comb in itertools.combinations(COMB_FEATURE, 2):
feat = comb[0] + "_" + comb[1]
train_test[feat] = train_test[comb[0]] + train_test[comb[1]]
train_test[feat] = train_test[feat].apply(encode)
print('Combining Columns:', feat)
print('')
for col in categorical_feats:
print('Analyzing Column:', col)
train_test[col] = train_test[col].apply(encode)
print(train_test[categorical_feats])
ss = StandardScaler()
train_test[numeric_feats] = \
ss.fit_transform(train_test[numeric_feats].values)
train = train_test.iloc[:ntrain, :].copy()
test = train_test.iloc[ntrain:, :].copy()
print('\nMedian Loss:', train.loss.median())
print('Mean Loss:', train.loss.mean())
ids = pd.read_csv('../input/test.csv')['id']
train_y = np.log(train['loss'] + shift)
train_x = train.drop(['loss','id'], axis=1)
test_x = test.drop(['loss','id'], axis=1)
n_folds = 10
cv_sum = 0
early_stopping = 200
fpred = []
xgb_rounds = []
d_train_full = xgb.DMatrix(train_x, label=train_y)
d_test = xgb.DMatrix(test_x)
kf = KFold(n_splits = 7)
for i, (train_index, test_index) in enumerate(kf.split(train_x)):
print('\n Fold %d' % (i+1))
X_train, X_val = train_x.iloc[train_index], train_x.iloc[test_index]
y_train, y_val = train_y.iloc[train_index], train_y.iloc[test_index]
rand_state = 2016
params = {
'seed': 0,
'colsample_bytree': 0.7,
'silent': 1,
'subsample': 0.7,
'learning_rate': 0.01,
'objective': 'reg:linear',
'max_depth': 20,
'min_child_weight': 100,
'booster': 'gbtree'}
d_train = xgb.DMatrix(X_train, label=y_train)
d_valid = xgb.DMatrix(X_val, label=y_val)
watchlist = [(d_train, 'train'), (d_valid, 'eval')]
clf = xgb.train(params,
d_train,
10000,
watchlist,
early_stopping_rounds = early_stopping,
obj = fair_obj,
feval=xg_eval_mae)
xgb_rounds.append(clf.best_iteration)
scores_val = clf.predict(d_valid, ntree_limit=clf.best_ntree_limit)
cv_score = mean_absolute_error(np.exp(y_val), np.exp(scores_val))
print('eval-MAE: %.6f' % cv_score)
y_pred = np.exp(clf.predict(d_test, ntree_limit=clf.best_ntree_limit)) - shift
if i > 0:
fpred = pred + y_pred
else:
fpred = y_pred
pred = fpred
cv_sum = cv_sum + cv_score
mpred = pred / n_folds
score = cv_sum / n_folds
print('Average eval-MAE: %.6f' % score)
n_rounds = int(np.mean(xgb_rounds))
print("Writing results")
result = pd.DataFrame(mpred, columns=['loss'])
result["id"] = ids
result = result.set_index("id")
print("%d-fold average prediction:" % n_folds)
now = datetime.now()
score = str(round((cv_sum / n_folds), 6))
sub_file = './submission_5fold-average-xgb_fairobj_' + str(score) + '_' + str(
now.strftime("%Y-%m-%d-%H-%M")) + '.csv'
print("Writing submission: %s" % sub_file)
result.to_csv(sub_file, index=True, index_label='id')