분류 - 앙상블

voting

• 서로 다른 알고리즘이 결합. 분류에서는 voting¹으로 결정

Example

import pandas as pd

from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import warnings

warnings.filterwarnings('ignore')

cancer = load_breast_cancer()

df = pd.DataFrame(cancer.data, columns=cancer.feature_names)
df

	mean radius	mean texture	mean perimeter	mean area	mean smoothness	mean compactness	mean concavity	mean concave points	mean symmetry	mean fractal dimension	...	worst radius	worst texture	worst perimeter	worst area	worst smoothness	worst compactness	worst concavity	worst concave points	worst symmetry	worst fractal dimension
0	17.99	10.38	122.80	1001.0	0.11840	0.27760	0.30010	0.14710	0.2419	0.07871	...	25.380	17.33	184.60	2019.0	0.16220	0.66560	0.7119	0.2654	0.4601	0.11890
1	20.57	17.77	132.90	1326.0	0.08474	0.07864	0.08690	0.07017	0.1812	0.05667	...	24.990	23.41	158.80	1956.0	0.12380	0.18660	0.2416	0.1860	0.2750	0.08902
2	19.69	21.25	130.00	1203.0	0.10960	0.15990	0.19740	0.12790	0.2069	0.05999	...	23.570	25.53	152.50	1709.0	0.14440	0.42450	0.4504	0.2430	0.3613	0.08758
3	11.42	20.38	77.58	386.1	0.14250	0.28390	0.24140	0.10520	0.2597	0.09744	...	14.910	26.50	98.87	567.7	0.20980	0.86630	0.6869	0.2575	0.6638	0.17300
4	20.29	14.34	135.10	1297.0	0.10030	0.13280	0.19800	0.10430	0.1809	0.05883	...	22.540	16.67	152.20	1575.0	0.13740	0.20500	0.4000	0.1625	0.2364	0.07678
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
564	21.56	22.39	142.00	1479.0	0.11100	0.11590	0.24390	0.13890	0.1726	0.05623	...	25.450	26.40	166.10	2027.0	0.14100	0.21130	0.4107	0.2216	0.2060	0.07115
565	20.13	28.25	131.20	1261.0	0.09780	0.10340	0.14400	0.09791	0.1752	0.05533	...	23.690	38.25	155.00	1731.0	0.11660	0.19220	0.3215	0.1628	0.2572	0.06637
566	16.60	28.08	108.30	858.1	0.08455	0.10230	0.09251	0.05302	0.1590	0.05648	...	18.980	34.12	126.70	1124.0	0.11390	0.30940	0.3403	0.1418	0.2218	0.07820
567	20.60	29.33	140.10	1265.0	0.11780	0.27700	0.35140	0.15200	0.2397	0.07016	...	25.740	39.42	184.60	1821.0	0.16500	0.86810	0.9387	0.2650	0.4087	0.12400
568	7.76	24.54	47.92	181.0	0.05263	0.04362	0.00000	0.00000	0.1587	0.05884	...	9.456	30.37	59.16	268.6	0.08996	0.06444	0.0000	0.0000	0.2871	0.07039

569 rows × 30 columns

lr_clf = LogisticRegression(solver='liblinear')
knn_clf = KNeighborsClassifier(n_neighbors=8)

vo_clf = VotingClassifier(estimators=[('LR', lr_clf), ('KNN', knn_clf)],
                          voting='soft')
X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, test_size=0.2)
vo_clf.fit(X_train, y_train)
pred = vo_clf.predict(X_test)
accuracy = accuracy_score(y_test, pred)
accuracy

0.9385964912280702

for classifier in [lr_clf, knn_clf]:
    classifier.fit(X_train, y_train)
    pred = classifier.predict(X_test)
    class_name = classifier.__class__.__name__
    print(f'{class_name} 정확도: {accuracy_score(y_test, pred):.4f}')

LogisticRegression 정확도: 0.9386
KNeighborsClassifier 정확도: 0.9123

• 반드시 voting이 제일 좋은 모델을 선택하는 것보다 좋은건 아님

bagging

• 같은 유형의 알고리즘의 분류기가 boostrap 해가서 예측. random forest가 대표적. 분류에서는 voting²으로 결정

RandomForest

from sklearn.ensemble import RandomForestClassifier

def get_new_feature_name_df(old):
    df = pd.DataFrame(data=old.groupby('column_name').cumcount(), columns=['dup_cnt'])
    df = df.reset_index()
    new_df = pd.merge(old.reset_index(), df, how='outer')
    new_df['column_name'] = new_df[['column_name', 'dup_cnt']].apply(lambda x: x[0] + '_' + str(x[1]) if x[1] > 0 else x[0], axis=1)
    new_df = new_df.drop(['index'], axis=1)
    return new_df

def get_human_dataset():
    feature_name_df = pd.read_csv('_data/human_activity/features.txt', sep='\s+', header=None, names=['column_index', 'column_name'])
    new_feature_name_df = get_new_feature_name_df(feature_name_df)
    feature_name = new_feature_name_df.iloc[:, 1].values.tolist()

    X_train = pd.read_csv('_data/human_activity/train/X_train.txt', sep='\s+', names=feature_name)
    X_test = pd.read_csv('_data/human_activity/test/X_test.txt', sep='\s+', names=feature_name)

    y_train = pd.read_csv('_data/human_activity/train/y_train.txt', sep='\s+', header=None, names=['action'])
    y_test = pd.read_csv('_data/human_activity/test/y_test.txt', sep='\s+', header=None, names=['action'])

    return X_train, X_test, y_train, y_test

X_train, X_test, y_train, y_test = get_human_dataset()

rf_clf = RandomForestClassifier(max_depth=8)
rf_clf.fit(X_train, y_train)
pred = rf_clf.predict(X_test)
accuracy = accuracy_score(y_test, pred)
accuracy

0.9216152019002375

boosting

GBM

# from sklearn.ensemble import GradientBoostingClassifier
# import time
# 
# X_train, X_test, y_train, y_test = get_human_dataset()
# start_time = time.time()
# 
# gb_clf = GradientBoostingClassifier()
# gb_clf.fit(X_train, y_train)
# gb_pred = gb_clf.predict(X_test)
# gb_accuracy = accuracy_score(y_test, gb_pred)
#
# end_time = time.time()
#
# print(f'{gb_accuracy:.3f}, {end_time - start_time}초')

0.939, 701.6343066692352초

• 아주 오래 걸림.

XGBoost

• 결손값을 자체 처리할 수 있다.

• 조기 종료 기능이 있다.

• 자체적으로 교차 검증, 성능 평가, 피처 중요도 시각화 기능이 있다.

• python xgboost

import xgboost as xgb
from xgboost import plot_importance
import numpy as np

dataset = load_breast_cancer()

X_train, X_test, y_train, y_test = train_test_split(dataset.data, dataset.target, test_size=0.2)
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.1)

dtr = xgb.DMatrix(data=X_tr, label=y_tr)
dval = xgb.DMatrix(data=X_val, label=y_val)
dtest = xgb.DMatrix(data=X_test, label=y_test)

params = {
    'max_depth': 3,
    'eta': 0.05,
    'objective': 'binary:logistic',
    'eval_metric': 'logloss'
}
num_rounds = 400

eval_list = [(dtr, 'train'), (dval, 'eval')]

xgb_model = xgb.train(params=params, dtrain=dtr, num_boost_round=num_rounds, early_stopping_rounds=50, evals=eval_list)

[0] train-logloss:0.62424   eval-logloss:0.59777
[1] train-logloss:0.58779   eval-logloss:0.56156
[2] train-logloss:0.55432   eval-logloss:0.53054
[3] train-logloss:0.52351   eval-logloss:0.50160
[4] train-logloss:0.49539   eval-logloss:0.47522
[5] train-logloss:0.46757   eval-logloss:0.44992
[6] train-logloss:0.44383   eval-logloss:0.42761
[7] train-logloss:0.42005   eval-logloss:0.40613
[8] train-logloss:0.39808   eval-logloss:0.38633
[9] train-logloss:0.37870   eval-logloss:0.36842
[10]    train-logloss:0.35969   eval-logloss:0.35141
[11]    train-logloss:0.34290   eval-logloss:0.33598
[12]    train-logloss:0.32758   eval-logloss:0.32125
[13]    train-logloss:0.31287   eval-logloss:0.30786
[14]    train-logloss:0.29892   eval-logloss:0.29789
[15]    train-logloss:0.28558   eval-logloss:0.28766
[16]    train-logloss:0.27366   eval-logloss:0.27631
[17]    train-logloss:0.26184   eval-logloss:0.26744
[18]    train-logloss:0.25113   eval-logloss:0.25662
[19]    train-logloss:0.24069   eval-logloss:0.24660
[20]    train-logloss:0.23095   eval-logloss:0.24040
[21]    train-logloss:0.22133   eval-logloss:0.23252
[22]    train-logloss:0.21279   eval-logloss:0.22393
[23]    train-logloss:0.20420   eval-logloss:0.21701
[24]    train-logloss:0.19629   eval-logloss:0.20948
[25]    train-logloss:0.18899   eval-logloss:0.20315
[26]    train-logloss:0.18165   eval-logloss:0.19747
[27]    train-logloss:0.17512   eval-logloss:0.19161
[28]    train-logloss:0.16837   eval-logloss:0.18409
[29]    train-logloss:0.16213   eval-logloss:0.17747
[30]    train-logloss:0.15644   eval-logloss:0.17195
[31]    train-logloss:0.15069   eval-logloss:0.16564
[32]    train-logloss:0.14545   eval-logloss:0.16122
[33]    train-logloss:0.14039   eval-logloss:0.15579
[34]    train-logloss:0.13553   eval-logloss:0.15171
[35]    train-logloss:0.13076   eval-logloss:0.14631
[36]    train-logloss:0.12661   eval-logloss:0.14321
[37]    train-logloss:0.12243   eval-logloss:0.13868
[38]    train-logloss:0.11868   eval-logloss:0.13596
[39]    train-logloss:0.11490   eval-logloss:0.13172
[40]    train-logloss:0.11126   eval-logloss:0.12735
[41]    train-logloss:0.10781   eval-logloss:0.12351
[42]    train-logloss:0.10450   eval-logloss:0.11957
[43]    train-logloss:0.10131   eval-logloss:0.11603
[44]    train-logloss:0.09846   eval-logloss:0.11396
[45]    train-logloss:0.09545   eval-logloss:0.11227
[46]    train-logloss:0.09284   eval-logloss:0.11015
[47]    train-logloss:0.09020   eval-logloss:0.10715
[48]    train-logloss:0.08763   eval-logloss:0.10404
[49]    train-logloss:0.08535   eval-logloss:0.10262
[50]    train-logloss:0.08302   eval-logloss:0.10000
[51]    train-logloss:0.08075   eval-logloss:0.09821
[52]    train-logloss:0.07862   eval-logloss:0.09605
[53]    train-logloss:0.07660   eval-logloss:0.09381
[54]    train-logloss:0.07470   eval-logloss:0.09237
[55]    train-logloss:0.07281   eval-logloss:0.09051
[56]    train-logloss:0.07107   eval-logloss:0.08923
[57]    train-logloss:0.06933   eval-logloss:0.08746
[58]    train-logloss:0.06769   eval-logloss:0.08634
[59]    train-logloss:0.06609   eval-logloss:0.08479
[60]    train-logloss:0.06459   eval-logloss:0.08378
[61]    train-logloss:0.06299   eval-logloss:0.08323
[62]    train-logloss:0.06160   eval-logloss:0.08241
[63]    train-logloss:0.06027   eval-logloss:0.08159
[64]    train-logloss:0.05892   eval-logloss:0.08046
[65]    train-logloss:0.05770   eval-logloss:0.07975
[66]    train-logloss:0.05651   eval-logloss:0.07913
[67]    train-logloss:0.05529   eval-logloss:0.07875
[68]    train-logloss:0.05421   eval-logloss:0.07815
[69]    train-logloss:0.05280   eval-logloss:0.07674
[70]    train-logloss:0.05177   eval-logloss:0.07657
[71]    train-logloss:0.05074   eval-logloss:0.07522
[72]    train-logloss:0.04977   eval-logloss:0.07474
[73]    train-logloss:0.04878   eval-logloss:0.07455
[74]    train-logloss:0.04788   eval-logloss:0.07450
[75]    train-logloss:0.04706   eval-logloss:0.07308
[76]    train-logloss:0.04608   eval-logloss:0.07206
[77]    train-logloss:0.04528   eval-logloss:0.07178
[78]    train-logloss:0.04443   eval-logloss:0.07170
[79]    train-logloss:0.04341   eval-logloss:0.07047
[80]    train-logloss:0.04251   eval-logloss:0.06994
[81]    train-logloss:0.04184   eval-logloss:0.06921
[82]    train-logloss:0.04118   eval-logloss:0.06898
[83]    train-logloss:0.04037   eval-logloss:0.06854
[84]    train-logloss:0.03966   eval-logloss:0.06873
[85]    train-logloss:0.03903   eval-logloss:0.06884
[86]    train-logloss:0.03844   eval-logloss:0.06833
[87]    train-logloss:0.03779   eval-logloss:0.06856
[88]    train-logloss:0.03722   eval-logloss:0.06870
[89]    train-logloss:0.03648   eval-logloss:0.06859
[90]    train-logloss:0.03580   eval-logloss:0.06858
[91]    train-logloss:0.03525   eval-logloss:0.06808
[92]    train-logloss:0.03472   eval-logloss:0.06694
[93]    train-logloss:0.03410   eval-logloss:0.06672
[94]    train-logloss:0.03361   eval-logloss:0.06564
[95]    train-logloss:0.03307   eval-logloss:0.06545
[96]    train-logloss:0.03257   eval-logloss:0.06536
[97]    train-logloss:0.03205   eval-logloss:0.06465
[98]    train-logloss:0.03150   eval-logloss:0.06441
[99]    train-logloss:0.03105   eval-logloss:0.06357
[100]   train-logloss:0.03057   eval-logloss:0.06387
[101]   train-logloss:0.03006   eval-logloss:0.06370
[102]   train-logloss:0.02967   eval-logloss:0.06274
[103]   train-logloss:0.02925   eval-logloss:0.06244
[104]   train-logloss:0.02878   eval-logloss:0.06232
[105]   train-logloss:0.02836   eval-logloss:0.06265
[106]   train-logloss:0.02799   eval-logloss:0.06195
[107]   train-logloss:0.02764   eval-logloss:0.06176
[108]   train-logloss:0.02731   eval-logloss:0.06152
[109]   train-logloss:0.02690   eval-logloss:0.06142
[110]   train-logloss:0.02657   eval-logloss:0.06076
[111]   train-logloss:0.02626   eval-logloss:0.06059
[112]   train-logloss:0.02589   eval-logloss:0.06010
[113]   train-logloss:0.02553   eval-logloss:0.06003
[114]   train-logloss:0.02519   eval-logloss:0.05933
[115]   train-logloss:0.02480   eval-logloss:0.05840
[116]   train-logloss:0.02451   eval-logloss:0.05855
[117]   train-logloss:0.02414   eval-logloss:0.05766
[118]   train-logloss:0.02382   eval-logloss:0.05761
[119]   train-logloss:0.02348   eval-logloss:0.05676
[120]   train-logloss:0.02322   eval-logloss:0.05692
[121]   train-logloss:0.02290   eval-logloss:0.05610
[122]   train-logloss:0.02265   eval-logloss:0.05592
[123]   train-logloss:0.02243   eval-logloss:0.05622
[124]   train-logloss:0.02213   eval-logloss:0.05542
[125]   train-logloss:0.02189   eval-logloss:0.05473
[126]   train-logloss:0.02168   eval-logloss:0.05504
[127]   train-logloss:0.02140   eval-logloss:0.05427
[128]   train-logloss:0.02118   eval-logloss:0.05401
[129]   train-logloss:0.02093   eval-logloss:0.05346
[130]   train-logloss:0.02075   eval-logloss:0.05376
[131]   train-logloss:0.02049   eval-logloss:0.05298
[132]   train-logloss:0.02026   eval-logloss:0.05259
[133]   train-logloss:0.02003   eval-logloss:0.05251
[134]   train-logloss:0.01981   eval-logloss:0.05233
[135]   train-logloss:0.01962   eval-logloss:0.05210
[136]   train-logloss:0.01945   eval-logloss:0.05208
[137]   train-logloss:0.01925   eval-logloss:0.05146
[138]   train-logloss:0.01909   eval-logloss:0.05080
[139]   train-logloss:0.01891   eval-logloss:0.05059
[140]   train-logloss:0.01877   eval-logloss:0.05075
[141]   train-logloss:0.01862   eval-logloss:0.05036
[142]   train-logloss:0.01846   eval-logloss:0.05019
[143]   train-logloss:0.01826   eval-logloss:0.04959
[144]   train-logloss:0.01807   eval-logloss:0.04956
[145]   train-logloss:0.01791   eval-logloss:0.04940
[146]   train-logloss:0.01776   eval-logloss:0.04897
[147]   train-logloss:0.01761   eval-logloss:0.04879
[148]   train-logloss:0.01743   eval-logloss:0.04864
[149]   train-logloss:0.01730   eval-logloss:0.04864
[150]   train-logloss:0.01718   eval-logloss:0.04833
[151]   train-logloss:0.01703   eval-logloss:0.04792
[152]   train-logloss:0.01690   eval-logloss:0.04819
[153]   train-logloss:0.01677   eval-logloss:0.04801
[154]   train-logloss:0.01662   eval-logloss:0.04794
[155]   train-logloss:0.01646   eval-logloss:0.04764
[156]   train-logloss:0.01631   eval-logloss:0.04754
[157]   train-logloss:0.01619   eval-logloss:0.04778
[158]   train-logloss:0.01609   eval-logloss:0.04780
[159]   train-logloss:0.01598   eval-logloss:0.04805
[160]   train-logloss:0.01584   eval-logloss:0.04791
[161]   train-logloss:0.01575   eval-logloss:0.04757
[162]   train-logloss:0.01565   eval-logloss:0.04760
[163]   train-logloss:0.01555   eval-logloss:0.04782
[164]   train-logloss:0.01547   eval-logloss:0.04758
[165]   train-logloss:0.01532   eval-logloss:0.04775
[166]   train-logloss:0.01522   eval-logloss:0.04799
[167]   train-logloss:0.01514   eval-logloss:0.04821
[168]   train-logloss:0.01507   eval-logloss:0.04827
[169]   train-logloss:0.01499   eval-logloss:0.04803
[170]   train-logloss:0.01485   eval-logloss:0.04819
[171]   train-logloss:0.01475   eval-logloss:0.04788
[172]   train-logloss:0.01465   eval-logloss:0.04811
[173]   train-logloss:0.01456   eval-logloss:0.04763
[174]   train-logloss:0.01445   eval-logloss:0.04770
[175]   train-logloss:0.01432   eval-logloss:0.04777
[176]   train-logloss:0.01425   eval-logloss:0.04780
[177]   train-logloss:0.01418   eval-logloss:0.04809
[178]   train-logloss:0.01411   eval-logloss:0.04782
[179]   train-logloss:0.01405   eval-logloss:0.04787
[180]   train-logloss:0.01394   eval-logloss:0.04793
[181]   train-logloss:0.01387   eval-logloss:0.04771
[182]   train-logloss:0.01377   eval-logloss:0.04778
[183]   train-logloss:0.01371   eval-logloss:0.04738
[184]   train-logloss:0.01363   eval-logloss:0.04761
[185]   train-logloss:0.01353   eval-logloss:0.04763
[186]   train-logloss:0.01346   eval-logloss:0.04737
[187]   train-logloss:0.01340   eval-logloss:0.04761
[188]   train-logloss:0.01335   eval-logloss:0.04761
[189]   train-logloss:0.01329   eval-logloss:0.04757
[190]   train-logloss:0.01323   eval-logloss:0.04718
[191]   train-logloss:0.01318   eval-logloss:0.04718
[192]   train-logloss:0.01312   eval-logloss:0.04715
[193]   train-logloss:0.01306   eval-logloss:0.04690
[194]   train-logloss:0.01300   eval-logloss:0.04713
[195]   train-logloss:0.01295   eval-logloss:0.04713
[196]   train-logloss:0.01287   eval-logloss:0.04715
[197]   train-logloss:0.01282   eval-logloss:0.04691
[198]   train-logloss:0.01272   eval-logloss:0.04719
[199]   train-logloss:0.01266   eval-logloss:0.04742
[200]   train-logloss:0.01261   eval-logloss:0.04718
[201]   train-logloss:0.01256   eval-logloss:0.04697
[202]   train-logloss:0.01251   eval-logloss:0.04703
[203]   train-logloss:0.01243   eval-logloss:0.04704
[204]   train-logloss:0.01235   eval-logloss:0.04707
[205]   train-logloss:0.01230   eval-logloss:0.04694
[206]   train-logloss:0.01225   eval-logloss:0.04699
[207]   train-logloss:0.01218   eval-logloss:0.04693
[208]   train-logloss:0.01213   eval-logloss:0.04713
[209]   train-logloss:0.01208   eval-logloss:0.04689
[210]   train-logloss:0.01203   eval-logloss:0.04676
[211]   train-logloss:0.01195   eval-logloss:0.04645
[212]   train-logloss:0.01190   eval-logloss:0.04666
[213]   train-logloss:0.01185   eval-logloss:0.04634
[214]   train-logloss:0.01181   eval-logloss:0.04640
[215]   train-logloss:0.01176   eval-logloss:0.04627
[216]   train-logloss:0.01169   eval-logloss:0.04621
[217]   train-logloss:0.01164   eval-logloss:0.04611
[218]   train-logloss:0.01160   eval-logloss:0.04588
[219]   train-logloss:0.01155   eval-logloss:0.04576
[220]   train-logloss:0.01149   eval-logloss:0.04570
[221]   train-logloss:0.01145   eval-logloss:0.04594
[222]   train-logloss:0.01141   eval-logloss:0.04600
[223]   train-logloss:0.01137   eval-logloss:0.04573
[224]   train-logloss:0.01133   eval-logloss:0.04552
[225]   train-logloss:0.01128   eval-logloss:0.04539
[226]   train-logloss:0.01122   eval-logloss:0.04534
[227]   train-logloss:0.01118   eval-logloss:0.04554
[228]   train-logloss:0.01114   eval-logloss:0.04536
[229]   train-logloss:0.01109   eval-logloss:0.04550
[230]   train-logloss:0.01105   eval-logloss:0.04529
[231]   train-logloss:0.01100   eval-logloss:0.04516
[232]   train-logloss:0.01095   eval-logloss:0.04511
[233]   train-logloss:0.01091   eval-logloss:0.04516
[234]   train-logloss:0.01087   eval-logloss:0.04536
[235]   train-logloss:0.01083   eval-logloss:0.04514
[236]   train-logloss:0.01079   eval-logloss:0.04536
[237]   train-logloss:0.01075   eval-logloss:0.04515
[238]   train-logloss:0.01071   eval-logloss:0.04503
[239]   train-logloss:0.01066   eval-logloss:0.04498
[240]   train-logloss:0.01063   eval-logloss:0.04505
[241]   train-logloss:0.01058   eval-logloss:0.04519
[242]   train-logloss:0.01053   eval-logloss:0.04511
[243]   train-logloss:0.01049   eval-logloss:0.04499
[244]   train-logloss:0.01045   eval-logloss:0.04478
[245]   train-logloss:0.01041   eval-logloss:0.04500
[246]   train-logloss:0.01036   eval-logloss:0.04492
[247]   train-logloss:0.01032   eval-logloss:0.04506
[248]   train-logloss:0.01028   eval-logloss:0.04485
[249]   train-logloss:0.01025   eval-logloss:0.04473
[250]   train-logloss:0.01022   eval-logloss:0.04477
[251]   train-logloss:0.01016   eval-logloss:0.04469
[252]   train-logloss:0.01013   eval-logloss:0.04488
[253]   train-logloss:0.01009   eval-logloss:0.04477
[254]   train-logloss:0.01006   eval-logloss:0.04456
[255]   train-logloss:0.01003   eval-logloss:0.04436
[256]   train-logloss:0.01000   eval-logloss:0.04440
[257]   train-logloss:0.00995   eval-logloss:0.04433
[258]   train-logloss:0.00991   eval-logloss:0.04446
[259]   train-logloss:0.00987   eval-logloss:0.04435
[260]   train-logloss:0.00984   eval-logloss:0.04415
[261]   train-logloss:0.00981   eval-logloss:0.04398
[262]   train-logloss:0.00976   eval-logloss:0.04391
[263]   train-logloss:0.00973   eval-logloss:0.04380
[264]   train-logloss:0.00970   eval-logloss:0.04386
[265]   train-logloss:0.00968   eval-logloss:0.04365
[266]   train-logloss:0.00964   eval-logloss:0.04378
[267]   train-logloss:0.00961   eval-logloss:0.04383
[268]   train-logloss:0.00958   eval-logloss:0.04373
[269]   train-logloss:0.00955   eval-logloss:0.04356
[270]   train-logloss:0.00952   eval-logloss:0.04374
[271]   train-logloss:0.00947   eval-logloss:0.04369
[272]   train-logloss:0.00944   eval-logloss:0.04358
[273]   train-logloss:0.00942   eval-logloss:0.04364
[274]   train-logloss:0.00939   eval-logloss:0.04345
[275]   train-logloss:0.00936   eval-logloss:0.04362
[276]   train-logloss:0.00933   eval-logloss:0.04346
[277]   train-logloss:0.00929   eval-logloss:0.04359
[278]   train-logloss:0.00927   eval-logloss:0.04338
[279]   train-logloss:0.00924   eval-logloss:0.04342
[280]   train-logloss:0.00922   eval-logloss:0.04357
[281]   train-logloss:0.00917   eval-logloss:0.04352
[282]   train-logloss:0.00916   eval-logloss:0.04339
[283]   train-logloss:0.00914   eval-logloss:0.04321
[284]   train-logloss:0.00911   eval-logloss:0.04305
[285]   train-logloss:0.00907   eval-logloss:0.04317
[286]   train-logloss:0.00905   eval-logloss:0.04331
[287]   train-logloss:0.00903   eval-logloss:0.04311
[288]   train-logloss:0.00900   eval-logloss:0.04296
[289]   train-logloss:0.00899   eval-logloss:0.04299
[290]   train-logloss:0.00896   eval-logloss:0.04317
[291]   train-logloss:0.00894   eval-logloss:0.04307
[292]   train-logloss:0.00892   eval-logloss:0.04309
[293]   train-logloss:0.00890   eval-logloss:0.04327
[294]   train-logloss:0.00887   eval-logloss:0.04308
[295]   train-logloss:0.00885   eval-logloss:0.04286
[296]   train-logloss:0.00883   eval-logloss:0.04301
[297]   train-logloss:0.00882   eval-logloss:0.04288
[298]   train-logloss:0.00878   eval-logloss:0.04285
[299]   train-logloss:0.00876   eval-logloss:0.04266
[300]   train-logloss:0.00874   eval-logloss:0.04253
[301]   train-logloss:0.00873   eval-logloss:0.04244
[302]   train-logloss:0.00871   eval-logloss:0.04230
[303]   train-logloss:0.00869   eval-logloss:0.04233
[304]   train-logloss:0.00868   eval-logloss:0.04249
[305]   train-logloss:0.00867   eval-logloss:0.04236
[306]   train-logloss:0.00866   eval-logloss:0.04239
[307]   train-logloss:0.00865   eval-logloss:0.04255
[308]   train-logloss:0.00863   eval-logloss:0.04243
[309]   train-logloss:0.00862   eval-logloss:0.04234
[310]   train-logloss:0.00860   eval-logloss:0.04220
[311]   train-logloss:0.00857   eval-logloss:0.04237
[312]   train-logloss:0.00855   eval-logloss:0.04219
[313]   train-logloss:0.00854   eval-logloss:0.04223
[314]   train-logloss:0.00853   eval-logloss:0.04237
[315]   train-logloss:0.00850   eval-logloss:0.04223
[316]   train-logloss:0.00849   eval-logloss:0.04227
[317]   train-logloss:0.00848   eval-logloss:0.04215
[318]   train-logloss:0.00847   eval-logloss:0.04206
[319]   train-logloss:0.00844   eval-logloss:0.04223
[320]   train-logloss:0.00843   eval-logloss:0.04211
[321]   train-logloss:0.00842   eval-logloss:0.04226
[322]   train-logloss:0.00841   eval-logloss:0.04229
[323]   train-logloss:0.00840   eval-logloss:0.04245
[324]   train-logloss:0.00839   eval-logloss:0.04233
[325]   train-logloss:0.00838   eval-logloss:0.04216
[326]   train-logloss:0.00836   eval-logloss:0.04199
[327]   train-logloss:0.00835   eval-logloss:0.04202
[328]   train-logloss:0.00834   eval-logloss:0.04217
[329]   train-logloss:0.00831   eval-logloss:0.04220
[330]   train-logloss:0.00827   eval-logloss:0.04218
[331]   train-logloss:0.00826   eval-logloss:0.04206
[332]   train-logloss:0.00825   eval-logloss:0.04188
[333]   train-logloss:0.00823   eval-logloss:0.04182
[334]   train-logloss:0.00822   eval-logloss:0.04174
[335]   train-logloss:0.00820   eval-logloss:0.04168
[336]   train-logloss:0.00819   eval-logloss:0.04154
[337]   train-logloss:0.00818   eval-logloss:0.04171
[338]   train-logloss:0.00814   eval-logloss:0.04169
[339]   train-logloss:0.00813   eval-logloss:0.04158
[340]   train-logloss:0.00812   eval-logloss:0.04145
[341]   train-logloss:0.00811   eval-logloss:0.04129
[342]   train-logloss:0.00810   eval-logloss:0.04145
[343]   train-logloss:0.00809   eval-logloss:0.04149
[344]   train-logloss:0.00808   eval-logloss:0.04138
[345]   train-logloss:0.00807   eval-logloss:0.04152
[346]   train-logloss:0.00806   eval-logloss:0.04138
[347]   train-logloss:0.00805   eval-logloss:0.04145
[348]   train-logloss:0.00804   eval-logloss:0.04136
[349]   train-logloss:0.00803   eval-logloss:0.04120
[350]   train-logloss:0.00802   eval-logloss:0.04124
[351]   train-logloss:0.00801   eval-logloss:0.04138
[352]   train-logloss:0.00800   eval-logloss:0.04127
[353]   train-logloss:0.00799   eval-logloss:0.04114
[354]   train-logloss:0.00798   eval-logloss:0.04128
[355]   train-logloss:0.00797   eval-logloss:0.04116
[356]   train-logloss:0.00796   eval-logloss:0.04103
[357]   train-logloss:0.00795   eval-logloss:0.04110
[358]   train-logloss:0.00794   eval-logloss:0.04113
[359]   train-logloss:0.00793   eval-logloss:0.04108
[360]   train-logloss:0.00792   eval-logloss:0.04099
[361]   train-logloss:0.00791   eval-logloss:0.04084
[362]   train-logloss:0.00790   eval-logloss:0.04090
[363]   train-logloss:0.00789   eval-logloss:0.04077
[364]   train-logloss:0.00788   eval-logloss:0.04080
[365]   train-logloss:0.00787   eval-logloss:0.04095
[366]   train-logloss:0.00786   eval-logloss:0.04084
[367]   train-logloss:0.00785   eval-logloss:0.04075
[368]   train-logloss:0.00784   eval-logloss:0.04060
[369]   train-logloss:0.00783   eval-logloss:0.04055
[370]   train-logloss:0.00783   eval-logloss:0.04058
[371]   train-logloss:0.00782   eval-logloss:0.04072
[372]   train-logloss:0.00781   eval-logloss:0.04059
[373]   train-logloss:0.00780   eval-logloss:0.04065
[374]   train-logloss:0.00779   eval-logloss:0.04057
[375]   train-logloss:0.00778   eval-logloss:0.04046
[376]   train-logloss:0.00777   eval-logloss:0.04060
[377]   train-logloss:0.00776   eval-logloss:0.04063
[378]   train-logloss:0.00775   eval-logloss:0.04053
[379]   train-logloss:0.00774   eval-logloss:0.04066
[380]   train-logloss:0.00773   eval-logloss:0.04053
[381]   train-logloss:0.00772   eval-logloss:0.04048
[382]   train-logloss:0.00772   eval-logloss:0.04034
[383]   train-logloss:0.00771   eval-logloss:0.04037
[384]   train-logloss:0.00770   eval-logloss:0.04053
[385]   train-logloss:0.00769   eval-logloss:0.04045
[386]   train-logloss:0.00768   eval-logloss:0.04034
[387]   train-logloss:0.00767   eval-logloss:0.04020
[388]   train-logloss:0.00766   eval-logloss:0.04023
[389]   train-logloss:0.00765   eval-logloss:0.04037
[390]   train-logloss:0.00764   eval-logloss:0.04025
[391]   train-logloss:0.00764   eval-logloss:0.04038
[392]   train-logloss:0.00763   eval-logloss:0.04033
[393]   train-logloss:0.00762   eval-logloss:0.04022
[394]   train-logloss:0.00761   eval-logloss:0.04015
[395]   train-logloss:0.00760   eval-logloss:0.04018
[396]   train-logloss:0.00759   eval-logloss:0.04031
[397]   train-logloss:0.00758   eval-logloss:0.04021
[398]   train-logloss:0.00758   eval-logloss:0.04007
[399]   train-logloss:0.00757   eval-logloss:0.04022

pred_probs = xgb_model.predict(dtest)
preds = [1 if x > 0.5 else 0 for x in pred_probs]

• sklearn xgboost

from xgboost import XGBClassifier

evals = [(X_tr, y_tr), (X_val, y_val)]
xgb = XGBClassifier(n_estimators=400, 
                    learning_rate=0.05, 
                    max_depth=3, 
                    early_stopping_rounds=50,
                    eval_metric=['logloss'])
xgb.fit(X_tr, y_tr, eval_set=evals)
preds = xgb.predict(X_test)
pred_probs = xgb.predict_proba(X_test)[:, 1]

[0] validation_0-logloss:0.62424    validation_1-logloss:0.59777
[1] validation_0-logloss:0.58779    validation_1-logloss:0.56156
[2] validation_0-logloss:0.55432    validation_1-logloss:0.53054
[3] validation_0-logloss:0.52351    validation_1-logloss:0.50160
[4] validation_0-logloss:0.49539    validation_1-logloss:0.47522
[5] validation_0-logloss:0.46757    validation_1-logloss:0.44992
[6] validation_0-logloss:0.44383    validation_1-logloss:0.42761
[7] validation_0-logloss:0.42005    validation_1-logloss:0.40613
[8] validation_0-logloss:0.39808    validation_1-logloss:0.38633
[9] validation_0-logloss:0.37870    validation_1-logloss:0.36842
[10]    validation_0-logloss:0.35969    validation_1-logloss:0.35141
[11]    validation_0-logloss:0.34290    validation_1-logloss:0.33598
[12]    validation_0-logloss:0.32758    validation_1-logloss:0.32125
[13]    validation_0-logloss:0.31287    validation_1-logloss:0.30786
[14]    validation_0-logloss:0.29892    validation_1-logloss:0.29789
[15]    validation_0-logloss:0.28558    validation_1-logloss:0.28766
[16]    validation_0-logloss:0.27366    validation_1-logloss:0.27631
[17]    validation_0-logloss:0.26184    validation_1-logloss:0.26744
[18]    validation_0-logloss:0.25113    validation_1-logloss:0.25662
[19]    validation_0-logloss:0.24069    validation_1-logloss:0.24660
[20]    validation_0-logloss:0.23095    validation_1-logloss:0.24040
[21]    validation_0-logloss:0.22133    validation_1-logloss:0.23252
[22]    validation_0-logloss:0.21279    validation_1-logloss:0.22393
[23]    validation_0-logloss:0.20420    validation_1-logloss:0.21701
[24]    validation_0-logloss:0.19629    validation_1-logloss:0.20948
[25]    validation_0-logloss:0.18899    validation_1-logloss:0.20315
[26]    validation_0-logloss:0.18165    validation_1-logloss:0.19747
[27]    validation_0-logloss:0.17512    validation_1-logloss:0.19161
[28]    validation_0-logloss:0.16837    validation_1-logloss:0.18409
[29]    validation_0-logloss:0.16213    validation_1-logloss:0.17747
[30]    validation_0-logloss:0.15644    validation_1-logloss:0.17195
[31]    validation_0-logloss:0.15069    validation_1-logloss:0.16564
[32]    validation_0-logloss:0.14545    validation_1-logloss:0.16122
[33]    validation_0-logloss:0.14039    validation_1-logloss:0.15579
[34]    validation_0-logloss:0.13553    validation_1-logloss:0.15171
[35]    validation_0-logloss:0.13076    validation_1-logloss:0.14631
[36]    validation_0-logloss:0.12661    validation_1-logloss:0.14321
[37]    validation_0-logloss:0.12243    validation_1-logloss:0.13868
[38]    validation_0-logloss:0.11868    validation_1-logloss:0.13596
[39]    validation_0-logloss:0.11490    validation_1-logloss:0.13172
[40]    validation_0-logloss:0.11126    validation_1-logloss:0.12735
[41]    validation_0-logloss:0.10781    validation_1-logloss:0.12351
[42]    validation_0-logloss:0.10450    validation_1-logloss:0.11957
[43]    validation_0-logloss:0.10131    validation_1-logloss:0.11603
[44]    validation_0-logloss:0.09846    validation_1-logloss:0.11396
[45]    validation_0-logloss:0.09545    validation_1-logloss:0.11227
[46]    validation_0-logloss:0.09284    validation_1-logloss:0.11015
[47]    validation_0-logloss:0.09020    validation_1-logloss:0.10715
[48]    validation_0-logloss:0.08763    validation_1-logloss:0.10404
[49]    validation_0-logloss:0.08535    validation_1-logloss:0.10262
[50]    validation_0-logloss:0.08302    validation_1-logloss:0.10000
[51]    validation_0-logloss:0.08075    validation_1-logloss:0.09821
[52]    validation_0-logloss:0.07862    validation_1-logloss:0.09605
[53]    validation_0-logloss:0.07660    validation_1-logloss:0.09381
[54]    validation_0-logloss:0.07470    validation_1-logloss:0.09237
[55]    validation_0-logloss:0.07281    validation_1-logloss:0.09051
[56]    validation_0-logloss:0.07107    validation_1-logloss:0.08923
[57]    validation_0-logloss:0.06933    validation_1-logloss:0.08746
[58]    validation_0-logloss:0.06769    validation_1-logloss:0.08634
[59]    validation_0-logloss:0.06609    validation_1-logloss:0.08479
[60]    validation_0-logloss:0.06459    validation_1-logloss:0.08378
[61]    validation_0-logloss:0.06299    validation_1-logloss:0.08323
[62]    validation_0-logloss:0.06160    validation_1-logloss:0.08241
[63]    validation_0-logloss:0.06027    validation_1-logloss:0.08159
[64]    validation_0-logloss:0.05892    validation_1-logloss:0.08046
[65]    validation_0-logloss:0.05770    validation_1-logloss:0.07975
[66]    validation_0-logloss:0.05651    validation_1-logloss:0.07913
[67]    validation_0-logloss:0.05529    validation_1-logloss:0.07875
[68]    validation_0-logloss:0.05421    validation_1-logloss:0.07815
[69]    validation_0-logloss:0.05280    validation_1-logloss:0.07674
[70]    validation_0-logloss:0.05177    validation_1-logloss:0.07657
[71]    validation_0-logloss:0.05074    validation_1-logloss:0.07522
[72]    validation_0-logloss:0.04977    validation_1-logloss:0.07474
[73]    validation_0-logloss:0.04878    validation_1-logloss:0.07455
[74]    validation_0-logloss:0.04788    validation_1-logloss:0.07450
[75]    validation_0-logloss:0.04706    validation_1-logloss:0.07308
[76]    validation_0-logloss:0.04608    validation_1-logloss:0.07206
[77]    validation_0-logloss:0.04528    validation_1-logloss:0.07178
[78]    validation_0-logloss:0.04443    validation_1-logloss:0.07170
[79]    validation_0-logloss:0.04341    validation_1-logloss:0.07047
[80]    validation_0-logloss:0.04251    validation_1-logloss:0.06994
[81]    validation_0-logloss:0.04184    validation_1-logloss:0.06921
[82]    validation_0-logloss:0.04118    validation_1-logloss:0.06898
[83]    validation_0-logloss:0.04037    validation_1-logloss:0.06854
[84]    validation_0-logloss:0.03966    validation_1-logloss:0.06873
[85]    validation_0-logloss:0.03903    validation_1-logloss:0.06884
[86]    validation_0-logloss:0.03844    validation_1-logloss:0.06833
[87]    validation_0-logloss:0.03779    validation_1-logloss:0.06856
[88]    validation_0-logloss:0.03722    validation_1-logloss:0.06870
[89]    validation_0-logloss:0.03648    validation_1-logloss:0.06859
[90]    validation_0-logloss:0.03580    validation_1-logloss:0.06858
[91]    validation_0-logloss:0.03525    validation_1-logloss:0.06808
[92]    validation_0-logloss:0.03472    validation_1-logloss:0.06694
[93]    validation_0-logloss:0.03410    validation_1-logloss:0.06672
[94]    validation_0-logloss:0.03361    validation_1-logloss:0.06564
[95]    validation_0-logloss:0.03307    validation_1-logloss:0.06545
[96]    validation_0-logloss:0.03257    validation_1-logloss:0.06536
[97]    validation_0-logloss:0.03205    validation_1-logloss:0.06465
[98]    validation_0-logloss:0.03150    validation_1-logloss:0.06441
[99]    validation_0-logloss:0.03105    validation_1-logloss:0.06357
[100]   validation_0-logloss:0.03057    validation_1-logloss:0.06387
[101]   validation_0-logloss:0.03006    validation_1-logloss:0.06370
[102]   validation_0-logloss:0.02967    validation_1-logloss:0.06274
[103]   validation_0-logloss:0.02925    validation_1-logloss:0.06244
[104]   validation_0-logloss:0.02878    validation_1-logloss:0.06232
[105]   validation_0-logloss:0.02836    validation_1-logloss:0.06265
[106]   validation_0-logloss:0.02799    validation_1-logloss:0.06195
[107]   validation_0-logloss:0.02764    validation_1-logloss:0.06176
[108]   validation_0-logloss:0.02731    validation_1-logloss:0.06152
[109]   validation_0-logloss:0.02690    validation_1-logloss:0.06142
[110]   validation_0-logloss:0.02657    validation_1-logloss:0.06076
[111]   validation_0-logloss:0.02626    validation_1-logloss:0.06059
[112]   validation_0-logloss:0.02589    validation_1-logloss:0.06010
[113]   validation_0-logloss:0.02553    validation_1-logloss:0.06003
[114]   validation_0-logloss:0.02519    validation_1-logloss:0.05933
[115]   validation_0-logloss:0.02480    validation_1-logloss:0.05840
[116]   validation_0-logloss:0.02451    validation_1-logloss:0.05855
[117]   validation_0-logloss:0.02414    validation_1-logloss:0.05766
[118]   validation_0-logloss:0.02382    validation_1-logloss:0.05761
[119]   validation_0-logloss:0.02348    validation_1-logloss:0.05676
[120]   validation_0-logloss:0.02322    validation_1-logloss:0.05692
[121]   validation_0-logloss:0.02290    validation_1-logloss:0.05610
[122]   validation_0-logloss:0.02265    validation_1-logloss:0.05592
[123]   validation_0-logloss:0.02243    validation_1-logloss:0.05622
[124]   validation_0-logloss:0.02213    validation_1-logloss:0.05542
[125]   validation_0-logloss:0.02189    validation_1-logloss:0.05473
[126]   validation_0-logloss:0.02168    validation_1-logloss:0.05504
[127]   validation_0-logloss:0.02140    validation_1-logloss:0.05427
[128]   validation_0-logloss:0.02118    validation_1-logloss:0.05401
[129]   validation_0-logloss:0.02093    validation_1-logloss:0.05346
[130]   validation_0-logloss:0.02075    validation_1-logloss:0.05376
[131]   validation_0-logloss:0.02049    validation_1-logloss:0.05298
[132]   validation_0-logloss:0.02026    validation_1-logloss:0.05259
[133]   validation_0-logloss:0.02003    validation_1-logloss:0.05251
[134]   validation_0-logloss:0.01981    validation_1-logloss:0.05233
[135]   validation_0-logloss:0.01962    validation_1-logloss:0.05210
[136]   validation_0-logloss:0.01945    validation_1-logloss:0.05208
[137]   validation_0-logloss:0.01925    validation_1-logloss:0.05146
[138]   validation_0-logloss:0.01909    validation_1-logloss:0.05080
[139]   validation_0-logloss:0.01891    validation_1-logloss:0.05059
[140]   validation_0-logloss:0.01877    validation_1-logloss:0.05075
[141]   validation_0-logloss:0.01862    validation_1-logloss:0.05036
[142]   validation_0-logloss:0.01846    validation_1-logloss:0.05019
[143]   validation_0-logloss:0.01826    validation_1-logloss:0.04959
[144]   validation_0-logloss:0.01807    validation_1-logloss:0.04956
[145]   validation_0-logloss:0.01791    validation_1-logloss:0.04940
[146]   validation_0-logloss:0.01776    validation_1-logloss:0.04897
[147]   validation_0-logloss:0.01761    validation_1-logloss:0.04879
[148]   validation_0-logloss:0.01743    validation_1-logloss:0.04864
[149]   validation_0-logloss:0.01730    validation_1-logloss:0.04864
[150]   validation_0-logloss:0.01718    validation_1-logloss:0.04833
[151]   validation_0-logloss:0.01703    validation_1-logloss:0.04792
[152]   validation_0-logloss:0.01690    validation_1-logloss:0.04819
[153]   validation_0-logloss:0.01677    validation_1-logloss:0.04801
[154]   validation_0-logloss:0.01662    validation_1-logloss:0.04794
[155]   validation_0-logloss:0.01646    validation_1-logloss:0.04764
[156]   validation_0-logloss:0.01631    validation_1-logloss:0.04754
[157]   validation_0-logloss:0.01619    validation_1-logloss:0.04778
[158]   validation_0-logloss:0.01609    validation_1-logloss:0.04780
[159]   validation_0-logloss:0.01598    validation_1-logloss:0.04805
[160]   validation_0-logloss:0.01584    validation_1-logloss:0.04791
[161]   validation_0-logloss:0.01575    validation_1-logloss:0.04757
[162]   validation_0-logloss:0.01565    validation_1-logloss:0.04760
[163]   validation_0-logloss:0.01555    validation_1-logloss:0.04782
[164]   validation_0-logloss:0.01547    validation_1-logloss:0.04758
[165]   validation_0-logloss:0.01532    validation_1-logloss:0.04775
[166]   validation_0-logloss:0.01522    validation_1-logloss:0.04799
[167]   validation_0-logloss:0.01514    validation_1-logloss:0.04821
[168]   validation_0-logloss:0.01507    validation_1-logloss:0.04827
[169]   validation_0-logloss:0.01499    validation_1-logloss:0.04803
[170]   validation_0-logloss:0.01485    validation_1-logloss:0.04819
[171]   validation_0-logloss:0.01475    validation_1-logloss:0.04788
[172]   validation_0-logloss:0.01465    validation_1-logloss:0.04811
[173]   validation_0-logloss:0.01456    validation_1-logloss:0.04763
[174]   validation_0-logloss:0.01445    validation_1-logloss:0.04770
[175]   validation_0-logloss:0.01432    validation_1-logloss:0.04777
[176]   validation_0-logloss:0.01425    validation_1-logloss:0.04780
[177]   validation_0-logloss:0.01418    validation_1-logloss:0.04809
[178]   validation_0-logloss:0.01411    validation_1-logloss:0.04782
[179]   validation_0-logloss:0.01405    validation_1-logloss:0.04787
[180]   validation_0-logloss:0.01394    validation_1-logloss:0.04793
[181]   validation_0-logloss:0.01387    validation_1-logloss:0.04771
[182]   validation_0-logloss:0.01377    validation_1-logloss:0.04778
[183]   validation_0-logloss:0.01371    validation_1-logloss:0.04738
[184]   validation_0-logloss:0.01363    validation_1-logloss:0.04761
[185]   validation_0-logloss:0.01353    validation_1-logloss:0.04763
[186]   validation_0-logloss:0.01346    validation_1-logloss:0.04737
[187]   validation_0-logloss:0.01340    validation_1-logloss:0.04761
[188]   validation_0-logloss:0.01335    validation_1-logloss:0.04761
[189]   validation_0-logloss:0.01329    validation_1-logloss:0.04757
[190]   validation_0-logloss:0.01323    validation_1-logloss:0.04718
[191]   validation_0-logloss:0.01318    validation_1-logloss:0.04718
[192]   validation_0-logloss:0.01312    validation_1-logloss:0.04715
[193]   validation_0-logloss:0.01306    validation_1-logloss:0.04690
[194]   validation_0-logloss:0.01300    validation_1-logloss:0.04713
[195]   validation_0-logloss:0.01295    validation_1-logloss:0.04713
[196]   validation_0-logloss:0.01287    validation_1-logloss:0.04715
[197]   validation_0-logloss:0.01282    validation_1-logloss:0.04691
[198]   validation_0-logloss:0.01272    validation_1-logloss:0.04719
[199]   validation_0-logloss:0.01266    validation_1-logloss:0.04742
[200]   validation_0-logloss:0.01261    validation_1-logloss:0.04718
[201]   validation_0-logloss:0.01256    validation_1-logloss:0.04697
[202]   validation_0-logloss:0.01251    validation_1-logloss:0.04703
[203]   validation_0-logloss:0.01243    validation_1-logloss:0.04704
[204]   validation_0-logloss:0.01235    validation_1-logloss:0.04707
[205]   validation_0-logloss:0.01230    validation_1-logloss:0.04694
[206]   validation_0-logloss:0.01225    validation_1-logloss:0.04699
[207]   validation_0-logloss:0.01218    validation_1-logloss:0.04693
[208]   validation_0-logloss:0.01213    validation_1-logloss:0.04713
[209]   validation_0-logloss:0.01208    validation_1-logloss:0.04689
[210]   validation_0-logloss:0.01203    validation_1-logloss:0.04676
[211]   validation_0-logloss:0.01195    validation_1-logloss:0.04645
[212]   validation_0-logloss:0.01190    validation_1-logloss:0.04666
[213]   validation_0-logloss:0.01185    validation_1-logloss:0.04634
[214]   validation_0-logloss:0.01181    validation_1-logloss:0.04640
[215]   validation_0-logloss:0.01176    validation_1-logloss:0.04627
[216]   validation_0-logloss:0.01169    validation_1-logloss:0.04621
[217]   validation_0-logloss:0.01164    validation_1-logloss:0.04611
[218]   validation_0-logloss:0.01160    validation_1-logloss:0.04588
[219]   validation_0-logloss:0.01155    validation_1-logloss:0.04576
[220]   validation_0-logloss:0.01149    validation_1-logloss:0.04570
[221]   validation_0-logloss:0.01145    validation_1-logloss:0.04594
[222]   validation_0-logloss:0.01141    validation_1-logloss:0.04600
[223]   validation_0-logloss:0.01137    validation_1-logloss:0.04573
[224]   validation_0-logloss:0.01133    validation_1-logloss:0.04552
[225]   validation_0-logloss:0.01128    validation_1-logloss:0.04539
[226]   validation_0-logloss:0.01122    validation_1-logloss:0.04534
[227]   validation_0-logloss:0.01118    validation_1-logloss:0.04554
[228]   validation_0-logloss:0.01114    validation_1-logloss:0.04536
[229]   validation_0-logloss:0.01109    validation_1-logloss:0.04550
[230]   validation_0-logloss:0.01105    validation_1-logloss:0.04529
[231]   validation_0-logloss:0.01100    validation_1-logloss:0.04516
[232]   validation_0-logloss:0.01095    validation_1-logloss:0.04511
[233]   validation_0-logloss:0.01091    validation_1-logloss:0.04516
[234]   validation_0-logloss:0.01087    validation_1-logloss:0.04536
[235]   validation_0-logloss:0.01083    validation_1-logloss:0.04514
[236]   validation_0-logloss:0.01079    validation_1-logloss:0.04536
[237]   validation_0-logloss:0.01075    validation_1-logloss:0.04515
[238]   validation_0-logloss:0.01071    validation_1-logloss:0.04503
[239]   validation_0-logloss:0.01066    validation_1-logloss:0.04498
[240]   validation_0-logloss:0.01063    validation_1-logloss:0.04505
[241]   validation_0-logloss:0.01058    validation_1-logloss:0.04519
[242]   validation_0-logloss:0.01053    validation_1-logloss:0.04511
[243]   validation_0-logloss:0.01049    validation_1-logloss:0.04499
[244]   validation_0-logloss:0.01045    validation_1-logloss:0.04478
[245]   validation_0-logloss:0.01041    validation_1-logloss:0.04500
[246]   validation_0-logloss:0.01036    validation_1-logloss:0.04492
[247]   validation_0-logloss:0.01032    validation_1-logloss:0.04506
[248]   validation_0-logloss:0.01028    validation_1-logloss:0.04485
[249]   validation_0-logloss:0.01025    validation_1-logloss:0.04473
[250]   validation_0-logloss:0.01022    validation_1-logloss:0.04477
[251]   validation_0-logloss:0.01016    validation_1-logloss:0.04469
[252]   validation_0-logloss:0.01013    validation_1-logloss:0.04488
[253]   validation_0-logloss:0.01009    validation_1-logloss:0.04477
[254]   validation_0-logloss:0.01006    validation_1-logloss:0.04456
[255]   validation_0-logloss:0.01003    validation_1-logloss:0.04436
[256]   validation_0-logloss:0.01000    validation_1-logloss:0.04440
[257]   validation_0-logloss:0.00995    validation_1-logloss:0.04433
[258]   validation_0-logloss:0.00991    validation_1-logloss:0.04446
[259]   validation_0-logloss:0.00987    validation_1-logloss:0.04435
[260]   validation_0-logloss:0.00984    validation_1-logloss:0.04415
[261]   validation_0-logloss:0.00981    validation_1-logloss:0.04398
[262]   validation_0-logloss:0.00976    validation_1-logloss:0.04391
[263]   validation_0-logloss:0.00973    validation_1-logloss:0.04380
[264]   validation_0-logloss:0.00970    validation_1-logloss:0.04386
[265]   validation_0-logloss:0.00968    validation_1-logloss:0.04365
[266]   validation_0-logloss:0.00964    validation_1-logloss:0.04378
[267]   validation_0-logloss:0.00961    validation_1-logloss:0.04383
[268]   validation_0-logloss:0.00958    validation_1-logloss:0.04373
[269]   validation_0-logloss:0.00955    validation_1-logloss:0.04356
[270]   validation_0-logloss:0.00952    validation_1-logloss:0.04374
[271]   validation_0-logloss:0.00947    validation_1-logloss:0.04369
[272]   validation_0-logloss:0.00944    validation_1-logloss:0.04358
[273]   validation_0-logloss:0.00942    validation_1-logloss:0.04364
[274]   validation_0-logloss:0.00939    validation_1-logloss:0.04345
[275]   validation_0-logloss:0.00936    validation_1-logloss:0.04362
[276]   validation_0-logloss:0.00933    validation_1-logloss:0.04346
[277]   validation_0-logloss:0.00929    validation_1-logloss:0.04359
[278]   validation_0-logloss:0.00927    validation_1-logloss:0.04338
[279]   validation_0-logloss:0.00924    validation_1-logloss:0.04342
[280]   validation_0-logloss:0.00922    validation_1-logloss:0.04357
[281]   validation_0-logloss:0.00917    validation_1-logloss:0.04352
[282]   validation_0-logloss:0.00916    validation_1-logloss:0.04339
[283]   validation_0-logloss:0.00914    validation_1-logloss:0.04321
[284]   validation_0-logloss:0.00911    validation_1-logloss:0.04305
[285]   validation_0-logloss:0.00907    validation_1-logloss:0.04317
[286]   validation_0-logloss:0.00905    validation_1-logloss:0.04331
[287]   validation_0-logloss:0.00903    validation_1-logloss:0.04311
[288]   validation_0-logloss:0.00900    validation_1-logloss:0.04296
[289]   validation_0-logloss:0.00899    validation_1-logloss:0.04299
[290]   validation_0-logloss:0.00896    validation_1-logloss:0.04317
[291]   validation_0-logloss:0.00894    validation_1-logloss:0.04307
[292]   validation_0-logloss:0.00892    validation_1-logloss:0.04309
[293]   validation_0-logloss:0.00890    validation_1-logloss:0.04327
[294]   validation_0-logloss:0.00887    validation_1-logloss:0.04308
[295]   validation_0-logloss:0.00885    validation_1-logloss:0.04286
[296]   validation_0-logloss:0.00883    validation_1-logloss:0.04301
[297]   validation_0-logloss:0.00882    validation_1-logloss:0.04288
[298]   validation_0-logloss:0.00878    validation_1-logloss:0.04285
[299]   validation_0-logloss:0.00876    validation_1-logloss:0.04266
[300]   validation_0-logloss:0.00874    validation_1-logloss:0.04253
[301]   validation_0-logloss:0.00873    validation_1-logloss:0.04244
[302]   validation_0-logloss:0.00871    validation_1-logloss:0.04230
[303]   validation_0-logloss:0.00869    validation_1-logloss:0.04233
[304]   validation_0-logloss:0.00868    validation_1-logloss:0.04249
[305]   validation_0-logloss:0.00867    validation_1-logloss:0.04236
[306]   validation_0-logloss:0.00866    validation_1-logloss:0.04239
[307]   validation_0-logloss:0.00865    validation_1-logloss:0.04255
[308]   validation_0-logloss:0.00863    validation_1-logloss:0.04243
[309]   validation_0-logloss:0.00862    validation_1-logloss:0.04234
[310]   validation_0-logloss:0.00860    validation_1-logloss:0.04220
[311]   validation_0-logloss:0.00857    validation_1-logloss:0.04237
[312]   validation_0-logloss:0.00855    validation_1-logloss:0.04219
[313]   validation_0-logloss:0.00854    validation_1-logloss:0.04223
[314]   validation_0-logloss:0.00853    validation_1-logloss:0.04237
[315]   validation_0-logloss:0.00850    validation_1-logloss:0.04223
[316]   validation_0-logloss:0.00849    validation_1-logloss:0.04227
[317]   validation_0-logloss:0.00848    validation_1-logloss:0.04215
[318]   validation_0-logloss:0.00847    validation_1-logloss:0.04206
[319]   validation_0-logloss:0.00844    validation_1-logloss:0.04223
[320]   validation_0-logloss:0.00843    validation_1-logloss:0.04211
[321]   validation_0-logloss:0.00842    validation_1-logloss:0.04226
[322]   validation_0-logloss:0.00841    validation_1-logloss:0.04229
[323]   validation_0-logloss:0.00840    validation_1-logloss:0.04245
[324]   validation_0-logloss:0.00839    validation_1-logloss:0.04233
[325]   validation_0-logloss:0.00838    validation_1-logloss:0.04216
[326]   validation_0-logloss:0.00836    validation_1-logloss:0.04199
[327]   validation_0-logloss:0.00835    validation_1-logloss:0.04202
[328]   validation_0-logloss:0.00834    validation_1-logloss:0.04217
[329]   validation_0-logloss:0.00831    validation_1-logloss:0.04220
[330]   validation_0-logloss:0.00827    validation_1-logloss:0.04218
[331]   validation_0-logloss:0.00826    validation_1-logloss:0.04206
[332]   validation_0-logloss:0.00825    validation_1-logloss:0.04188
[333]   validation_0-logloss:0.00823    validation_1-logloss:0.04182
[334]   validation_0-logloss:0.00822    validation_1-logloss:0.04174
[335]   validation_0-logloss:0.00820    validation_1-logloss:0.04168
[336]   validation_0-logloss:0.00819    validation_1-logloss:0.04154
[337]   validation_0-logloss:0.00818    validation_1-logloss:0.04171
[338]   validation_0-logloss:0.00814    validation_1-logloss:0.04169
[339]   validation_0-logloss:0.00813    validation_1-logloss:0.04158
[340]   validation_0-logloss:0.00812    validation_1-logloss:0.04145
[341]   validation_0-logloss:0.00811    validation_1-logloss:0.04129
[342]   validation_0-logloss:0.00810    validation_1-logloss:0.04145
[343]   validation_0-logloss:0.00809    validation_1-logloss:0.04149
[344]   validation_0-logloss:0.00808    validation_1-logloss:0.04138
[345]   validation_0-logloss:0.00807    validation_1-logloss:0.04152
[346]   validation_0-logloss:0.00806    validation_1-logloss:0.04138
[347]   validation_0-logloss:0.00805    validation_1-logloss:0.04145
[348]   validation_0-logloss:0.00804    validation_1-logloss:0.04136
[349]   validation_0-logloss:0.00803    validation_1-logloss:0.04120
[350]   validation_0-logloss:0.00802    validation_1-logloss:0.04124
[351]   validation_0-logloss:0.00801    validation_1-logloss:0.04138
[352]   validation_0-logloss:0.00800    validation_1-logloss:0.04127
[353]   validation_0-logloss:0.00799    validation_1-logloss:0.04114
[354]   validation_0-logloss:0.00798    validation_1-logloss:0.04128
[355]   validation_0-logloss:0.00797    validation_1-logloss:0.04116
[356]   validation_0-logloss:0.00796    validation_1-logloss:0.04103
[357]   validation_0-logloss:0.00795    validation_1-logloss:0.04110
[358]   validation_0-logloss:0.00794    validation_1-logloss:0.04113
[359]   validation_0-logloss:0.00793    validation_1-logloss:0.04108
[360]   validation_0-logloss:0.00792    validation_1-logloss:0.04099
[361]   validation_0-logloss:0.00791    validation_1-logloss:0.04084
[362]   validation_0-logloss:0.00790    validation_1-logloss:0.04090
[363]   validation_0-logloss:0.00789    validation_1-logloss:0.04077
[364]   validation_0-logloss:0.00788    validation_1-logloss:0.04080
[365]   validation_0-logloss:0.00787    validation_1-logloss:0.04095
[366]   validation_0-logloss:0.00786    validation_1-logloss:0.04084
[367]   validation_0-logloss:0.00785    validation_1-logloss:0.04075
[368]   validation_0-logloss:0.00784    validation_1-logloss:0.04060
[369]   validation_0-logloss:0.00783    validation_1-logloss:0.04055
[370]   validation_0-logloss:0.00783    validation_1-logloss:0.04058
[371]   validation_0-logloss:0.00782    validation_1-logloss:0.04072
[372]   validation_0-logloss:0.00781    validation_1-logloss:0.04059
[373]   validation_0-logloss:0.00780    validation_1-logloss:0.04065
[374]   validation_0-logloss:0.00779    validation_1-logloss:0.04057
[375]   validation_0-logloss:0.00778    validation_1-logloss:0.04046
[376]   validation_0-logloss:0.00777    validation_1-logloss:0.04060
[377]   validation_0-logloss:0.00776    validation_1-logloss:0.04063
[378]   validation_0-logloss:0.00775    validation_1-logloss:0.04053
[379]   validation_0-logloss:0.00774    validation_1-logloss:0.04066
[380]   validation_0-logloss:0.00773    validation_1-logloss:0.04053
[381]   validation_0-logloss:0.00772    validation_1-logloss:0.04048
[382]   validation_0-logloss:0.00772    validation_1-logloss:0.04034
[383]   validation_0-logloss:0.00771    validation_1-logloss:0.04037
[384]   validation_0-logloss:0.00770    validation_1-logloss:0.04053
[385]   validation_0-logloss:0.00769    validation_1-logloss:0.04045
[386]   validation_0-logloss:0.00768    validation_1-logloss:0.04034
[387]   validation_0-logloss:0.00767    validation_1-logloss:0.04020
[388]   validation_0-logloss:0.00766    validation_1-logloss:0.04023
[389]   validation_0-logloss:0.00765    validation_1-logloss:0.04037
[390]   validation_0-logloss:0.00764    validation_1-logloss:0.04025
[391]   validation_0-logloss:0.00764    validation_1-logloss:0.04038
[392]   validation_0-logloss:0.00763    validation_1-logloss:0.04033
[393]   validation_0-logloss:0.00762    validation_1-logloss:0.04022
[394]   validation_0-logloss:0.00761    validation_1-logloss:0.04015
[395]   validation_0-logloss:0.00760    validation_1-logloss:0.04018
[396]   validation_0-logloss:0.00759    validation_1-logloss:0.04031
[397]   validation_0-logloss:0.00758    validation_1-logloss:0.04021
[398]   validation_0-logloss:0.00758    validation_1-logloss:0.04007
[399]   validation_0-logloss:0.00757    validation_1-logloss:0.04022

LightGBM

• 성능은 xgboost랑 별로 차이가 없음.

• 1만건 이하의 데이터 세트에 대해 과적합이 발생할 가능성이 높다.

• one hot 인코딩 필요 없음

• python lightgbm

from lightgbm import LGBMClassifier, early_stopping, plot_importance
import matplotlib.pyplot as plt

lgbm = LGBMClassifier(n_estimators=400, learning_rate=0.05)
evals = [(X_tr, y_tr), (X_val, y_val)]
lgbm.fit(X_tr, y_tr, 
         callbacks = [early_stopping(stopping_rounds = 50)],
         eval_metric='logloss', 
         eval_set=evals)
preds = lgbm.predict(X_test)
pred_proba = lgbm.predict_proba(X_test)[:, 1]

plot_importance(lgbm)
plt.show()

[LightGBM] [Info] Number of positive: 253, number of negative: 156
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002472 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 4092
[LightGBM] [Info] Number of data points in the train set: 409, number of used features: 30
[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.618582 -> initscore=0.483533
[LightGBM] [Info] Start training from score 0.483533
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
Training until validation scores don't improve for 50 rounds
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
Early stopping, best iteration is:
[291]   training's binary_logloss: 2.39157e-05  valid_1's binary_logloss: 0.00285194

stacking

from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression

X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, test_size=0.2)

knn_clf = KNeighborsClassifier(n_neighbors=4)
rf_clf = RandomForestClassifier(n_estimators=100)
dt_clf = DecisionTreeClassifier()
ada_clf = AdaBoostClassifier(n_estimators=100)

lr_final = LogisticRegression()

knn_clf.fit(X_train, y_train)
rf_clf.fit(X_train, y_train)
dt_clf.fit(X_train, y_train)
ada_clf.fit(X_train, y_train)

knn_pred = knn_clf.predict(X_test)
rf_pred = rf_clf.predict(X_test)
dt_pred = dt_clf.predict(X_test)
ada_pred = ada_clf.predict(X_test)

pred = np.array([knn_pred, rf_pred, dt_pred, ada_pred])
pred = np.transpose(pred)

lr_final.fit(pred, y_test)
final = lr_final.predict(pred)
print(f'{accuracy_score(y_test, final):.3f}')

0.965

• test 셋으로 훈련을 하고 있는 부분이 문제 → cv 세트로 해야함

CV 세트 기반 stacking

from sklearn.model_selection import KFold
from sklearn.metrics import mean_absolute_error

def get_stacking_base_datasets(model, X_train_n, y_train_n, X_test_n, n_folds):
    kf = KFold(n_splits=n_folds, shuffle=False)
    train_fold_pred = np.zeros((X_train_n.shape[0], 1))
    test_pred = np.zeros((X_test_n.shape[0], n_folds))
    for folder_counter, (train_index, valid_index) in enumerate(kf.split(X_train_n)):
        X_tr = X_train_n[train_index]
        y_tr = y_train_n[train_index]
        X_te = X_train_n[valid_index]

        model.fit(X_tr, y_tr)
        train_fold_pred[valid_index, :] = model.predict(X_te).reshape(-1, 1)
        test_pred[:, folder_counter] = model.predict(X_test_n)

    test_pred_mean = np.mean(test_pred, axis=1).reshape(-1, 1)

    return train_fold_pred, test_pred_mean

knn_train, knn_test = get_stacking_base_datasets(knn_clf, X_train, y_train, X_test, 7)
rf_train, rf_test = get_stacking_base_datasets(rf_clf, X_train, y_train, X_test, 7)
dt_train, dt_test = get_stacking_base_datasets(dt_clf, X_train, y_train, X_test, 7)
ada_train, ada_test = get_stacking_base_datasets(ada_clf, X_train, y_train, X_test, 7)

Stack_final_X_train = np.concatenate((knn_train, rf_train, dt_train, ada_train), axis=1)
Stack_final_X_test = np.concatenate((knn_test, rf_test, dt_test, ada_test), axis=1)

lr_final.fit(Stack_final_X_train, y_train)
stack_final = lr_final.predict(Stack_final_X_test)

print(f'{accuracy_score(y_test, stack_final):.3f}')

0.965

Baysian Optimization

• Grid search로는 시간이 너무 오래 걸리는 경우

• 목표 함수: 하이퍼파라미터 입력 n개에 대한 모델 성능 출력 1개의 모델

• Surrogate model: 목표 함수에 대한 예상 모델. 사전확률 분포에서 최적해 나감.

• acquisition function: 불확실성이 가장 큰 point를 다음 관측 데이터로 결정.

from hyperopt import hp, fmin, tpe, Trials, STATUS_OK

search_space = {'x': hp.quniform('x', -10, 10, 1),
                'y': hp.quniform('y', -15, 15, 1)}
def objective_func(search_space):
    x = search_space['x']
    y = search_space['y']

    return x ** 2 - 20 * y

trial_val = Trials()
best = fmin(fn=objective_func,
            space=search_space,
            algo=tpe.suggest,
            max_evals=20,
            trials=trial_val)
best

  0%|          | 0/20 [00:00<?, ?trial/s, best loss=?]100%|██████████| 20/20 [00:00<00:00, 1720.92trial/s, best loss: -224.0]

{'x': -4.0, 'y': 12.0}

XGBoost 하이퍼파라미터 최적화

dataset = load_breast_cancer()

X_train, X_test, y_train, y_test = train_test_split(dataset.data, dataset.target, test_size=0.2)
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.1)

xgb_search_space = {
    'max_depth': hp.quniform('max_depth', 5, 20, 1),
    'min_child_weight': hp.quniform('min_child_weight', 1, 2, 1),
    'learning_rate': hp.uniform('learning_rate', 0.01, 0.2),
    'colsample_bytree': hp.uniform('colsample_bytree', 0.5, 1)
}
# hp.choice('tree_criterion', ['gini', 'entropy']) 이런식으로도 가능

from sklearn.model_selection import cross_val_score

def objective_func(search_space):
    xgb_clf = XGBClassifier(n_estimators=100, 
                            max_depth=int(search_space['max_depth']),
                            min_child_weight=int(search_space['min_child_weight']),
                            learning_rate=search_space['learning_rate'],
                            colsample_bytree=search_space['colsample_bytree'],
                            eval_metric='logloss')
    accuracy = cross_val_score(xgb_clf, X_train, y_train, scoring='accuracy', cv=3)
    return {'loss': -1 * np.mean(accuracy), 'status': STATUS_OK}

trial_val = Trials()
best = fmin(fn=objective_func,
            space=xgb_search_space,
            algo=tpe.suggest,
            max_evals=50,
            trials=trial_val)
best

  0%|          | 0/50 [00:00<?, ?trial/s, best loss=?]  2%|▏         | 1/50 [00:00<00:07,  6.51trial/s, best loss: -0.9560241663762055]  4%|▍         | 2/50 [00:00<00:06,  7.84trial/s, best loss: -0.9560241663762055]  6%|▌         | 3/50 [00:00<00:06,  7.20trial/s, best loss: -0.9560241663762055]  8%|▊         | 4/50 [00:00<00:06,  7.19trial/s, best loss: -0.9560241663762055] 10%|█         | 5/50 [00:00<00:06,  6.92trial/s, best loss: -0.9560241663762055] 12%|█▏        | 6/50 [00:00<00:07,  5.84trial/s, best loss: -0.9560241663762055] 14%|█▍        | 7/50 [00:01<00:07,  5.90trial/s, best loss: -0.9560241663762055] 16%|█▌        | 8/50 [00:01<00:06,  6.59trial/s, best loss: -0.9603956082258627] 18%|█▊        | 9/50 [00:01<00:06,  6.20trial/s, best loss: -0.9603956082258627] 20%|██        | 10/50 [00:01<00:06,  6.43trial/s, best loss: -0.9626031137446264] 22%|██▏       | 11/50 [00:01<00:05,  6.50trial/s, best loss: -0.9626031137446264] 24%|██▍       | 12/50 [00:01<00:05,  6.64trial/s, best loss: -0.9626031137446264] 26%|██▌       | 13/50 [00:01<00:05,  7.00trial/s, best loss: -0.9626031137446264] 28%|██▊       | 14/50 [00:02<00:05,  7.00trial/s, best loss: -0.9626031137446264] 30%|███       | 15/50 [00:02<00:04,  7.28trial/s, best loss: -0.9626031137446264] 32%|███▏      | 16/50 [00:02<00:05,  6.69trial/s, best loss: -0.9626031137446264] 34%|███▍      | 17/50 [00:02<00:05,  6.60trial/s, best loss: -0.9626031137446264] 36%|███▌      | 18/50 [00:02<00:05,  6.27trial/s, best loss: -0.9626031137446264] 38%|███▊      | 19/50 [00:03<00:06,  4.83trial/s, best loss: -0.9626031137446264] 40%|████      | 20/50 [00:03<00:07,  3.98trial/s, best loss: -0.9626031137446264] 42%|████▏     | 21/50 [00:03<00:07,  3.99trial/s, best loss: -0.9626031137446264] 44%|████▍     | 22/50 [00:03<00:06,  4.04trial/s, best loss: -0.9626031137446264] 46%|████▌     | 23/50 [00:04<00:06,  4.19trial/s, best loss: -0.9626031137446264] 48%|████▊     | 24/50 [00:04<00:06,  3.93trial/s, best loss: -0.9626031137446264] 50%|█████     | 25/50 [00:04<00:06,  4.04trial/s, best loss: -0.9626031137446264] 52%|█████▏    | 26/50 [00:04<00:06,  3.90trial/s, best loss: -0.9626031137446264] 54%|█████▍    | 27/50 [00:05<00:05,  4.09trial/s, best loss: -0.9626031137446264] 56%|█████▌    | 28/50 [00:05<00:04,  4.52trial/s, best loss: -0.9647960962007668] 58%|█████▊    | 29/50 [00:05<00:04,  4.55trial/s, best loss: -0.9647960962007668] 60%|██████    | 30/50 [00:05<00:04,  4.54trial/s, best loss: -0.9647960962007668] 62%|██████▏   | 31/50 [00:05<00:04,  4.72trial/s, best loss: -0.9647960962007668] 64%|██████▍   | 32/50 [00:06<00:03,  5.13trial/s, best loss: -0.9647960962007668] 66%|██████▌   | 33/50 [00:06<00:03,  5.28trial/s, best loss: -0.9647960962007668] 68%|██████▊   | 34/50 [00:06<00:02,  5.54trial/s, best loss: -0.9647960962007668] 78%|███████▊  | 39/50 [00:06<00:00, 11.51trial/s, best loss: -0.9647960962007668] 82%|████████▏ | 41/50 [00:07<00:01,  8.29trial/s, best loss: -0.9647960962007668] 84%|████████▍ | 42/50 [00:07<00:01,  7.71trial/s, best loss: -0.9647960962007668] 86%|████████▌ | 43/50 [00:07<00:01,  6.58trial/s, best loss: -0.9647960962007668] 88%|████████▊ | 44/50 [00:07<00:00,  6.65trial/s, best loss: -0.9647960962007668] 90%|█████████ | 45/50 [00:07<00:00,  6.76trial/s, best loss: -0.9647960962007668] 92%|█████████▏| 46/50 [00:08<00:00,  5.00trial/s, best loss: -0.9647960962007668] 94%|█████████▍| 47/50 [00:08<00:00,  3.87trial/s, best loss: -0.9647960962007668] 96%|█████████▌| 48/50 [00:08<00:00,  3.48trial/s, best loss: -0.9647960962007668] 98%|█████████▊| 49/50 [00:09<00:00,  3.49trial/s, best loss: -0.9647960962007668]100%|██████████| 50/50 [00:09<00:00,  3.29trial/s, best loss: -0.9647960962007668]100%|██████████| 50/50 [00:09<00:00,  5.25trial/s, best loss: -0.9647960962007668]

{'colsample_bytree': 0.8333015582921072,
 'learning_rate': 0.181846600140329,
 'max_depth': 11.0,
 'min_child_weight': 1.0}

각주

1. hard voting (단순 다수결), soft voting(label을 예측할 확률의 가중 평균으로 분류)으로 나뉨. 일반적으로 soft voting이 사용됨.

2. hard voting (단순 다수결), soft voting(label을 예측할 확률의 가중 평균으로 분류)으로 나뉨. 일반적으로 soft voting이 사용됨.