【kaggle】ベイズ最適化とXGBでtitanicの予測問題を解く|python

import numpy as np
import pandas as pd
from pandas import DataFrame, Series
import seaborn as sns
from sklearn.model_selection import train_test_split, KFold, cross_val_score, StratifiedKFold, validation_curve
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
import matplotlib.pyplot as plt
import sklearn as skl
import optuna
from sklearn.metrics import roc_curve, auc

df = pd.read_csv("train.csv")
df_test = pd.read_csv("test.csv")

# 推定に使用する項目を抽出
age_df = df[['Age', 'Pclass','Sex','Parch','SibSp']]

# ラベル特徴量をワンホットエンコーディング
age_df=pd.get_dummies(age_df)]

# 学習データとテストデータに分離し、numpyに変換
known_age = age_df[age_df.Age.notnull()].values  
unknown_age = age_df[age_df.Age.isnull()].values

# 学習データをX, yに分離
X = known_age[:, 1:]  
y = known_age[:, 0]

# ランダムフォレストで推定モデルを構築
rfr = RandomForestRegressor(random_state=0, n_estimators=100, n_jobs=-1)
rfr.fit(X, y)

# 推定モデルを使って、テストデータのAgeを予測し、補完
predictedAges = rfr.predict(unknown_age[:, 1::])
df.loc[(df.Age.isnull()), 'Age'] = predictedAges 

# 推定に使用する項目を抽出
age_df_test = df_test[['Age', 'Pclass','Sex','Parch','SibSp']]

# ラベル特徴量をワンホットエンコーディング
age_df_test=pd.get_dummies(age_df_test)

# 学習データとテストデータに分離し、numpyに変換
known_age_test = age_df_test[age_df_test.Age.notnull()].values  
unknown_age_test = age_df_test[age_df_test.Age.isnull()].values

# 学習データをX, yに分離
X = known_age_test[:, 1:]  
y = known_age_test[:, 0]

# ランダムフォレストで推定モデルを構築
rfr = RandomForestRegressor(random_state=0, n_estimators=100, n_jobs=-1)
rfr.fit(X, y)

# 推定モデルを使って、テストデータのAgeを予測し、補完
predictedAges_test = rfr.predict(unknown_age_test[:, 1::])
df_test.loc[(df_test.Age.isnull()), 'Age'] = predictedAges_test

#cabinはほとんど欠損値なので削除する
df.drop('Cabin', axis=1, inplace=True)
df_test.drop('Cabin', axis=1, inplace=True)

sns.countplot('Embarked',data=df)
plt.title('Number of Passengers Boarded')
plt.show()

df['Embarked'].fillna('S', inplace=True)
df_test['Embarked'].fillna('S', inplace=True)
df_test.isnull().sum()

#Fareは平均値を補完
df_test['Fare'].fillna('35.627188', inplace=True)
#NameやTikcetは削除
df.drop('Name', axis=1, inplace=True)
df_test.drop('Name', axis=1, inplace=True)
df.drop('Ticket', axis=1, inplace=True)
df_test.drop('Ticket', axis=1, inplace=True)

#男性と女性は2値分類にする
df.replace({'Sex': {'male': 0, 'female': 1}}, inplace=True)
df_test.replace({'Sex': {'male': 0, 'female': 1}}, inplace=True)

embarked = pd.concat([df['Embarked'], df_test['Embarked']])

embarked_ohe = pd.get_dummies(embarked)

embarked_ohe_train = embarked_ohe[:891]
embarked_ohe_test = embarked_ohe[891:]

df = pd.concat([df, embarked_ohe_train], axis=1)
df_test = pd.concat([df_test, embarked_ohe_test], axis=1)

#元のembarkedを消す
df.drop('Embarked', axis=1, inplace=True)
df_test.drop('Embarked', axis=1, inplace=True)

# 乱数シード
seed = 42

# チューニング前のモデル
model = XGBRegressor(booster='gbtree', objective='binary:logistic',
                     random_state=seed, n_estimators=100)  
# 学習時fitパラメータ指定
fit_params = {'early_stopping_rounds': 10,
              'eval_metric': 'logloss',
              'eval_set': [(X, y)]
              }

# クロスバリデーションして決定境界を可視化
cv = KFold(n_splits=10, shuffle=True, random_state=seed)  # KFoldでクロスバリデーション分割指定

scoring = 'neg_mean_squared_error'

# ベイズ最適化時の評価指標算出メソッド
def bayes_objective(trial):
    params = {
        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),
        'min_child_weight': trial.suggest_int('min_child_weight', 2, 8),
        'max_depth': trial.suggest_int('max_depth', 1, 10),
        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.2, 1.0),
        'subsample': trial.suggest_float('subsample', 0.2, 1.0),
        'reg_alpha': trial.suggest_float('reg_alpha', 0.001, 0.1, log=True),
        'reg_lambda': trial.suggest_float('reg_lambda', 0.001, 0.1, log=True),
        'gamma': trial.suggest_float('gamma', 0.0001, 0.1, log=True)
    }
    # モデルにパラメータ適用
    model.set_params(**params)

    # cross_val_scoreでクロスバリデーション
    scores = cross_val_score(model, X, y, cv=cv,
                             scoring=scoring, fit_params=fit_params, n_jobs=-1)
    val = scores.mean()
    return val

# ベイズ最適化を実行
study = optuna.create_study(direction='maximize',
                            sampler=optuna.samplers.TPESampler(seed=seed))
study.optimize(bayes_objective, n_trials=100)

study.optimize(bayes_objective, timeout=180)

# 最適パラメータの表示と保持
best_params = study.best_trial.params
best_score = study.best_trial.value
print(f'最適パラメータ {best_params}\nスコア {best_score}')

→最適パラメータ {'learning_rate': 0.10316346044446496, 'min_child_weight': 4, 'max_depth': 8, 'colsample_bytree': 0.7004439463296043, 'subsample': 0.815334361981248, 'reg_alpha': 0.08602317024652126, 'reg_lambda': 0.08689223246655367, 'gamma': 0.003225305003064497}
スコア -0.12102113895277018

X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, random_state=42)

# 最適パラメータを学習器にセット
model.set_params(**best_params)
xgb_pred = model.fit(X_train,y_train)
y_pred_train = model.predict(X_train)
y_pred_valid = model.predict(X_valid)

fpr, tpr, thresholds = roc_curve(y_valid, y_pred_valid)

# AUCの算出
auc = auc(fpr, tpr)

# ROC曲線の描画

plt.plot(fpr, tpr, color='red', label='ROC curve (area = %.3f)' % auc)
plt.plot([0, 1], [0, 1], color='black', linestyle='--')

plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="best")

xgb_pred = model.predict(X_test)
#0と1にラベリング
xgb_pred = np.where(xgb_pred > 0.5, 1, 0)
xgb_pred[:5]