import pandas as pd
from sklearn.model_selection import train_test_split

data = pd.read_csv('data.csv')
# 会話率が0のデータを取り除く
data = data[data.kaiwaritu != 0]

train, test = train_test_split(data,test_size=0.2)
train.to_csv("train.csv")
test.to_csv("test.csv")

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

data = pd.read_csv('data.csv')
# 会話率が0のデータを取り除く
data = data[data["kaiwaritu"] != 0]

plt.figure(figsize=(16, 12))
sns.set(font_scale=2)
sns.distplot(data["bookmark"])

print("歪度 :" , data["bookmark"].skew())
print("尖度 :" , data["bookmark"].kurtosis())
print(data["bookmark"].describe())

>>出力結果
歪度 : 16.267647775193502
尖度 : 366.7858370210778
count     3016.000000
mean       218.227122
std       1675.647169
min          0.000000
25%          0.000000
50%          2.000000
75%         15.000000
max      51077.000000

print(data["bookmark"].describe(percentiles=[0.1, 0.2,0.3, 0.4,0.5, 0.6,0.7,0.71, 0.8,0.9]))

>>出力結果
count     3016.000000
mean       218.227122
std       1675.647169
min          0.000000
10%          0.000000
20%          0.000000
30%          0.000000
40%          1.000000
50%          2.000000
60%          4.000000
70%          9.000000
71%         10.000000
80%         25.000000
90%        119.000000
max      51077.000000

import pandas as pd
import numpy as np

import warnings
warnings.filterwarnings('ignore')

from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from xgboost import XGBClassifier
from sklearn.model_selection import cross_validate

from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_curve
from sklearn.metrics import auc

train = pd.read_csv("train.csv")


# 説明変数
x = train.drop("ten_or_more_bookmarks", axis=1)
# 目的変数
y = train["ten_or_more_bookmarks"]

mlas = [
    RandomForestClassifier(random_state=0),
    GradientBoostingClassifier(),
    XGBClassifier()
]

# 交差検証
for mla in mlas:
    result = cross_validate(mla, x, y, cv=5, scoring = ["accuracy", "roc_auc"], return_estimator=True)
    print("accuracy", np.average(result["test_accuracy"]))
    print("AUC     ", np.average(result["test_roc_auc"]))


# テストデータで検証
test = pd.read_csv("test.csv")
test_x = test.drop("ten_or_more_bookmarks", axis=1)
test_y = test["ten_or_more_bookmarks"]

for mla in mlas:
    pred = mla.fit(x, y).predict(test_x)
    print("accuracy", accuracy_score(pred, test_y))

    fpr, tpr, thresholds = roc_curve(test_y, pred)
    print("AUC     ", auc(fpr, tpr))


>>出力結果
# ---交差検証---
# ランダムフォレスト
accuracy 0.7873776106141251
AUC      0.8129739925467003
# 勾配ブースティング
accuracy 0.8152069081529663
AUC      0.8612213954204841
# XGBoost
accuracy 0.8235108392784742
AUC      0.8655443385400972

# ---テストデータ---
# ランダムフォレスト
accuracy 0.813953488372093
AUC      0.7215116279069768
# 勾配ブースティング
accuracy 0.8355481727574751
AUC      0.7715116279069768
# XGBoost
accuracy 0.8421926910299004
AUC      0.7796511627906977

xgbc = XGBClassifier()
result = cross_validate(xgbc, x, y, cv=5, return_estimator=True)

feature_importances_list = []

for idx,estimator in enumerate(result['estimator']):
    feature_importances = pd.DataFrame(estimator.feature_importances_,
                                       index = x.columns,
                                        columns=['importance']).sort_values('importance', ascending=False)
    feature_importances_list.append(feature_importances)

dataframe_mean = 0
for dataframe in feature_importances_list:
    dataframe_mean += dataframe

df = dataframe_mean / len(feature_importances_list)
df = df.sort_values("importance")
sns.set(font_scale=1.5)
df.plot(figsize=(16,12),kind="barh")

train["biggenre"] = train["biggenre"].replace({
                                        1:"恋愛",
                                        2:"ファンタジー",
                                        3:"文芸",
                                        4:"SF",
                                        99:"その他",
                                        98:"ノンジャンル"
                                    })
sns.countplot(x="biggenre", hue="ten_or_more_bookmarks", data=train)

sns.countplot(x="genre", hue="ten_or_more_bookmarks", data=train)

sns.countplot(x="isr15", hue="ten_or_more_bookmarks", data=train)

x_axis_names = [
    "story_character_count",
    "title_character_count",
    "vocabulary_rate",
    "blank_line_rate"
]

for x_axis in x_axis_names:
    fg = sns.FacetGrid( train, hue = 'ten_or_more_bookmarks', aspect=4 )
    fg.map(sns.kdeplot, x_axis, shade= True )
    fg.set(xlim=(0 , train[x_axis].max()))
    fg.add_legend()

test = pd.read_csv("test.csv", index_col=0)
test  = test.loc[:,["biggenre","genre","isr15","title_character_count","story_character_count","vocabulary_rate","blank_line_rate","bookmark"]]

test =  test[( test["biggenre"] == 1) | ( test["biggenre"] == 2)]
test =  test[( test["genre"] == 101) | ( test["genre"] == 102) | ( test["genre"] == 201)]
test =  test[ test["isr15"] == 1]
test =  test[( test["story_character_count"] >=200) & ( test["story_character_count"] <=400)]
test =  test[( test["title_character_count"] >=20) & ( test["title_character_count"] <=30)]

test.sort_values(by=["bookmark"], ascending=False)