from numpy import mean
from numpy import std
from numpy import arange
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedStratifiedKFold
from xgboost import XGBRFClassifier
from matplotlib import pyplot


# get the dataset
def get_dataset ():
    X,y = make_classification(n_samples=1000,n_features=20,n_informative=15,n_redundant=5,random_state=7)
    return X,y


# get a list of models to evaluate
def get_models ():
    models = dict()
    for v in arange(0.1,1.1,0.1):
        key = '%.1f' % v
        models[key] = XGBRFClassifier(n_estimators=100,subsample=0.9,colsample_bynode=v)
    return models


# evaluate a give model using cross-validation
def evaluate_model (model,X,y):
    # define the model evaluation procedure
    cv = RepeatedStratifiedKFold(n_splits=10,n_repeats=3,random_state=1)
    # evaluate the model
    scores = cross_val_score(model,X,y,scoring='accuracy',cv=cv,n_jobs=-1)
    return scores


# define dataset
X,y = get_dataset()
# get the models to evaluate
models = get_models()
# evaluate the models and store results
results,names = list(),list()
for name,model in models.items():
    # evaluate the model and collect the results
    scores = evaluate_model(model,X,y)
    # store the results
    results.append(scores)
    names.append(name)
    # summarize performance along the way
    print('>%s %.3f (%.3f)' % (name,mean(scores),std(scores)))
# plot model performance for comparison
pyplot.boxplot(results,labels=names,showmeans=True)
pyplot.show()
