import pandas as pd

# Let's load the data from some CSV files I prepared:
X_train = pd.read_csv("classify X_train.csv", index_col=0)
X_test = pd.read_csv("classify X_test.csv", index_col=0)

print(X_train.columns)

n_samples, n_features = X_train.shape
print n_samples, n_features

X_train.head(n=5)

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier()
rf.fit(X_train[cols].dropna(), 
       X_train[cols+["label"]].dropna()["label"])  
score = rf.score(X_test[cols].dropna(), 
                 X_test[cols+["label"]].dropna()["label"])
print "Score: {:.2f}".format(score)

# Great performance, but there's a catch:
n_samples, n_features = X_test.shape
n_samples_dropna, n_features_dropna = X_test.dropna().shape

print "Number of samples before dropna: {}, after dropna: {}".format(
    n_samples, n_samples_dropna)
print "Missing due to NaN: {}".format(n_samples-n_samples_dropna)

from sklearn.preprocessing import Imputer

# Create the imputer, which will replace the NaN's with the mean along the columns
imputer = Imputer(missing_values="NaN", strategy="mean", axis=0)

X_train2 = []
for subject_id, subject_group in X_train.groupby("subject_id"):
    # Impute the missing values
    subject_group[cols] = imputer.fit_transform(subject_group[cols])
    X_train2.append(subject_group)

# Create a data frame again
X_train2 = pd.concat(X_train2)


X_test2 = []
for subject_id, subject_group in X_test.groupby("subject_id"):
    subject_group[cols] = imputer.fit_transform(subject_group[cols])
    X_test2.append(subject_group)
    
X_test2 = pd.concat(X_test2)

# Let's see how many samples we have left now
n_samples, n_features = X_test.shape
n_samples_dropna, n_features_dropna = X_test.dropna().shape
n_samples_imputed, n_features_imputed = X_test2.dropna().shape
print("Number of samples before dropna: {}, after dropna: {}, "),
print("after imputing: {}".format(n_samples, n_samples_dropna, n_samples_imputed))

rf = RandomForestClassifier()
rf.fit(X_train2[cols], X_train2["label"])  
score_imputed = rf.score(X_test2[cols], X_test2["label"])

print "Score after imputation: {:.2f}".format(score_imputed)

from sklearn import preprocessing

X_train3 = []
for subject_id, subject_group in X_train.groupby("subject_id"):
    # First impute the missing values
    subject_group[cols] = imputer.fit_transform(subject_group[cols])  
    # Then scale the values
    subject_group[cols] = preprocessing.scale(subject_group[cols])
    X_train3.append(subject_group)

X_train3 = pd.concat(X_train3)

X_test3 = []
for subject_id, subject_group in X_test.groupby("subject_id"):
    subject_group[cols] = imputer.fit_transform(subject_group[cols])
    subject_group[cols] = preprocessing.scale(subject_group[cols])      
    X_test3.append(subject_group)
    
X_test3 = pd.concat(X_test3)

rf = RandomForestClassifier()
rf.fit(X_train3[cols], X_train3["label"])  
score_scaled = rf.score(X_test3[cols], X_test3["label"])

print "Score after scaling and imputation: {:.2f}".format(score_scaled)

import numpy as np
from sklearn import cross_validation

k_fold = cross_validation.KFold(n=len(dognames), n_folds=5)

X = pd.concat([X_train2, X_test2])

scores_cv = []

for index, (train_indices, test_indices) in enumerate(k_fold):   
    train_dogs = dognames[train_indices]
    test_dogs = dognames[test_indices]
    
    # Little helper function to split the dataset
    x_train, x_test = split_data_set(X, train_dogs)
        
    # Now fit the classifier
    rf = RandomForestClassifier()
    rf.fit(x_train[cols], x_train["label"])  
    score_cv = rf.score(x_test[cols], x_test["label"])
    scores_cv.append(score_cv)
    
    print("fold-{: <2} {:.2f}\t".format(index, score_cv))

print("")
print("Average: {:.2f} +/- {:.2f}".format(np.mean(scores_cv), np.std(scores_cv)))

# On most other data sets this can be done a lot simpler
from sklearn import cross_validation

k_fold = cross_validation.KFold(n=len(X), n_folds=5)
# Scikit-Learn comes with a useful helper function to 
# compute the cross validation score
score_cv = cross_validation.cross_val_score(rf, X[cols], X["label"], cv=k_fold)

print("Average: {:.2f} +/- {:.2f}".format(np.mean(scores_cv), np.std(scores_cv)))

from sklearn.grid_search import GridSearchCV

# Use a full grid over all parameters
param_grid = {"max_depth": [3, None],
              "max_features": [1, 3, 10],
              "min_samples_split": [1, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]}

clf = RandomForestClassifier(n_estimators=20)

# Run grid search
grid_search = GridSearchCV(clf, param_grid=param_grid)
start = time()
grid_search.fit(X[cols], X["label"])

print("GridSearchCV took {:.2f} seconds for {} candidate parameter settings.".format(
    time() - start, len(grid_search.grid_scores_)))

print("")
# Helper function to pretty print the results
report(grid_search.grid_scores_)

rf = RandomForestClassifier(bootstrap=False, min_samples_leaf=1, 
                            min_samples_split=10, criterion='entropy', 
                            max_features=3, max_depth=None)

# Lets try it on one of our last train/test splits
rf.fit(x_train[cols], x_train["label"])  
score_grid = rf.score(x_test[cols], x_test["label"])

print("Score after grid search: {:.2f}".format(score_grid))

from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
from sklearn import cross_validation

# Create X again from the raw data
X = pd.concat([X_train, X_test])

pipeline = Pipeline([
    ("imputer", Imputer(missing_values="NaN", 
                        strategy="mean", axis=0)),
    ("forest", RandomForestClassifier(n_estimators=20))])

# Note the difference in the params, 
# you have to use double underscore notation
# See: http://stackoverflow.com/questions/16437022/how-to-tune-parameters-of-nested-pipelines-by-gridsearchcv-in-scikit-learn
param_grid = {
    "forest__min_samples_split": [1, 3, 10],
    "forest__min_samples_leaf": [1, 3, 10],
    "forest__bootstrap": [True, False],
    "forest__criterion": ["gini", "entropy"]
}

grid_search = GridSearchCV(pipeline, param_grid=param_grid)

grid_search.fit(X[cols], X["label"])
print("Best score: {:0.3f}".format(grid_search.best_score_))

%matplotlib inline
from sklearn.metrics import confusion_matrix

y_pred = rf.predict(X_test2[cols])

# Compute confusion matrix
cm = confusion_matrix(X_test2["label"], y_pred)
print(cm)

import matplotlib.pyplot as plt
plt.matshow(cm)
plt.title('Confusion matrix')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()

import StringIO
import pydot
from sklearn import tree
from IPython.core.display import Image

# Visualize tree
dot_data = StringIO.StringIO()
tree.export_graphviz(rf.estimators_[0], out_file=dot_data)
graph = pydot.graph_from_dot_data(dot_data.getvalue())
image = graph.write_png("./images/random_network.png")

print("Feature")
for col, importance in zip(cols, rf.feature_importances_):
    if importance > 0.05:
        print "{:>3}:\t{:.4f}".format(col, importance)

import matplotlib.pyplot as plt

# Example from: http://scikit-learn.org/stable/auto_examples/ensemble/plot_forest_importances.html
importances = rf.feature_importances_
std = np.std([tree.feature_importances_ for tree in rf.estimators_], axis=0)
indices = np.argsort(importances)[::-1]

fig, ax = plt.subplots(figsize=(12,6))
fig.suptitle("Feature importances")
ax.bar(range(len(importances)), importances[indices], color="r", yerr=std[indices], align="center")
ax.set_xticks(range(len(importances)))
cols = np.array(cols)
ax.set_xticklabels(cols[indices], rotation=90)
ax.set_ylim([0, 0.3])
_ = ax.set_xlim([-1, len(cols)])