Feature Engineering

Klassifikation

Regression

Validierung und mehr

Sampling und Resampling

Validierungstechniken

Grid Search und Random Search

In [1]:

import sys
sys.path.append('./')

In [2]:

## preparation

## import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

sns.set()
%matplotlib inline

## load data
datapath = '../3_data'
from os import chdir
chdir(datapath)

from bfh_cas_pml import prep_data
X_train, X_test, y_train, y_test = prep_data(
    'bank_data_prep.csv', 'y', seed=1234)
#X_demo, y_demo = prep_demo_data('demo_data_class.csv', 'y')

Rekapitulation Parameter Tuning

In [3]:

## recap parameter tuning for DecisionTreeClassifier in 2.2.1.5
## min_impurity_decrease from 0 to 0.001 in steps 0.0001

from sklearn.tree import DecisionTreeClassifier
scores = []
params = np.arange(0, 0.001, 0.0001) 
for param in params:
    model = DecisionTreeClassifier(min_impurity_decrease = param)
    model.fit(X_train, y_train) 
    score = model.score(X_test, y_test)
    scores.append(score)

fig = sns.lineplot(x=params, y=scores)
plt.scatter(x=params[scores.index(max(scores))], y=max(scores), color="black")
plt.xlabel('min_impurity_decrease')
plt.ylabel('accuracy');

In [4]:

## and another interesting parameter: min_samples_leaf
scores = []
params = range(1, 41, 5) 
for param in params:
    model = DecisionTreeClassifier(min_samples_leaf = param)
    model.fit(X_train, y_train) 
    score = model.score(X_test, y_test)
    scores.append(score)

fig = sns.lineplot(x=params, y=scores)
plt.scatter(x=params[scores.index(max(scores))], y=max(scores), color="black")
plt.xlabel('min_samples_leaf')
plt.ylabel('accuracy');

Grid Search mit CV

Mit einem Parameter

In [5]:

## construction of a parameter grid
params = {'min_impurity_decrease': np.arange(0, 0.001, 0.0001)}
print(type(params))
print(params)

<class 'dict'>
{'min_impurity_decrease': array([0.    , 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007,
       0.0008, 0.0009])}

In [6]:

## using the defined parameter grid
from sklearn.model_selection import GridSearchCV
params = {'min_impurity_decrease': np.arange(0, 0.001, 0.0001)}
model = DecisionTreeClassifier(random_state=1234)
gscv = GridSearchCV(model, param_grid=params, cv=10)
gscv.fit(X_train, y_train)
cv_results = pd.DataFrame(gscv.cv_results_)

In [7]:

plt.errorbar(
    'param_min_impurity_decrease', 
    'mean_test_score', 
    'std_test_score', 
    data = cv_results, linestyle='None');

In [8]:

cv_results

Out [8]:

	mean_fit_time	std_fit_time	mean_score_time	std_score_time	param_min_impurity_decrease	params	split0_test_score	split1_test_score	split2_test_score	split3_test_score	split4_test_score	split5_test_score	split6_test_score	split7_test_score	split8_test_score	split9_test_score	mean_test_score	std_test_score	rank_test_score
0	0.018237	0.000573	0.000778	0.000029	0.0000	{'min_impurity_decrease': 0.0}	0.811550	0.820669	0.828267	0.837139	0.837139	0.814307	0.850837	0.852359	0.841705	0.831050	0.832502	0.013373	10
1	0.015662	0.000387	0.000747	0.000007	0.0001	{'min_impurity_decrease': 0.0001}	0.831307	0.822188	0.855623	0.873668	0.850837	0.841705	0.853881	0.869102	0.844749	0.843227	0.848629	0.014896	9
2	0.013681	0.000354	0.000734	0.000012	0.0002	{'min_impurity_decrease': 0.0002}	0.851064	0.858663	0.873860	0.885845	0.866058	0.869102	0.861492	0.879756	0.856925	0.855403	0.865817	0.010705	8
3	0.012099	0.000255	0.000708	0.000010	0.0003	{'min_impurity_decrease': 0.00030000000000000003}	0.866261	0.863222	0.887538	0.890411	0.878234	0.875190	0.866058	0.902588	0.870624	0.859970	0.876010	0.013024	6
4	0.011296	0.000203	0.000702	0.000017	0.0004	{'min_impurity_decrease': 0.0004}	0.873860	0.870821	0.882979	0.893455	0.869102	0.882801	0.870624	0.885845	0.872146	0.870624	0.877226	0.007957	4
5	0.010792	0.000356	0.000712	0.000042	0.0005	{'min_impurity_decrease': 0.0005}	0.876900	0.869301	0.882979	0.893455	0.867580	0.881279	0.884323	0.885845	0.873668	0.872146	0.878747	0.007803	1
6	0.010433	0.000228	0.000705	0.000013	0.0006	{'min_impurity_decrease': 0.0006000000000000001}	0.876900	0.867781	0.882979	0.890411	0.870624	0.884323	0.881279	0.881279	0.873668	0.872146	0.878139	0.006732	2
7	0.010160	0.000169	0.000695	0.000012	0.0007	{'min_impurity_decrease': 0.0007}	0.876900	0.866261	0.886018	0.890411	0.864536	0.888889	0.878234	0.887367	0.869102	0.867580	0.877530	0.009646	3
8	0.010084	0.000180	0.000705	0.000022	0.0008	{'min_impurity_decrease': 0.0008}	0.872340	0.866261	0.886018	0.891933	0.864536	0.888889	0.876712	0.882801	0.870624	0.867580	0.876769	0.009490	5
9	0.009884	0.000287	0.000700	0.000014	0.0009	{'min_impurity_decrease': 0.0009000000000000001}	0.872340	0.866261	0.886018	0.887367	0.864536	0.888889	0.879756	0.879756	0.855403	0.873668	0.875400	0.010439	7

Mit mehr als einem Parameter

In [9]:

## combination of more than one parameter
from sklearn.model_selection import GridSearchCV
params = {'min_impurity_decrease': np.arange(0.0002, 0.001, 0.0002),
          'min_samples_leaf': range(5, 25, 5)}

model = DecisionTreeClassifier(random_state=1234)
gscv = GridSearchCV(model, param_grid=params, cv=10)
gscv.fit(X_train, y_train)

## the pivot table (as pandas data frame)
cv_results = pd.pivot_table(pd.DataFrame(gscv.cv_results_),
    values='mean_test_score', 
    index='param_min_impurity_decrease', 
    columns='param_min_samples_leaf')
print(cv_results)

param_min_samples_leaf             5         10        15        20
param_min_impurity_decrease                                        
0.0002                       0.870228  0.871142  0.870684  0.873724
0.0004                       0.874791  0.875857  0.875857  0.876616
0.0006                       0.877986  0.878443  0.877835  0.879507
0.0008                       0.876769  0.876162  0.875401  0.877226

Cell:

[Cell type raw - unsupported, skipped]

In [10]:

## format y labels
plt.subplots(figsize=(6, 5))   
g = sns.heatmap(cv_results)
ylabels = ['{:,.4f}'.format(x) for x in cv_results.index]
g.set_yticklabels(ylabels);

In [11]:

## the most important attributes
print('best_params_ :', gscv.best_params_)
print('best_score_  :', gscv.best_score_)
print('refit_time_  :', gscv.refit_time_)

best_params_ : {'min_impurity_decrease': np.float64(0.0006000000000000001), 'min_samples_leaf': 20}
best_score_  : 0.8795073397084472
refit_time_  : 0.011003255844116211

In [12]:

%%time
## the above is a magic command which measures the execution time for the whole cell
## must be in the first line of cell

## a more detailed granularity
from sklearn.model_selection import GridSearchCV
params = {'min_impurity_decrease': np.arange(0, 0.001, 0.0001),
          'min_samples_leaf': range(1, 41, 5)}
model = DecisionTreeClassifier(random_state=1234)
gscv = GridSearchCV(model, param_grid=params, cv=10)
gscv.fit(X_train, y_train)

## pivot table and heatmap
cv_results = pd.pivot_table(pd.DataFrame(gscv.cv_results_),
    values = 'mean_test_score', 
    index = 'param_min_impurity_decrease', 
    columns = 'param_min_samples_leaf')
#print(cv_results)

plt.figure(figsize=(6,5))
sns.heatmap(cv_results,
            yticklabels=cv_results.index.values.round(4));

## the most important attributes
print('best_params_ :', gscv.best_params_)
print('best_score_  :', gscv.best_score_)
print('refit_time_  :', gscv.refit_time_)

best_params_ : {'min_impurity_decrease': np.float64(0.0005), 'min_samples_leaf': 16}
best_score_  : 0.879964654665908
refit_time_  : 0.011229991912841797
CPU times: user 9.61 s, sys: 26 ms, total: 9.64 s
Wall time: 9.64 s

Random Search mit CV

In [13]:

%%time

## limitation of the above to 20 randomly chosen combinations
from sklearn.model_selection import RandomizedSearchCV
params = {'min_impurity_decrease': np.arange(0, 0.001, 0.0001),
          'min_samples_leaf': range(1, 41, 5)}
model = DecisionTreeClassifier(random_state=1234)
rscv = RandomizedSearchCV(
    model, 
    param_distributions=params, 
    cv=10, ## default: 5
    n_iter=20) ## default: 10
rscv.fit(X_train, y_train)

## pivot table and heatmap
plt.figure(figsize=(6,5))
cv_results = pd.pivot_table(pd.DataFrame(rscv.cv_results_),
    values='mean_test_score', 
    index='param_min_impurity_decrease', 
    columns='param_min_samples_leaf')
sns.heatmap(cv_results,
            yticklabels=cv_results.index.values.round(4));

## the most important attributes
print('best_params_ :', rscv.best_params_)
print('best_score_  :', rscv.best_score_)

best_params_ : {'min_samples_leaf': 11, 'min_impurity_decrease': np.float64(0.0005)}
best_score_  : 0.8788987430199905
CPU times: user 2.33 s, sys: 8.1 ms, total: 2.34 s
Wall time: 2.34 s

define a distribution instead of a grid

In [14]:

%%time

from scipy.stats.distributions import uniform
params = {
    'max_depth': range(1, 101),
    'min_samples_leaf': range(1, 21),
    'min_impurity_decrease': uniform(0, 0.001) ## instead of np.arange()
}

model = DecisionTreeClassifier(random_state=1234)
rscv = RandomizedSearchCV(
    model, 
    param_distributions=params, 
    cv=5,
    n_iter=20) ## default: 10
rscv.fit(X_train, y_train)

#print('best_estimator_ :', rscv.best_estimator_)
print('best_params_ :', rscv.best_params_)
print('best_score_     :', rscv.best_score_)

best_params_ : {'max_depth': 51, 'min_impurity_decrease': np.float64(0.0008772041621438149), 'min_samples_leaf': 16}
best_score_     : 0.879809712311405
CPU times: user 1.17 s, sys: 4.53 ms, total: 1.18 s
Wall time: 1.18 s

Cell:

[Cell type raw - unsupported, skipped]

192 KiB Raw Blame History

Feature Engineering

Klassifikation

Regression

Validierung und mehr

Sampling und Resampling

Validierungstechniken

Grid Search und Random Search

Rekapitulation Parameter Tuning

Grid Search mit CV

Mit einem Parameter

Mit mehr als einem Parameter

Random Search mit CV

192 KiB

Raw Blame History