1.2 MiB
1.2 MiB
Cell:
[Cell type raw - unsupported, skipped]
In [1]:
import sys
sys.path.append('./')In [2]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
%matplotlib inline
datapath = '../3_data'
from os import chdir; chdir(datapath)
from bfh_cas_pml import prep_data, prep_demo_data
X_train, X_test, y_train, y_test = prep_data(
'bank_data_prep.csv', 'y', seed = 1234)
X_demo, y_demo = prep_demo_data('demo_data_class.csv', 'y')In [3]:
## plot scatterplot and areas for predictions
fig, ax = plt.subplots(figsize=(4,4))
sns.scatterplot(x='X1', y='X2', data=X_demo, hue=y_demo,
palette=['darkorange', 'steelblue'], legend='full')
plt.plot([3, 7], [1.68, 1.68], linewidth=1, color='black')
plt.plot([4.95, 4.95], [1, 1.68], linewidth=1, color='black');
In [4]:
## potential first split positions
idx = np.unique(X_demo.X1)
for i in range(6):
## for formatted output
print('%2i %4.1f %4.1f %5.2f' % (
i, idx[i], idx[i+1], (idx[i] + idx[i+1]) / 2))0 3.0 3.3 3.15 1 3.3 3.5 3.40 2 3.5 3.6 3.55 3 3.6 3.7 3.65 4 3.7 3.8 3.75 5 3.8 3.9 3.85
In [5]:
## check split at X1 < 5
child_l = y_demo[X_demo.X1 < 5]
child_r = y_demo[X_demo.X1 >= 5]
print(pd.Series(child_l).value_counts(sort = False))
print(pd.Series(child_r).value_counts(sort = False))y A 5 B 34 Name: count, dtype: int64 y A 40 B 2 Name: count, dtype: int64
In [6]:
## accuvacy score
pred = np.where(X_demo['X1'] < 5, 'B', 'A')
from sklearn.metrics import accuracy_score
accuracy_score(pred, y_demo)Out [6]:
0.9135802469135802
In [7]:
## control: train model and show first rules
from sklearn.tree import DecisionTreeClassifier
model = DecisionTreeClassifier(
min_impurity_decrease=0.02,
random_state=1234)
model.fit(X_demo, y_demo)
from sklearn.tree import export_text
print(export_text(
model, feature_names=list(X_demo.columns)))|--- X2 <= 1.75 | |--- X1 <= 5.35 | | |--- class: B | |--- X1 > 5.35 | | |--- class: A |--- X2 > 1.75 | |--- class: A
Cell:
[Cell type raw - unsupported, skipped]
In [8]:
## viszalize tree
from sklearn.tree import plot_tree
import matplotlib.pyplot as plt
plt.figure(figsize=(6, 7))
plot_tree(model,
feature_names=X_demo.columns,
class_names=model.classes_,
filled=True); # Adds color accoding to class
Cell:
[Cell type raw - unsupported, skipped]
In [9]:
## define, parameterize and train the model
from sklearn.tree import DecisionTreeClassifier
model = DecisionTreeClassifier(random_state=1234) ## the one parameter (optional)
model.fit(X_train, y_train) Out [9]:
DecisionTreeClassifier(random_state=1234)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
DecisionTreeClassifier(random_state=1234)
In [10]:
## apply the model on the train data
y_pred = model.predict(X_test)In [11]:
## evaluate with model intern scorer (accuracy)
print(model.score(X_test, y_test))0.8296318831761484
In [12]:
## some more information about the trained model
print('depth:', model.get_depth())
print('n_leaves:', model.get_n_leaves())
print('score on train:', model.score(X_train, y_train))
print('score on test:', model.score(X_test, y_test))depth: 28 n_leaves: 777 score on train: 1.0 score on test: 0.8296318831761484
Cell:
[Cell type raw - unsupported, skipped]
In [13]:
## import from module
from bfh_cas_pml import inspect_decision_tree_model
## test call
inspect_decision_tree_model (
DecisionTreeClassifier(max_depth = 3), X_train, y_train, figsize = (6, 3))TREE DIAGNOSTICS: depth : 3 leaves : 8 score : 0.8647497337593184
In [14]:
## demo data
## produce trees with depths of 1, 2, 3 for presentation
inspect_decision_tree_model(
DecisionTreeClassifier(max_depth=1), X_demo, y_demo, figsize = (6, 4))TREE DIAGNOSTICS: depth : 1 leaves : 2 score : 0.9506172839506173
In [15]:
inspect_decision_tree_model (
DecisionTreeClassifier(max_depth = 2), X_demo, y_demo, figsize = (6, 6))TREE DIAGNOSTICS: depth : 2 leaves : 3 score : 0.9753086419753086
In [16]:
inspect_decision_tree_model (
DecisionTreeClassifier(max_depth = 3), X_demo, y_demo, figsize = (6, 6))TREE DIAGNOSTICS: depth : 3 leaves : 4 score : 0.9753086419753086
In [17]:
## bank customer dataset with depth = 5
inspect_decision_tree_model (
DecisionTreeClassifier(max_depth = 3), X_train, y_train, figsize = (6, 2))TREE DIAGNOSTICS: depth : 3 leaves : 8 score : 0.8647497337593184
In [18]:
## demo dataset
inspect_decision_tree_model (
DecisionTreeClassifier(min_samples_split = 41), X_demo, y_demo, figsize = (6, 4))TREE DIAGNOSTICS: depth : 1 leaves : 2 score : 0.9506172839506173
In [19]:
## demo dataset
inspect_decision_tree_model(
DecisionTreeClassifier(min_samples_split = 40), X_demo, y_demo, figsize = (6, 6))TREE DIAGNOSTICS: depth : 2 leaves : 3 score : 0.9753086419753086
Cell:
[Cell type raw - unsupported, skipped]
In [20]:
## demo dataset
inspect_decision_tree_model(
DecisionTreeClassifier(min_samples_leaf = 10), X_demo, y_demo, figsize = (6, 8))TREE DIAGNOSTICS: depth : 2 leaves : 3 score : 0.9506172839506173
In [21]:
## demo dataset
inspect_decision_tree_model(
DecisionTreeClassifier(min_samples_leaf = 5), X_demo, y_demo, figsize = (6, 8))TREE DIAGNOSTICS: depth : 3 leaves : 4 score : 0.9629629629629629
Cell:
[Cell type raw - unsupported, skipped]
In [22]:
## demo dataset
inspect_decision_tree_model(
DecisionTreeClassifier(max_leaf_nodes = 5), X_demo, y_demo)TREE DIAGNOSTICS: depth : 4 leaves : 5 score : 0.9876543209876543
Cell:
[Cell type raw - unsupported, skipped]
In [23]:
## demo dataset
inspect_decision_tree_model(
DecisionTreeClassifier(min_impurity_decrease = 0.042), X_demo, y_demo, figsize = (6, 7.5))TREE DIAGNOSTICS: depth : 2 leaves : 3 score : 0.9753086419753086
In [24]:
inspect_decision_tree_model (
DecisionTreeClassifier(min_impurity_decrease = 0.043), X_demo, y_demo)TREE DIAGNOSTICS: depth : 1 leaves : 2 score : 0.9506172839506173
Cell:
[Cell type raw - unsupported, skipped]
In [25]:
## max_depth in 1:20, incl. visualization
## prepare loop
model = DecisionTreeClassifier()
scores = [] ## empty list for collect iteration scorers
params = range(1, 21) ## define range
## iteration over params
for param in params:
model.set_params(max_depth = param)
model.fit(X_train, y_train)
score = model.score(X_test, y_test)
scores.append(score) ## add score to list
print(score) ## optional: progress display0.7064192272588987 0.8436264070581077 0.8594463036203225 0.8615759050806207 0.8646181928810466 0.8673562519014298 0.8670520231213873 0.8579251597201095 0.8588378460602373 0.8570124733799818 0.8530574992394281 0.8491025250988744 0.8411925768177669 0.8424094919379373 0.8378460602372985 0.8399756616975966 0.838150289017341 0.8323699421965318 0.8302403407362337 0.8308487982963189
In [26]:
fig = sns.lineplot(x=params, y=scores)
plt.scatter(x=params[scores.index(max(scores))], y=max(scores), color="black")
plt.xlabel('max_depth')
plt.ylabel('accuracy');
In [27]:
## show best parameter value and best score
print('best_param :', params[scores.index(max(scores))])
print('best_score :', max(scores))best_param : 6 best_score : 0.8673562519014298
Cell:
[Cell type raw - unsupported, skipped]
In [28]:
## compare tran and test errors
model = DecisionTreeClassifier()
scores_test = []
scores_train = []
params = range(1, 21)
for param in params:
model.set_params(max_depth=param)
model.fit(X_train, y_train)
scores_test.append(model.score(X_test, y_test))
scores_train.append(model.score(X_train, y_train))In [29]:
fig, ax = plt.subplots()
sns.lineplot(x=params, y=scores_test, ax=ax, label='test', color='b')
sns.lineplot(x=params, y=scores_train, ax=ax, label='train', color='r')
plt.scatter(x=params[scores_test.index(max(scores_test))], y=max(scores_test), color="black")
ax.set(xlabel='max. depth', ylabel='accuracy');
In [30]:
## train model
model = DecisionTreeClassifier()
model.fit(X_train, y_train) Out [30]:
DecisionTreeClassifier()In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
DecisionTreeClassifier()
In [31]:
## output of feature_importances_
print(X_train.columns)
print(model.feature_importances_)Index(['age', 'education', 'housing', 'contact_cellular', 'month',
'day_of_week', 'duration', 'campaign', 'pdays', 'previous',
'emp_var_rate', 'cons_price_idx', 'cons_conf_idx', 'euribor3m',
'nr_employed', 'job_blue_collar', 'job_entrepreneur', 'job_housemaid',
'job_management', 'job_retired', 'job_self_employed', 'job_services',
'job_student', 'job_technician', 'job_unemployed', 'marital_married',
'marital_single', 'loan_unknown', 'loan_yes'],
dtype='object')
[0.05632095 0.01934081 0.00996029 0.00709172 0.07161174 0.02696073
0.39938506 0.02068018 0.00733212 0.00699619 0.00848252 0.01526156
0.00957123 0.05411539 0.23217809 0.00356645 0.00402297 0.00239168
0.00294215 0.0068819 0.00181951 0.00607934 0.00330137 0.00545768
0.00230988 0.00526976 0.00338348 0.00098482 0.00630042]
In [32]:
## visualization
## collect values in another data frame
best = pd.DataFrame({
'feature': X_train.columns,
'importance': model.feature_importances_
})
## sort in descending order
best = best.sort_values(by=['importance'], ascending=False)
## plot
fig, ax = plt.subplots(figsize=(6, 6))
sns.barplot(x='importance', y='feature', data=best, color='steelblue');
In [33]:
## create Series of names for 10 best feaures
sel_vars = best.head(10).feature
## filter
X_train_new = X_train[sel_vars]
X_test_new = X_test[sel_vars]
## check
print(X_train_new.info())
print(X_test_new.info())<class 'pandas.core.frame.DataFrame'> Index: 6573 entries, 2751 to 8915 Data columns (total 10 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 duration 6573 non-null float64 1 nr_employed 6573 non-null float64 2 month 6573 non-null int64 3 age 6573 non-null float64 4 euribor3m 6573 non-null float64 5 day_of_week 6573 non-null int64 6 campaign 6573 non-null float64 7 education 6573 non-null int64 8 cons_price_idx 6573 non-null float64 9 housing 6573 non-null int64 dtypes: float64(6), int64(4) memory usage: 564.9 KB None <class 'pandas.core.frame.DataFrame'> Index: 3287 entries, 2809 to 9060 Data columns (total 10 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 duration 3287 non-null float64 1 nr_employed 3287 non-null float64 2 month 3287 non-null int64 3 age 3287 non-null float64 4 euribor3m 3287 non-null float64 5 day_of_week 3287 non-null int64 6 campaign 3287 non-null float64 7 education 3287 non-null int64 8 cons_price_idx 3287 non-null float64 9 housing 3287 non-null int64 dtypes: float64(6), int64(4) memory usage: 282.5 KB None
In [34]:
## prepare env
## (already done)In [35]:
## import trainer class
from sklearn.ensemble import RandomForestClassifierIn [36]:
## define and train model
model = RandomForestClassifier(random_state = 1234)
model.fit(X_train, y_train) Out [36]:
RandomForestClassifier(random_state=1234)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
RandomForestClassifier(random_state=1234)
In [37]:
## effective parameters
print(model.get_params()){'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': None, 'criterion': 'gini', 'max_depth': None, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'monotonic_cst': None, 'n_estimators': 100, 'n_jobs': None, 'oob_score': False, 'random_state': 1234, 'verbose': 0, 'warm_start': False}
In [38]:
## score with model intern scorer
print(model.score(X_test, y_test))0.8734408275022817
In [39]:
## additional information
print('model.n_classes_ :', model.n_classes_)
print('model.classes_ :', model.classes_)
print('model.n_features_in_ :', model.n_features_in_)
print('model.n_outputs_ :', model.n_outputs_)
## this later
## print(model.feature_importances_)model.n_classes_ : 2 model.classes_ : ['no' 'yes'] model.n_features_in_ : 29 model.n_outputs_ : 1
In [40]:
## oob_score_
model = RandomForestClassifier(random_state = 1234, oob_score = True)
model.fit(X_train, y_train)
print('model.oob_score_:', model.oob_score_)
#print('model.oob_decision_function_', model.oob_decision_function_)model.oob_score_: 0.8858968507530808
In [41]:
## full model with the found parameter values
model = RandomForestClassifier(
n_estimators = 200,
max_depth = 7)
model.fit(X_train, y_train)
## visualization
## collect values in data frame
best = pd.DataFrame({
'feature': X_train.columns,
'importance': model.feature_importances_
})
## sort in descending order
best = best.sort_values(by=['importance'], ascending=False)
## plot
fig, ax = plt.subplots(figsize=(6, 6))
sns.barplot(x='importance', y='feature', data=best, color='steelblue');
Cell:
[Cell type raw - unsupported, skipped]
In [42]:
from sklearn.ensemble import RandomForestClassifier
import time
jobs = [None, 1, -1]
for j in jobs:
start_time = time.time()
model = RandomForestClassifier(n_jobs = j, random_state=1234)
model.fit(X_train, y_train)
pred = model.predict(X_test)
t = time.time() - start_time ## used time
print(j, t)None 0.3224680423736572 1 0.31882715225219727 -1 0.10985803604125977
Cell:
[Cell type raw - unsupported, skipped]
In [43]:
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import accuracy_score
model = AdaBoostClassifier(algorithm='SAMME')
model.fit(X_train, y_train)
print(model.score(X_test, y_test))/Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages/sklearn/ensemble/_weight_boosting.py:519: FutureWarning: The parameter 'algorithm' is deprecated in 1.6 and has no effect. It will be removed in version 1.8. warnings.warn(
0.8539701855795558
In [44]:
print(model.get_params()){'algorithm': 'SAMME', 'estimator': None, 'learning_rate': 1.0, 'n_estimators': 50, 'random_state': None}
In [45]:
## weitere Informationen zum modell
print(model.estimator_)DecisionTreeClassifier(max_depth=1)
In [46]:
from sklearn.ensemble import GradientBoostingClassifier
model = GradientBoostingClassifier()
model.fit(X_train, y_train)
print(model.score(X_test, y_test))0.8767873440827503
In [47]:
from sklearn.ensemble import HistGradientBoostingClassifier
model = HistGradientBoostingClassifier()
model.fit(X_train, y_train)
print(model.score(X_test, y_test))0.8819592333434743
In [48]:
!pip install catboost
!pip install lightgbm
!pip install xgboostRequirement already satisfied: catboost in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (1.2.8) Requirement already satisfied: graphviz in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from catboost) (0.21) Requirement already satisfied: matplotlib in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from catboost) (3.10.3) Requirement already satisfied: numpy<3.0,>=1.16.0 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from catboost) (2.2.6) Requirement already satisfied: pandas>=0.24 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from catboost) (2.2.3) Requirement already satisfied: scipy in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from catboost) (1.15.3) Requirement already satisfied: plotly in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from catboost) (6.1.2) Requirement already satisfied: six in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from catboost) (1.17.0) Requirement already satisfied: python-dateutil>=2.8.2 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from pandas>=0.24->catboost) (2.9.0.post0) Requirement already satisfied: pytz>=2020.1 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from pandas>=0.24->catboost) (2025.2) Requirement already satisfied: tzdata>=2022.7 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from pandas>=0.24->catboost) (2025.2) Requirement already satisfied: contourpy>=1.0.1 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from matplotlib->catboost) (1.3.2) Requirement already satisfied: cycler>=0.10 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from matplotlib->catboost) (0.12.1) Requirement already satisfied: fonttools>=4.22.0 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from matplotlib->catboost) (4.58.1) Requirement already satisfied: kiwisolver>=1.3.1 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from matplotlib->catboost) (1.4.8) Requirement already satisfied: packaging>=20.0 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from matplotlib->catboost) (25.0) Requirement already satisfied: pillow>=8 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from matplotlib->catboost) (11.2.1) Requirement already satisfied: pyparsing>=2.3.1 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from matplotlib->catboost) (3.2.3) Requirement already satisfied: narwhals>=1.15.1 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from plotly->catboost) (1.43.1) Requirement already satisfied: lightgbm in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (4.6.0) Requirement already satisfied: numpy>=1.17.0 in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from lightgbm) (2.2.6) Requirement already satisfied: scipy in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from lightgbm) (1.15.3) Requirement already satisfied: xgboost in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (3.0.2) Requirement already satisfied: numpy in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from xgboost) (2.2.6) Requirement already satisfied: scipy in /Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages (from xgboost) (1.15.3)
In [49]:
## import trainer classes
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import HistGradientBoostingClassifier
from catboost import CatBoostClassifier
from lightgbm.sklearn import LGBMClassifier
import time ## for timekeepingIn [50]:
## define models
models = [
KNeighborsClassifier(n_neighbors=5),
DecisionTreeClassifier(min_impurity_decrease=0.002),
RandomForestClassifier(n_estimators=200),
AdaBoostClassifier(algorithm='SAMME'),
GradientBoostingClassifier(),
HistGradientBoostingClassifier(),
CatBoostClassifier(logging_level='Silent'), ## optional
LGBMClassifier() ## optional
]In [51]:
## to collect the results
scores = []
used_times = []
model_names = []In [52]:
## report titel
print('Classifier Score Time')
print('==================================================')
## iteration over models
for model in models:
start_time = time.time() ## start timer
model.fit(X_train, y_train) ## train
name = model.__class__.__name__ ## pick model name (for output only)
score = model.score(X_test, y_test) ## calculate score
t = time.time() - start_time ## calculate used time
## collect iteration results
scores.append(score)
used_times.append(t)
model_names.append(name)
print('%-32s %6.4f %6.3f' % (name, score, t)) ## console outputClassifier Score Time ================================================== KNeighborsClassifier 0.7493 0.075 DecisionTreeClassifier 0.8588 0.012 RandomForestClassifier 0.8795 0.656
/Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages/sklearn/ensemble/_weight_boosting.py:519: FutureWarning: The parameter 'algorithm' is deprecated in 1.6 and has no effect. It will be removed in version 1.8. warnings.warn(
AdaBoostClassifier 0.8540 0.234 GradientBoostingClassifier 0.8768 0.504 HistGradientBoostingClassifier 0.8820 0.371 CatBoostClassifier 0.8829 1.504 [LightGBM] [Info] Number of positive: 3087, number of negative: 3486 [LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000475 seconds. You can set `force_row_wise=true` to remove the overhead. And if memory is not enough, you can set `force_col_wise=true`. [LightGBM] [Info] Total Bins 701 [LightGBM] [Info] Number of data points in the train set: 6573, number of used features: 29 [LightGBM] [Info] [binary:BoostFromScore]: pavg=0.469649 -> initscore=-0.121555 [LightGBM] [Info] Start training from score -0.121555 LGBMClassifier 0.8850 0.366
In [53]:
## final evaluation
print('best_model :', model_names[scores.index(max(scores))])
print('best_score :', max(scores))
print('used_time :', used_times[scores.index(max(scores))])best_model : LGBMClassifier best_score : 0.8850015211439002 used_time : 0.36551785469055176
In [54]:
## vizualisation, ordered by decreasing accuracy
results = pd.DataFrame({
'models' : model_names,
'scores' : scores,
'used_times' : used_times}
).sort_values(by='scores', ascending=False)
#print(results)
order = results.sort_values('scores', ascending=False).models
sns.set(font_scale=1.5)
f, axes = plt.subplots(nrows=1, ncols=2, figsize=(10, 4), sharex=False)
ax = sns.barplot(data=results, x=scores, y=model_names, color='lightblue', order=order, ax=axes[0])
ax.set_xlabel('accuracy')
ax.set(xlim=(0.5, 1))
ax = sns.barplot(x=used_times, y=model_names, color='darksalmon', order=order, ax=axes[1])
ax.set_xlabel('time')
ax.set_yticklabels('')
ax.set_ylabel('');