cas-pml/SL/aufgaben/template/2_Code/3.3 Regression - ML Methoden.ipynb

Feature Engineering

Klassifikation

Regression

Einleitung

Klassische Methoden

ML Methoden

Cell:

[Cell type raw - unsupported, skipped]

In [1]:

import sys
sys.path.append('./')

In [2]:

## prepare environment and data
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
%matplotlib inline

datapath = '../3_data'
from os import chdir; chdir(datapath)

from bfh_cas_pml import prep_data, prep_demo_data
X_train, X_test, y_train, y_test = prep_data('melb_data_prep.csv', 'Price', seed = 1234)

X_demo, y_demo = prep_demo_data('demo_data_regr.csv', 'y')

In [3]:

## preparation for theoretical part

## create a synthetic dataset
X_synth = np.linspace(X_demo.min(), X_demo.max(), 100).reshape(-1,1)

## function for testing different regressors with synthetic dataset (from module bfh_cas_pml)
from bfh_cas_pml import show_pred_on_synth

## test call
from sklearn.neighbors import KNeighborsRegressor
show_pred_on_synth(KNeighborsRegressor(n_neighbors=3), X_demo, y_demo, X_synth, 'n_neighbors=3')

In [4]:

## preparations for praxis part
## function for testing different regressors (in module bfh_cas_pml)
from bfh_cas_pml import test_regression_model
from sklearn.linear_model import LinearRegression
this_model = test_regression_model(
    LinearRegression(), 
    X_train, y_train, X_test, y_test,
    show_plot=False)

R2 = 0.5601

KNeighborsRegressor

Theorie

In [5]:

from sklearn.neighbors import KNeighborsRegressor
show_pred_on_synth(KNeighborsRegressor(n_neighbors=3), X_demo, y_demo, X_synth, 'n_neighbors=3')
show_pred_on_synth(KNeighborsRegressor(n_neighbors=10), X_demo, y_demo, X_synth, 'n_neighbors=10')

Praxis

In [6]:

## melb dataset (already prepared) using the function testRegressionModel
from sklearn.neighbors import KNeighborsRegressor
this_model = test_regression_model(
    KNeighborsRegressor(), 
    X_train, y_train, X_test, y_test)

R2 = 0.4497

In [7]:

print(this_model.get_params())

{'algorithm': 'auto', 'leaf_size': 30, 'metric': 'minkowski', 'metric_params': None, 'n_jobs': None, 'n_neighbors': 5, 'p': 2, 'weights': 'uniform'}

DecisionTreeRegressor

Theorie

In [8]:

#from bfh_cas_pml import show_pred_on_synth
from sklearn.tree import DecisionTreeRegressor
show_pred_on_synth(DecisionTreeRegressor(max_depth=2), X_demo, y_demo, X_synth, 'max_depth=2')
show_pred_on_synth(DecisionTreeRegressor(max_depth=None), X_demo, y_demo, X_synth, 'max_depth=None')

Praxis

In [9]:

from sklearn.tree import DecisionTreeRegressor
this_model = test_regression_model(DecisionTreeRegressor(random_state=1234), 
    X_test, y_test, X_train, y_train)

R2 = 0.5502

Methoden

In [10]:

print('get_depth()    : ', this_model.get_depth())
print('get_n_leaves() : ', this_model.get_n_leaves())

get_depth()    :  33
get_n_leaves() :  5947

In [11]:

## tuning max depth
err_train = []
err_test = []

params = range(1, 21)

for param in params:
    model = DecisionTreeRegressor(max_depth = param)
    model.fit(X_train ,y_train)
    err_train.append(model.score(X_train, y_train))
    err_test.append(model.score(X_test, y_test))

fig, ax = plt.subplots()
sns.lineplot(x=params, y=err_test, ax=ax, label='test', color='b')
sns.lineplot(x=params, y=err_train, ax=ax, label='train', color='r')
ax.set(xlabel='max_depth', ylabel='r2_score');

In [12]:

## outlook on grid search and cross validation (capter 4)
from sklearn.model_selection import GridSearchCV
parameters = {'min_samples_leaf': range(1,11)}
model = DecisionTreeRegressor(random_state=1234)
clf = GridSearchCV(model, parameters) ## cv=5

clf.fit(X_train, y_train)
cv_df = pd.DataFrame(clf.cv_results_)

plt.errorbar(
    'param_min_samples_leaf', 
    'mean_test_score', 
    'std_test_score', data = cv_df, linestyle='None', marker='^');

RandomForestRegressor

Theorie

In [13]:

from sklearn.ensemble import RandomForestRegressor
show_pred_on_synth(RandomForestRegressor(), X_demo, y_demo, X_synth, 'default')

Praxis

In [14]:

from sklearn.ensemble import RandomForestRegressor
this_model = test_regression_model(
    RandomForestRegressor(n_estimators=100, random_state=1234), 
    X_train, y_train, X_test, y_test)

R2 = 0.7778

Weitere Ensemble Regressoren

vgl. extra_3.3.4_weitere_ensemble_regressoren.ipynb

SVR

Theorie

In [15]:

from sklearn.svm import SVR
show_pred_on_synth(SVR(), X_demo, y_demo, X_synth, 'default')

Praxis

In [16]:

## baseline
from sklearn.svm import SVR
this_model = test_regression_model(
    SVR(),
    X_train, y_train, X_test, y_test)

R2 = -0.0773

• for variants of preprocessing, see extra_3.3.5.2_variants_of_SVR.ipynb

Skalieren und Trainieren in einer Pipeline (Ausblick)

Cell:

[Cell type raw - unsupported, skipped]

MLPRegressor

Theorie

In [17]:

from sklearn.neural_network import MLPRegressor
show_pred_on_synth(MLPRegressor(), X_demo, y_demo, X_synth, 'default')

Praxis

In [18]:

from sklearn.neural_network import MLPRegressor
this_model = test_regression_model(
    MLPRegressor(random_state=1234),
    X_train, y_train, X_test, y_test,
    show_plot=False)

R2 = 0.0258

/Users/vgv1/.pyenv/versions/teaching/lib/python3.13/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.
  warnings.warn(

• for variants of preprocessing, see extra_3.3.6.2_variants_of_MLPRegressor.ipynb