91 KiB
91 KiB
In [3]:
## prepare env, read and prepare data
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
#codepath = '../2_code' ## for import of user defined module
#datapath = '../3_data'
codepath = '.././2_code' ## for import of user defined module
datapath = '../../3_data'
from sys import path; path.insert(1, codepath)
from os import chdir; chdir(datapath)
from bfh_cas_pml import prep_data
X_train, X_test, y_train, y_test = prep_data('melb_data_prep.csv', target='Price', seed=1234)
from bfh_cas_pml import test_regression_modelIn [4]:
## baseline
from sklearn.ensemble import AdaBoostRegressor
this_model = test_regression_model(
AdaBoostRegressor(random_state=1234),
X_train, y_train, X_test, y_test,
show_plot=False)R2 = -0.3023
In [5]:
## tune learning_rate
model = AdaBoostRegressor()
#params = np.arange(0.1, 1, 0.1) ## vorlage
params = np.arange(0.05, 0.15, 0.01) ## loesung
scores = []
for param in params:
model.set_params(learning_rate=param, random_state=1234)
model.fit(X_train, y_train)
score = model.score(X_test, y_test)
scores.append(score)
print(param, score)
sns.lineplot(x=params, y=scores);0.05 0.5221143978043213 0.060000000000000005 0.5240912532917807 0.07 0.5255798946120415 0.08000000000000002 0.5204059905040977 0.09000000000000001 0.522828809581406 0.1 0.5141772439961587 0.11000000000000001 0.5146383601832738 0.12000000000000001 0.5140298684772102 0.13 0.5113124946735497 0.14 0.5030628610086694
C:\Users\werne\anaconda3\Lib\site-packages\seaborn\_oldcore.py:1119: FutureWarning: use_inf_as_na option is deprecated and will be removed in a future version. Convert inf values to NaN before operating instead.
with pd.option_context('mode.use_inf_as_na', True):
C:\Users\werne\anaconda3\Lib\site-packages\seaborn\_oldcore.py:1119: FutureWarning: use_inf_as_na option is deprecated and will be removed in a future version. Convert inf values to NaN before operating instead.
with pd.option_context('mode.use_inf_as_na', True):
In [6]:
## tune max_depth
from sklearn.tree import DecisionTreeRegressor
model = AdaBoostRegressor()
#params = range(1, 11) ## vorlage
params = range(10, 20) ## loesung
scores = []
for param in params:
model.set_params(
estimator=DecisionTreeRegressor(max_depth=param),
random_state=1234
)
model.fit(X_train, y_train)
score = model.score(X_test, y_test)
scores.append(score)
print(param, score)
sns.lineplot(x=params, y=scores);10 0.7557299631257758 11 0.758500977810915 12 0.7626992985412937 13 0.7674329084019647 14 0.7671307844633629 15 0.7689296285653345 16 0.7655448595315713 17 0.7634543831075334 18 0.7565334380954082 19 0.7633021297334406
C:\Users\werne\anaconda3\Lib\site-packages\seaborn\_oldcore.py:1119: FutureWarning: use_inf_as_na option is deprecated and will be removed in a future version. Convert inf values to NaN before operating instead.
with pd.option_context('mode.use_inf_as_na', True):
C:\Users\werne\anaconda3\Lib\site-packages\seaborn\_oldcore.py:1119: FutureWarning: use_inf_as_na option is deprecated and will be removed in a future version. Convert inf values to NaN before operating instead.
with pd.option_context('mode.use_inf_as_na', True):
In [7]:
## best combination of single parameters
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import AdaBoostRegressor
model=AdaBoostRegressor(
estimator=DecisionTreeRegressor(max_depth=15),
learning_rate=0.07,
random_state=1234
)
model.fit(X_train, y_train)
print(model.score(X_test, y_test))0.7633058383893074
In [9]:
help(AdaBoostRegressor)Help on class AdaBoostRegressor in module sklearn.ensemble._weight_boosting:
class AdaBoostRegressor(sklearn.utils._metadata_requests._RoutingNotSupportedMixin, sklearn.base.RegressorMixin, BaseWeightBoosting)
| AdaBoostRegressor(estimator=None, *, n_estimators=50, learning_rate=1.0, loss='linear', random_state=None)
|
| An AdaBoost regressor.
|
| An AdaBoost [1] regressor is a meta-estimator that begins by fitting a
| regressor on the original dataset and then fits additional copies of the
| regressor on the same dataset but where the weights of instances are
| adjusted according to the error of the current prediction. As such,
| subsequent regressors focus more on difficult cases.
|
| This class implements the algorithm known as AdaBoost.R2 [2].
|
| Read more in the :ref:`User Guide <adaboost>`.
|
| .. versionadded:: 0.14
|
| Parameters
| ----------
| estimator : object, default=None
| The base estimator from which the boosted ensemble is built.
| If ``None``, then the base estimator is
| :class:`~sklearn.tree.DecisionTreeRegressor` initialized with
| `max_depth=3`.
|
| .. versionadded:: 1.2
| `base_estimator` was renamed to `estimator`.
|
| n_estimators : int, default=50
| The maximum number of estimators at which boosting is terminated.
| In case of perfect fit, the learning procedure is stopped early.
| Values must be in the range `[1, inf)`.
|
| learning_rate : float, default=1.0
| Weight applied to each regressor at each boosting iteration. A higher
| learning rate increases the contribution of each regressor. There is
| a trade-off between the `learning_rate` and `n_estimators` parameters.
| Values must be in the range `(0.0, inf)`.
|
| loss : {'linear', 'square', 'exponential'}, default='linear'
| The loss function to use when updating the weights after each
| boosting iteration.
|
| random_state : int, RandomState instance or None, default=None
| Controls the random seed given at each `estimator` at each
| boosting iteration.
| Thus, it is only used when `estimator` exposes a `random_state`.
| In addition, it controls the bootstrap of the weights used to train the
| `estimator` at each boosting iteration.
| Pass an int for reproducible output across multiple function calls.
| See :term:`Glossary <random_state>`.
|
| Attributes
| ----------
| estimator_ : estimator
| The base estimator from which the ensemble is grown.
|
| .. versionadded:: 1.2
| `base_estimator_` was renamed to `estimator_`.
|
| estimators_ : list of regressors
| The collection of fitted sub-estimators.
|
| estimator_weights_ : ndarray of floats
| Weights for each estimator in the boosted ensemble.
|
| estimator_errors_ : ndarray of floats
| Regression error for each estimator in the boosted ensemble.
|
| feature_importances_ : ndarray of shape (n_features,)
| The impurity-based feature importances if supported by the
| ``estimator`` (when based on decision trees).
|
| Warning: impurity-based feature importances can be misleading for
| high cardinality features (many unique values). See
| :func:`sklearn.inspection.permutation_importance` as an alternative.
|
| n_features_in_ : int
| Number of features seen during :term:`fit`.
|
| .. versionadded:: 0.24
|
| feature_names_in_ : ndarray of shape (`n_features_in_`,)
| Names of features seen during :term:`fit`. Defined only when `X`
| has feature names that are all strings.
|
| .. versionadded:: 1.0
|
| See Also
| --------
| AdaBoostClassifier : An AdaBoost classifier.
| GradientBoostingRegressor : Gradient Boosting Classification Tree.
| sklearn.tree.DecisionTreeRegressor : A decision tree regressor.
|
| References
| ----------
| .. [1] Y. Freund, R. Schapire, "A Decision-Theoretic Generalization of
| on-Line Learning and an Application to Boosting", 1995.
|
| .. [2] H. Drucker, "Improving Regressors using Boosting Techniques", 1997.
|
| Examples
| --------
| >>> from sklearn.ensemble import AdaBoostRegressor
| >>> from sklearn.datasets import make_regression
| >>> X, y = make_regression(n_features=4, n_informative=2,
| ... random_state=0, shuffle=False)
| >>> regr = AdaBoostRegressor(random_state=0, n_estimators=100)
| >>> regr.fit(X, y)
| AdaBoostRegressor(n_estimators=100, random_state=0)
| >>> regr.predict([[0, 0, 0, 0]])
| array([4.7972...])
| >>> regr.score(X, y)
| 0.9771...
|
| Method resolution order:
| AdaBoostRegressor
| sklearn.utils._metadata_requests._RoutingNotSupportedMixin
| sklearn.base.RegressorMixin
| BaseWeightBoosting
| sklearn.ensemble._base.BaseEnsemble
| sklearn.base.MetaEstimatorMixin
| sklearn.base.BaseEstimator
| sklearn.utils._estimator_html_repr._HTMLDocumentationLinkMixin
| sklearn.utils._metadata_requests._MetadataRequester
| builtins.object
|
| Methods defined here:
|
| __init__(self, estimator=None, *, n_estimators=50, learning_rate=1.0, loss='linear', random_state=None)
| Initialize self. See help(type(self)) for accurate signature.
|
| predict(self, X)
| Predict regression value for X.
|
| The predicted regression value of an input sample is computed
| as the weighted median prediction of the regressors in the ensemble.
|
| Parameters
| ----------
| X : {array-like, sparse matrix} of shape (n_samples, n_features)
| The training input samples. Sparse matrix can be CSC, CSR, COO,
| DOK, or LIL. COO, DOK, and LIL are converted to CSR.
|
| Returns
| -------
| y : ndarray of shape (n_samples,)
| The predicted regression values.
|
| set_fit_request(self: sklearn.ensemble._weight_boosting.AdaBoostRegressor, *, sample_weight: Union[bool, NoneType, str] = '$UNCHANGED$') -> sklearn.ensemble._weight_boosting.AdaBoostRegressor
| Request metadata passed to the ``fit`` method.
|
| Note that this method is only relevant if
| ``enable_metadata_routing=True`` (see :func:`sklearn.set_config`).
| Please see :ref:`User Guide <metadata_routing>` on how the routing
| mechanism works.
|
| The options for each parameter are:
|
| - ``True``: metadata is requested, and passed to ``fit`` if provided. The request is ignored if metadata is not provided.
|
| - ``False``: metadata is not requested and the meta-estimator will not pass it to ``fit``.
|
| - ``None``: metadata is not requested, and the meta-estimator will raise an error if the user provides it.
|
| - ``str``: metadata should be passed to the meta-estimator with this given alias instead of the original name.
|
| The default (``sklearn.utils.metadata_routing.UNCHANGED``) retains the
| existing request. This allows you to change the request for some
| parameters and not others.
|
| .. versionadded:: 1.3
|
| .. note::
| This method is only relevant if this estimator is used as a
| sub-estimator of a meta-estimator, e.g. used inside a
| :class:`~sklearn.pipeline.Pipeline`. Otherwise it has no effect.
|
| Parameters
| ----------
| sample_weight : str, True, False, or None, default=sklearn.utils.metadata_routing.UNCHANGED
| Metadata routing for ``sample_weight`` parameter in ``fit``.
|
| Returns
| -------
| self : object
| The updated object.
|
| set_score_request(self: sklearn.ensemble._weight_boosting.AdaBoostRegressor, *, sample_weight: Union[bool, NoneType, str] = '$UNCHANGED$') -> sklearn.ensemble._weight_boosting.AdaBoostRegressor
| Request metadata passed to the ``score`` method.
|
| Note that this method is only relevant if
| ``enable_metadata_routing=True`` (see :func:`sklearn.set_config`).
| Please see :ref:`User Guide <metadata_routing>` on how the routing
| mechanism works.
|
| The options for each parameter are:
|
| - ``True``: metadata is requested, and passed to ``score`` if provided. The request is ignored if metadata is not provided.
|
| - ``False``: metadata is not requested and the meta-estimator will not pass it to ``score``.
|
| - ``None``: metadata is not requested, and the meta-estimator will raise an error if the user provides it.
|
| - ``str``: metadata should be passed to the meta-estimator with this given alias instead of the original name.
|
| The default (``sklearn.utils.metadata_routing.UNCHANGED``) retains the
| existing request. This allows you to change the request for some
| parameters and not others.
|
| .. versionadded:: 1.3
|
| .. note::
| This method is only relevant if this estimator is used as a
| sub-estimator of a meta-estimator, e.g. used inside a
| :class:`~sklearn.pipeline.Pipeline`. Otherwise it has no effect.
|
| Parameters
| ----------
| sample_weight : str, True, False, or None, default=sklearn.utils.metadata_routing.UNCHANGED
| Metadata routing for ``sample_weight`` parameter in ``score``.
|
| Returns
| -------
| self : object
| The updated object.
|
| staged_predict(self, X)
| Return staged predictions for X.
|
| The predicted regression value of an input sample is computed
| as the weighted median prediction of the regressors in the ensemble.
|
| This generator method yields the ensemble prediction after each
| iteration of boosting and therefore allows monitoring, such as to
| determine the prediction on a test set after each boost.
|
| Parameters
| ----------
| X : {array-like, sparse matrix} of shape (n_samples, n_features)
| The training input samples.
|
| Yields
| ------
| y : generator of ndarray of shape (n_samples,)
| The predicted regression values.
|
| ----------------------------------------------------------------------
| Data and other attributes defined here:
|
| __abstractmethods__ = frozenset()
|
| __annotations__ = {'_parameter_constraints': <class 'dict'>}
|
| ----------------------------------------------------------------------
| Methods inherited from sklearn.utils._metadata_requests._RoutingNotSupportedMixin:
|
| get_metadata_routing(self)
| Raise `NotImplementedError`.
|
| This estimator does not support metadata routing yet.
|
| ----------------------------------------------------------------------
| Data descriptors inherited from sklearn.utils._metadata_requests._RoutingNotSupportedMixin:
|
| __dict__
| dictionary for instance variables
|
| __weakref__
| list of weak references to the object
|
| ----------------------------------------------------------------------
| Methods inherited from sklearn.base.RegressorMixin:
|
| score(self, X, y, sample_weight=None)
| Return the coefficient of determination of the prediction.
|
| The coefficient of determination :math:`R^2` is defined as
| :math:`(1 - \frac{u}{v})`, where :math:`u` is the residual
| sum of squares ``((y_true - y_pred)** 2).sum()`` and :math:`v`
| is the total sum of squares ``((y_true - y_true.mean()) ** 2).sum()``.
| The best possible score is 1.0 and it can be negative (because the
| model can be arbitrarily worse). A constant model that always predicts
| the expected value of `y`, disregarding the input features, would get
| a :math:`R^2` score of 0.0.
|
| Parameters
| ----------
| X : array-like of shape (n_samples, n_features)
| Test samples. For some estimators this may be a precomputed
| kernel matrix or a list of generic objects instead with shape
| ``(n_samples, n_samples_fitted)``, where ``n_samples_fitted``
| is the number of samples used in the fitting for the estimator.
|
| y : array-like of shape (n_samples,) or (n_samples, n_outputs)
| True values for `X`.
|
| sample_weight : array-like of shape (n_samples,), default=None
| Sample weights.
|
| Returns
| -------
| score : float
| :math:`R^2` of ``self.predict(X)`` w.r.t. `y`.
|
| Notes
| -----
| The :math:`R^2` score used when calling ``score`` on a regressor uses
| ``multioutput='uniform_average'`` from version 0.23 to keep consistent
| with default value of :func:`~sklearn.metrics.r2_score`.
| This influences the ``score`` method of all the multioutput
| regressors (except for
| :class:`~sklearn.multioutput.MultiOutputRegressor`).
|
| ----------------------------------------------------------------------
| Methods inherited from BaseWeightBoosting:
|
| fit(self, X, y, sample_weight=None)
| Build a boosted classifier/regressor from the training set (X, y).
|
| Parameters
| ----------
| X : {array-like, sparse matrix} of shape (n_samples, n_features)
| The training input samples. Sparse matrix can be CSC, CSR, COO,
| DOK, or LIL. COO, DOK, and LIL are converted to CSR.
|
| y : array-like of shape (n_samples,)
| The target values.
|
| sample_weight : array-like of shape (n_samples,), default=None
| Sample weights. If None, the sample weights are initialized to
| 1 / n_samples.
|
| Returns
| -------
| self : object
| Fitted estimator.
|
| staged_score(self, X, y, sample_weight=None)
| Return staged scores for X, y.
|
| This generator method yields the ensemble score after each iteration of
| boosting and therefore allows monitoring, such as to determine the
| score on a test set after each boost.
|
| Parameters
| ----------
| X : {array-like, sparse matrix} of shape (n_samples, n_features)
| The training input samples. Sparse matrix can be CSC, CSR, COO,
| DOK, or LIL. COO, DOK, and LIL are converted to CSR.
|
| y : array-like of shape (n_samples,)
| Labels for X.
|
| sample_weight : array-like of shape (n_samples,), default=None
| Sample weights.
|
| Yields
| ------
| z : float
|
| ----------------------------------------------------------------------
| Readonly properties inherited from BaseWeightBoosting:
|
| feature_importances_
| The impurity-based feature importances.
|
| The higher, the more important the feature.
| The importance of a feature is computed as the (normalized)
| total reduction of the criterion brought by that feature. It is also
| known as the Gini importance.
|
| Warning: impurity-based feature importances can be misleading for
| high cardinality features (many unique values). See
| :func:`sklearn.inspection.permutation_importance` as an alternative.
|
| Returns
| -------
| feature_importances_ : ndarray of shape (n_features,)
| The feature importances.
|
| ----------------------------------------------------------------------
| Methods inherited from sklearn.ensemble._base.BaseEnsemble:
|
| __getitem__(self, index)
| Return the index'th estimator in the ensemble.
|
| __iter__(self)
| Return iterator over estimators in the ensemble.
|
| __len__(self)
| Return the number of estimators in the ensemble.
|
| ----------------------------------------------------------------------
| Methods inherited from sklearn.base.BaseEstimator:
|
| __getstate__(self)
| Helper for pickle.
|
| __repr__(self, N_CHAR_MAX=700)
| Return repr(self).
|
| __setstate__(self, state)
|
| __sklearn_clone__(self)
|
| get_params(self, deep=True)
| Get parameters for this estimator.
|
| Parameters
| ----------
| deep : bool, default=True
| If True, will return the parameters for this estimator and
| contained subobjects that are estimators.
|
| Returns
| -------
| params : dict
| Parameter names mapped to their values.
|
| set_params(self, **params)
| Set the parameters of this estimator.
|
| The method works on simple estimators as well as on nested objects
| (such as :class:`~sklearn.pipeline.Pipeline`). The latter have
| parameters of the form ``<component>__<parameter>`` so that it's
| possible to update each component of a nested object.
|
| Parameters
| ----------
| **params : dict
| Estimator parameters.
|
| Returns
| -------
| self : estimator instance
| Estimator instance.
|
| ----------------------------------------------------------------------
| Class methods inherited from sklearn.utils._metadata_requests._MetadataRequester:
|
| __init_subclass__(**kwargs) from abc.ABCMeta
| Set the ``set_{method}_request`` methods.
|
| This uses PEP-487 [1]_ to set the ``set_{method}_request`` methods. It
| looks for the information available in the set default values which are
| set using ``__metadata_request__*`` class attributes, or inferred
| from method signatures.
|
| The ``__metadata_request__*`` class attributes are used when a method
| does not explicitly accept a metadata through its arguments or if the
| developer would like to specify a request value for those metadata
| which are different from the default ``None``.
|
| References
| ----------
| .. [1] https://www.python.org/dev/peps/pep-0487