cas-pml/SL/aufgaben/template/4_WS/Loesungen/WS 03 Loesung.ipynb

Workshop 03 - Loesungsvorschlag

Data Frame

In [3]:

## prepare and read data
import pandas as pd
import numpy as np

datapath = '../../3_data'
filename = 'melb_data.csv'
from os import chdir; chdir(datapath)

data = pd.read_csv(filename)

Entfernen von Beobachtungen nach Bedingung

In [5]:

## remove outliers on Price
data = data[data.Price < 8000000]

## remove selected observation2
data = data[data.YearBuilt != 1196]

Entfernen von Duplikaten

hier kein Bedarf

Cell:

[Cell type raw - unsupported, skipped]

Entfernen fragwürdiger Variablen

In [9]:

vars_to_drop = ['Unnamed: 0', 'Suburb', 'Address', 'SellerG', 'Postcode', 'Bedroom2']
data = data.drop(vars_to_drop, axis=1)

Einsetzen von Werten für NAs

In [11]:

## mode for all cat vars (if any)
cat_feats = data.select_dtypes(include=['object']).columns
for c in cat_feats:
    data[c].fillna(data[c].mode()[0], inplace = True)

## median for all num features
num_feats = data.select_dtypes(include=['int64', 'float64']).columns
for c in num_feats:
    data[c].fillna(data[c].median(), inplace = True)    

Kategoriale Variablen

Reduzieren der Kardinalität

In [14]:

## Regionname: combine rarest 3 levels to 'Victoria'
data.Regionname = np.where(
        (data.Regionname == 'Eastern Victoria') |
        (data.Regionname == 'Northern Victoria') |
        (data.Regionname == 'Western Victoria'),
    'Victoria', data.Regionname)

## Method: combine 'SA' to 'S'
data.Method = np.where(data.Method == 'SA', 'S', data.Method)

Nummerisiren - Faktorisieren

In [16]:

data.CouncilArea = data.CouncilArea.factorize()[0]

Nummerisiren - Ordial Encodieren

In [18]:

data.Type.replace(
    ['h', 'u', 't'], 
    [1, 2, 3], 
    inplace=True)

Nummerisieren - Binär Encodieren

hier kein Bedarf

Nummerisieren - Ordinal Encodieren

In [22]:

## one-hot encoding
## apply for all categorical variables except Date (will be transformed later)
ignore = 'Date'
sel_vars = data.select_dtypes(include=['object']).columns.drop(ignore)
data = pd.get_dummies(data, columns=sel_vars, drop_first=True)

Numerische Variablen

Logarithmieren

In [25]:

## logarithme and rename
data.Landsize = np.log10(data.Landsize + 1)
data.BuildingArea = np.log10(data.BuildingArea + 1)
data.rename(columns={
    'Landsize' : 'logLandsize',
    'BuildingArea' : 'logBuildingArea'
}, inplace=True)

Binär umcodieren

hier kein Bedarf

Andere Tätigkeiten

Konstruktion

In [30]:

## construct month, year and day_of_week
Date = pd.to_datetime(data.Date, format='%d/%m/%Y')
data['month'] = Date.dt.month
data['year'] = Date.dt.year
data['day_of_week'] = Date.dt.day_of_week
data.drop('Date', axis=1, inplace=True) ## not longer used
#print(data.info()) ## check

Bereinigen der Variablennamen

In [32]:

old_names = data.columns
new_names = old_names.str.replace('[^a-zA-Z0-9_]', '_', regex=True)
for i in range(len(old_names)):
    data.rename(columns={old_names[i]:new_names[i]}, inplace=True)

Standardisieren

hier kein Bedarf

Speichern unter neuem Namen

In [ ]:

data.to_csv('melb_data_prep.csv', index=False)