cas-pml/SL/aufgaben/template/4_WS/WS 02 Feature Exploration.ipynb

WS 02 Feature Engineering Exploration Overview.ipynb

• compiles the most important characteristic values from a loaded data frame and stores them in an Excel spreadsheet

In [1]:

## import libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

## define data path
datapath = '../3_data'
from os import chdir; chdir(datapath)

## load data
data = pd.read_csv('melb_data.csv')
#data.info()

## var names
var_names = pd.Series(data.columns)
#print(var_names)

dtypes = pd.Series(data.dtypes.values)
#print(dtypes)

## nas
nas = pd.Series(data.isna().sum().values)
#print(nas)

## uniques
uniques = []
for c in var_names:
    uniques.append(data[c].nunique())
uniques = pd.Series(uniques)
#print(uniques)

modes = []
for i in range(len(var_names)):
    if dtypes[i] == 'object':
        #modes.append(data[c].mode()[0])
        modes.append(data[var_names[i]].mode()[0])
        
    else:
        modes.append(None)
modes = pd.Series(modes)
#print(modes)

## means
means = []
for i in range(len(var_names)):
        if dtypes[i] != 'object':
            means.append(data[var_names[i]].mean())
        else:
            means.append(None)
means = pd.Series(means)
#print(means)

## medians
medians = []
for i in range(len(var_names)):
        if dtypes[i] != 'object':
            medians.append(data[var_names[i]].median())
        else:
            medians.append(None)
medians = pd.Series(medians)
#print(medians)

## collect results
overview = pd.DataFrame(dict(
    var_names = var_names, 
    dtypes = dtypes,
    nas = nas,
    uniques = uniques,
    modes = modes,
    means = means,
    medians = medians
)).reset_index()
print(overview)

overview.to_excel('ws_02_overview.xlsx', index=False)

    index      var_names   dtypes    nas  uniques                  modes  \
0       0     Unnamed: 0    int64      0    18396                   None   
1       1         Suburb   object      0      330              Reservoir   
2       2        Address   object      0    18134       1/1 Clarendon St   
3       3          Rooms    int64      0       11                   None   
4       4           Type   object      0        3                      h   
5       5          Price  float64      0     2470                   None   
6       6         Method   object      0        5                      S   
7       7        SellerG   object      0      305                 Nelson   
8       8           Date   object      0       58             27/05/2017   
9       9       Distance  float64      1      210                   None   
10     10       Postcode  float64      1      205                   None   
11     11       Bedroom2  float64   3469       12                   None   
12     12       Bathroom  float64   3471        9                   None   
13     13            Car  float64   3576       11                   None   
14     14       Landsize  float64   4793     1449                   None   
15     15   BuildingArea  float64  10634      613                   None   
16     16      YearBuilt  float64   9438      144                   None   
17     17    CouncilArea   object   6163       33               Moreland   
18     18      Lattitude  float64   3332     7518                   None   
19     19     Longtitude  float64   3332     8168                   None   
20     20     Regionname   object      1        8  Southern Metropolitan   
21     21  Propertycount  float64      1      324                   None   

           means        medians  
0   1.182679e+04   11820.500000  
1            NaN            NaN  
2            NaN            NaN  
3   2.935040e+00       3.000000  
4            NaN            NaN  
5   1.056697e+06  880000.000000  
6            NaN            NaN  
7            NaN            NaN  
8            NaN            NaN  
9   1.038999e+01       9.700000  
10  3.107140e+03    3085.000000  
11  2.913043e+00       3.000000  
12  1.538492e+00       1.000000  
13  1.615520e+00       2.000000  
14  5.581164e+02     440.000000  
15  1.512202e+02     126.000000  
16  1.965880e+03    1970.000000  
17           NaN            NaN  
18 -3.780985e+01     -37.803625  
19  1.449963e+02     145.000920  
20           NaN            NaN  
21  7.517975e+03    6567.000000