{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "763373f9-1ba3-4fb6-9ae3-d0aeb6be07e2",
   "metadata": {},
   "source": [
    "**Workshop 03 - Loesungsvorschlag**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac5573af-46dd-43fa-886e-ca228e61edaf",
   "metadata": {},
   "source": [
    "# Data Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "22a6e66a-eeaa-4ebe-aa95-ad983a2b1110",
   "metadata": {},
   "outputs": [],
   "source": [
    "## prepare and read data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "datapath = '../../3_data'\n",
    "filename = 'melb_data.csv'\n",
    "from os import chdir; chdir(datapath)\n",
    "\n",
    "data = pd.read_csv(filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c42aa1f-2ab1-43b2-a9a7-808f83dd724f",
   "metadata": {},
   "source": [
    "## Entfernen von Beobachtungen nach Bedingung"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "71e9e7ca-c6cd-40bc-b9ff-adb11721d669",
   "metadata": {},
   "outputs": [],
   "source": [
    "## remove outliers on Price\n",
    "data = data[data.Price < 8000000]\n",
    "\n",
    "## remove selected observation2\n",
    "data = data[data.YearBuilt != 1196]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "309c5083-cefb-4b25-91e3-1eebda35555f",
   "metadata": {},
   "source": [
    "## Entfernen von Duplikaten"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1eb307e8-df31-4a37-a9ed-c80a1d354cf9",
   "metadata": {},
   "source": [
    "hier kein Bedarf"
   ]
  },
  {
   "cell_type": "raw",
   "id": "757d2596-04e0-445d-8ea9-9cd5c92ef702",
   "metadata": {},
   "source": [
    "## remove duplicates\n",
    "data.drop_duplicates(ignore_index=True, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da78d619-572c-4649-8c00-fde22bcbc8bb",
   "metadata": {},
   "source": [
    "## Entfernen fragwürdiger Variablen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c689473c-e969-4c69-a131-4d3a680b1f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "vars_to_drop = ['Unnamed: 0', 'Suburb', 'Address', 'SellerG', 'Postcode', 'Bedroom2']\n",
    "data = data.drop(vars_to_drop, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8816b6f2-1bb3-4795-9fdb-cd7a50494d09",
   "metadata": {},
   "source": [
    "## Einsetzen von Werten für NAs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c637332d-24e5-48b1-bb58-ea704ee3f92e",
   "metadata": {},
   "outputs": [],
   "source": [
    "## mode for all cat vars (if any)\n",
    "cat_feats = data.select_dtypes(include=['object']).columns\n",
    "for c in cat_feats:\n",
    "    data[c].fillna(data[c].mode()[0], inplace = True)\n",
    "\n",
    "## median for all num features\n",
    "num_feats = data.select_dtypes(include=['int64', 'float64']).columns\n",
    "for c in num_feats:\n",
    "    data[c].fillna(data[c].median(), inplace = True)    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e85ccf8-e99e-4630-a4e6-6d380994ab26",
   "metadata": {},
   "source": [
    "# Kategoriale Variablen"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b888038-88e8-46a5-988e-9f0dfd38d86d",
   "metadata": {},
   "source": [
    "## Reduzieren der Kardinalität"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e5a432e1-eb89-4d1e-b0cf-f799a8ac5216",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Regionname: combine rarest 3 levels to 'Victoria'\n",
    "data.Regionname = np.where(\n",
    "        (data.Regionname == 'Eastern Victoria') |\n",
    "        (data.Regionname == 'Northern Victoria') |\n",
    "        (data.Regionname == 'Western Victoria'),\n",
    "    'Victoria', data.Regionname)\n",
    "\n",
    "## Method: combine 'SA' to 'S'\n",
    "data.Method = np.where(data.Method == 'SA', 'S', data.Method)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbacc9d0-d357-4020-8062-6b968aba5e9e",
   "metadata": {},
   "source": [
    "## Nummerisiren - Faktorisieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ec29d19a-347b-4793-b2ce-4831f7a9bfa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.CouncilArea = data.CouncilArea.factorize()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3937e91f-85c4-4f1c-bd50-564e8de620ea",
   "metadata": {},
   "source": [
    "## Nummerisiren - Ordial Encodieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3f250d95-862f-4016-af38-1c6ff8750db2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.Type.replace(\n",
    "    ['h', 'u', 't'], \n",
    "    [1, 2, 3], \n",
    "    inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "154a4b0f-6e2f-498a-89f4-64e80de30fd4",
   "metadata": {},
   "source": [
    "## Nummerisieren - Binär Encodieren"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d422904-43d5-4c32-9c7b-cc16ddfbf39d",
   "metadata": {},
   "source": [
    "hier kein Bedarf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "807990cc-9fac-47b7-a165-3194f142b30c",
   "metadata": {},
   "source": [
    "## Nummerisieren - Ordinal Encodieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "91ef6cb1-157a-46e0-b9e6-0bf025b953b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "## one-hot encoding\n",
    "## apply for all categorical variables except Date (will be transformed later)\n",
    "ignore = 'Date'\n",
    "sel_vars = data.select_dtypes(include=['object']).columns.drop(ignore)\n",
    "data = pd.get_dummies(data, columns=sel_vars, drop_first=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a1dd82b-e0d9-44d2-a57b-80f4bb3b2c72",
   "metadata": {},
   "source": [
    "# Numerische Variablen"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "607eab42-5a74-40fd-813d-eed338e93d3e",
   "metadata": {},
   "source": [
    "## Logarithmieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d5a6ff61-f2dc-47d0-ac10-82c811afe903",
   "metadata": {},
   "outputs": [],
   "source": [
    "## logarithme and rename\n",
    "data.Landsize = np.log10(data.Landsize + 1)\n",
    "data.BuildingArea = np.log10(data.BuildingArea + 1)\n",
    "data.rename(columns={\n",
    "    'Landsize' : 'logLandsize',\n",
    "    'BuildingArea' : 'logBuildingArea'\n",
    "}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7dace176-8b5d-4533-bdd2-119e63a7bfa2",
   "metadata": {},
   "source": [
    "## Binär umcodieren"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f508469-97f4-4025-be52-45643c300bfe",
   "metadata": {},
   "source": [
    "hier kein Bedarf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "177c878f-ddf4-49f9-95c8-e9cc1af76473",
   "metadata": {},
   "source": [
    "# Andere Tätigkeiten"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1324181-c90f-4367-968f-1723bf2784ac",
   "metadata": {},
   "source": [
    "## Konstruktion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2e939e6a-7a96-4942-945f-4f21d0af48e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "## construct month, year and day_of_week\n",
    "Date = pd.to_datetime(data.Date, format='%d/%m/%Y')\n",
    "data['month'] = Date.dt.month\n",
    "data['year'] = Date.dt.year\n",
    "data['day_of_week'] = Date.dt.day_of_week\n",
    "data.drop('Date', axis=1, inplace=True) ## not longer used\n",
    "#print(data.info()) ## check"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3931a58a-6d47-494f-8bb2-9ea6f2914ff3",
   "metadata": {},
   "source": [
    "## Bereinigen der Variablennamen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "70daeeee-8257-4969-b9ff-49d700251b54",
   "metadata": {},
   "outputs": [],
   "source": [
    "old_names = data.columns\n",
    "new_names = old_names.str.replace('[^a-zA-Z0-9_]', '_', regex=True)\n",
    "for i in range(len(old_names)):\n",
    "    data.rename(columns={old_names[i]:new_names[i]}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbac2f0b-1f1e-41e2-aae9-60ec4238b28c",
   "metadata": {},
   "source": [
    "## Standardisieren"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "906c9083-a03d-4a21-930e-f87e21079359",
   "metadata": {},
   "source": [
    "hier kein Bedarf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92bae929-be6a-4061-bdb1-572cc9fc8390",
   "metadata": {},
   "source": [
    "## Speichern unter neuem Namen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "882cd439-5cb6-4ab3-8695-09af2670f8f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('melb_data_prep.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 5
}