mirror of
https://github.com/donnemartin/data-science-ipython-notebooks.git
synced 2024-03-22 13:30:56 +08:00
607 lines
16 KiB
Plaintext
607 lines
16 KiB
Plaintext
{
|
|
"metadata": {
|
|
"name": "",
|
|
"signature": "sha256:b619f1fd1f2d4495d6a2fe9d048c09b7319b119d4e10a5b2348f0ac6f380a27c"
|
|
},
|
|
"nbformat": 3,
|
|
"nbformat_minor": 0,
|
|
"worksheets": [
|
|
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Pandas Cleaning\n",
|
|
"* Clean\n",
|
|
"* Transform\n",
|
|
"* Merge\n",
|
|
"* Reshape\n",
|
|
"* Concatenate"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"from pandas import Series, DataFrame\n",
|
|
"import pandas as pd"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"prompt_number": 1
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Check for matching values in a specific column for replacement:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"data_1 = {'state' : ['VA', 'VA', 'VA', 'MD', 'MD'],\n",
|
|
" 'year' : [2012, 2013, 2014, 2014, 2015],\n",
|
|
" 'population' : [5.0, 5.1, 5.2, 4.0, 4.1]}\n",
|
|
"df_1 = DataFrame(data_1)\n",
|
|
"df_1"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"html": [
|
|
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>population</th>\n",
|
|
" <th>state</th>\n",
|
|
" <th>year</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td> 5.0</td>\n",
|
|
" <td> VA</td>\n",
|
|
" <td> 2012</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td> 5.1</td>\n",
|
|
" <td> VA</td>\n",
|
|
" <td> 2013</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td> 5.2</td>\n",
|
|
" <td> VA</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td> 4.0</td>\n",
|
|
" <td> MD</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td> 4.1</td>\n",
|
|
" <td> MD</td>\n",
|
|
" <td> 2015</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"metadata": {},
|
|
"output_type": "pyout",
|
|
"prompt_number": 2,
|
|
"text": [
|
|
" population state year\n",
|
|
"0 5.0 VA 2012\n",
|
|
"1 5.1 VA 2013\n",
|
|
"2 5.2 VA 2014\n",
|
|
"3 4.0 MD 2014\n",
|
|
"4 4.1 MD 2015"
|
|
]
|
|
}
|
|
],
|
|
"prompt_number": 2
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"df_1[df_1['state'] == 'VA']"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"html": [
|
|
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>population</th>\n",
|
|
" <th>state</th>\n",
|
|
" <th>year</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td> 5.0</td>\n",
|
|
" <td> VA</td>\n",
|
|
" <td> 2012</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td> 5.1</td>\n",
|
|
" <td> VA</td>\n",
|
|
" <td> 2013</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td> 5.2</td>\n",
|
|
" <td> VA</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"metadata": {},
|
|
"output_type": "pyout",
|
|
"prompt_number": 3,
|
|
"text": [
|
|
" population state year\n",
|
|
"0 5.0 VA 2012\n",
|
|
"1 5.1 VA 2013\n",
|
|
"2 5.2 VA 2014"
|
|
]
|
|
}
|
|
],
|
|
"prompt_number": 3
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Replace all occurrences of a string with another string, in place (no copy):"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"df_1.replace('VA', 'VIRGINIA', inplace=True)\n",
|
|
"df_1"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"html": [
|
|
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>population</th>\n",
|
|
" <th>state</th>\n",
|
|
" <th>year</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td> 5.0</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2012</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td> 5.1</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2013</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td> 5.2</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td> 4.0</td>\n",
|
|
" <td> MD</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td> 4.1</td>\n",
|
|
" <td> MD</td>\n",
|
|
" <td> 2015</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"metadata": {},
|
|
"output_type": "pyout",
|
|
"prompt_number": 4,
|
|
"text": [
|
|
" population state year\n",
|
|
"0 5.0 VIRGINIA 2012\n",
|
|
"1 5.1 VIRGINIA 2013\n",
|
|
"2 5.2 VIRGINIA 2014\n",
|
|
"3 4.0 MD 2014\n",
|
|
"4 4.1 MD 2015"
|
|
]
|
|
}
|
|
],
|
|
"prompt_number": 4
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"In a specified column, replace all occurrences of a string with another string, in place (no copy):"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"df_1.replace({'state' : { 'MD' : 'MARYLAND' }}, inplace=True)\n",
|
|
"df_1"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"html": [
|
|
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>population</th>\n",
|
|
" <th>state</th>\n",
|
|
" <th>year</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td> 5.0</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2012</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td> 5.1</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2013</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td> 5.2</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td> 4.0</td>\n",
|
|
" <td> MARYLAND</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td> 4.1</td>\n",
|
|
" <td> MARYLAND</td>\n",
|
|
" <td> 2015</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"metadata": {},
|
|
"output_type": "pyout",
|
|
"prompt_number": 5,
|
|
"text": [
|
|
" population state year\n",
|
|
"0 5.0 VIRGINIA 2012\n",
|
|
"1 5.1 VIRGINIA 2013\n",
|
|
"2 5.2 VIRGINIA 2014\n",
|
|
"3 4.0 MARYLAND 2014\n",
|
|
"4 4.1 MARYLAND 2015"
|
|
]
|
|
}
|
|
],
|
|
"prompt_number": 5
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Drop the 'population' column and return a copy of the DataFrame:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"df_2 = df_1.drop('population', axis=1)\n",
|
|
"df_2"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"html": [
|
|
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>state</th>\n",
|
|
" <th>year</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2012</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2013</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td> MARYLAND</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td> MARYLAND</td>\n",
|
|
" <td> 2015</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"metadata": {},
|
|
"output_type": "pyout",
|
|
"prompt_number": 6,
|
|
"text": [
|
|
" state year\n",
|
|
"0 VIRGINIA 2012\n",
|
|
"1 VIRGINIA 2013\n",
|
|
"2 VIRGINIA 2014\n",
|
|
"3 MARYLAND 2014\n",
|
|
"4 MARYLAND 2015"
|
|
]
|
|
}
|
|
],
|
|
"prompt_number": 6
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Concatenate two DataFrames:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"data_2 = {'state' : ['NY', 'NY', 'NY', 'FL', 'FL'],\n",
|
|
" 'year' : [2012, 2013, 2014, 2014, 2015],\n",
|
|
" 'population' : [6.0, 6.1, 6.2, 3.0, 3.1]}\n",
|
|
"df_3 = DataFrame(data_2)\n",
|
|
"df_3"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"html": [
|
|
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>population</th>\n",
|
|
" <th>state</th>\n",
|
|
" <th>year</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td> 6.0</td>\n",
|
|
" <td> NY</td>\n",
|
|
" <td> 2012</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td> 6.1</td>\n",
|
|
" <td> NY</td>\n",
|
|
" <td> 2013</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td> 6.2</td>\n",
|
|
" <td> NY</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td> 3.0</td>\n",
|
|
" <td> FL</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td> 3.1</td>\n",
|
|
" <td> FL</td>\n",
|
|
" <td> 2015</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"metadata": {},
|
|
"output_type": "pyout",
|
|
"prompt_number": 7,
|
|
"text": [
|
|
" population state year\n",
|
|
"0 6.0 NY 2012\n",
|
|
"1 6.1 NY 2013\n",
|
|
"2 6.2 NY 2014\n",
|
|
"3 3.0 FL 2014\n",
|
|
"4 3.1 FL 2015"
|
|
]
|
|
}
|
|
],
|
|
"prompt_number": 7
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"df_4 = pd.concat([df_1, df_3])\n",
|
|
"df_4"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"html": [
|
|
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>population</th>\n",
|
|
" <th>state</th>\n",
|
|
" <th>year</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td> 5.0</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2012</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td> 5.1</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2013</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td> 5.2</td>\n",
|
|
" <td> VIRGINIA</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td> 4.0</td>\n",
|
|
" <td> MARYLAND</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td> 4.1</td>\n",
|
|
" <td> MARYLAND</td>\n",
|
|
" <td> 2015</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td> 6.0</td>\n",
|
|
" <td> NY</td>\n",
|
|
" <td> 2012</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td> 6.1</td>\n",
|
|
" <td> NY</td>\n",
|
|
" <td> 2013</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td> 6.2</td>\n",
|
|
" <td> NY</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td> 3.0</td>\n",
|
|
" <td> FL</td>\n",
|
|
" <td> 2014</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td> 3.1</td>\n",
|
|
" <td> FL</td>\n",
|
|
" <td> 2015</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"metadata": {},
|
|
"output_type": "pyout",
|
|
"prompt_number": 8,
|
|
"text": [
|
|
" population state year\n",
|
|
"0 5.0 VIRGINIA 2012\n",
|
|
"1 5.1 VIRGINIA 2013\n",
|
|
"2 5.2 VIRGINIA 2014\n",
|
|
"3 4.0 MARYLAND 2014\n",
|
|
"4 4.1 MARYLAND 2015\n",
|
|
"0 6.0 NY 2012\n",
|
|
"1 6.1 NY 2013\n",
|
|
"2 6.2 NY 2014\n",
|
|
"3 3.0 FL 2014\n",
|
|
"4 3.1 FL 2015"
|
|
]
|
|
}
|
|
],
|
|
"prompt_number": 8
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"prompt_number": 8
|
|
}
|
|
],
|
|
"metadata": {}
|
|
}
|
|
]
|
|
} |