StarMirror/data-science-ipython-notebooks
1
0
Fork 0
mirror of https://github.com/donnemartin/data-science-ipython-notebooks.git synced 2024-03-22 13:30:56 +08:00
Code Issues Projects Releases Wiki Activity

Updated notebook to v3.

Browse Source
This commit is contained in:
Donne Martin 2015-06-03 13:45:42 -04:00
parent 84014940d9
commit c71f9d379a
1 changed files with 503 additions and 474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

975
commands/linux.ipynb
View File

@ -1,477 +1,506 @@
{ {
"metadata": { "cells": [
"name": "",
"signature": "sha256:7cdf1e36fef9e69cd9007eb0c5945e4d6440b31108bfddd6f19aa71d0d1a06ab"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{ {
"cells": [ "cell_type": "markdown",
{ "metadata": {},
"cell_type": "markdown", "source": [
"metadata": {}, "# Linux Commands\n",
"source": [ "\n",
"# Linux Commands\n", "* Disk Usage\n",
"\n", "* Splitting Files\n",
"* Disk Usage\n", "* Grep, Sed\n",
"* Splitting Files\n", "* Compression\n",
"* Grep, Sed\n", "* Curl\n",
"* Compression\n", "* View Running Processes\n",
"* Curl\n", "* Terminal Syntax Highlighting\n",
"* View Running Processes\n", "* Vim"
"* Terminal Syntax Highlighting\n", ]
"* Vim" },
] {
}, "cell_type": "markdown",
{ "metadata": {},
"cell_type": "markdown", "source": [
"metadata": {}, "## Disk Usage"
"source": [ ]
"## Disk Usage" },
] {
}, "cell_type": "markdown",
{ "metadata": {},
"cell_type": "markdown", "source": [
"metadata": {}, "Display human-readable (-h) free disk space:"
"source": [ ]
"Display human-readable (-h) free disk space:" },
] {
}, "cell_type": "code",
{ "execution_count": null,
"cell_type": "code", "metadata": {
"collapsed": false, "collapsed": false
"input": [ },
"!df -h" "outputs": [],
], "source": [
"language": "python", "!df -h"
"metadata": {}, ]
"outputs": [] },
}, {
{ "cell_type": "markdown",
"cell_type": "markdown", "metadata": {},
"metadata": {}, "source": [
"source": [ "Display human-readable (-h) disk usage statistics:"
"Display human-readable (-h) disk usage statistics:" ]
] },
}, {
{ "cell_type": "code",
"cell_type": "code", "execution_count": null,
"collapsed": false, "metadata": {
"input": [ "collapsed": false
"!du -h ./" },
], "outputs": [],
"language": "python", "source": [
"metadata": {}, "!du -h ./"
"outputs": [] ]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Display human-readable (-h) disk usage statistics, showing only the total usage (-s):" "Display human-readable (-h) disk usage statistics, showing only the total usage (-s):"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "execution_count": null,
"input": [ "metadata": {
"!du -sh ../" "collapsed": false
], },
"language": "python", "outputs": [],
"metadata": {}, "source": [
"outputs": [] "!du -sh ../"
}, ]
{ },
"cell_type": "markdown", {
"metadata": {}, "cell_type": "markdown",
"source": [ "metadata": {},
"Display the human-readable (-h) disk usage statistics, showing also the grand total for all file types (-c):" "source": [
] "Display the human-readable (-h) disk usage statistics, showing also the grand total for all file types (-c):"
}, ]
{ },
"cell_type": "code", {
"collapsed": false, "cell_type": "code",
"input": [ "execution_count": null,
"!du -csh ./" "metadata": {
], "collapsed": false
"language": "python", },
"metadata": {}, "outputs": [],
"outputs": [] "source": [
}, "!du -csh ./"
{ ]
"cell_type": "markdown", },
"metadata": {}, {
"source": [ "cell_type": "markdown",
"## Splitting Files" "metadata": {},
] "source": [
}, "## Splitting Files"
{ ]
"cell_type": "markdown", },
"metadata": {}, {
"source": [ "cell_type": "markdown",
"Count number of lines in a file with wc:" "metadata": {},
] "source": [
}, "Count number of lines in a file with wc:"
{ ]
"cell_type": "code", },
"collapsed": false, {
"input": [ "cell_type": "code",
"!wc -l < file.txt" "execution_count": null,
], "metadata": {
"language": "python", "collapsed": false
"metadata": {}, },
"outputs": [] "outputs": [],
}, "source": [
{ "!wc -l < file.txt"
"cell_type": "markdown", ]
"metadata": {}, },
"source": [ {
"Count the number of lines in a file with grep:" "cell_type": "markdown",
] "metadata": {},
}, "source": [
{ "Count the number of lines in a file with grep:"
"cell_type": "code", ]
"collapsed": false, },
"input": [ {
"!grep -c \".\" file.txt" "cell_type": "code",
], "execution_count": null,
"language": "python", "metadata": {
"metadata": {}, "collapsed": false
"outputs": [] },
}, "outputs": [],
{ "source": [
"cell_type": "markdown", "!grep -c \".\" file.txt"
"metadata": {}, ]
"source": [ },
"Split a file into multiple files based on line count:" {
] "cell_type": "markdown",
}, "metadata": {},
{ "source": [
"cell_type": "code", "Split a file into multiple files based on line count:"
"collapsed": false, ]
"input": [ },
"!split -l 20 file.txt new" {
], "cell_type": "code",
"language": "python", "execution_count": null,
"metadata": {}, "metadata": {
"outputs": [] "collapsed": false
}, },
{ "outputs": [],
"cell_type": "markdown", "source": [
"metadata": {}, "!split -l 20 file.txt new"
"source": [ ]
"Split a file into multiple files based on line count, use suffix of length 1:" },
] {
}, "cell_type": "markdown",
{ "metadata": {},
"cell_type": "code", "source": [
"collapsed": false, "Split a file into multiple files based on line count, use suffix of length 1:"
"input": [ ]
"!split -l 802 -a 1 file.csv dir/part-user-csv.tbl-" },
], {
"language": "python", "cell_type": "code",
"metadata": {}, "execution_count": null,
"outputs": [] "metadata": {
}, "collapsed": false
{ },
"cell_type": "markdown", "outputs": [],
"metadata": {}, "source": [
"source": [ "!split -l 802 -a 1 file.csv dir/part-user-csv.tbl-"
"## Grep, Sed" ]
] },
}, {
{ "cell_type": "markdown",
"cell_type": "markdown", "metadata": {},
"metadata": {}, "source": [
"source": [ "## Grep, Sed"
"List number of files matching \u201c.txt\":" ]
] },
}, {
{ "cell_type": "markdown",
"cell_type": "code", "metadata": {},
"collapsed": false, "source": [
"input": [ "List number of files matching “.txt\":"
"!ls -1 | grep .txt | wc -l" ]
], },
"language": "python", {
"metadata": {}, "cell_type": "code",
"outputs": [] "execution_count": null,
}, "metadata": {
{ "collapsed": false
"cell_type": "markdown", },
"metadata": {}, "outputs": [],
"source": [ "source": [
"Check number of MapReduce records processed, outputting the results to the terminal:" "!ls -1 | grep .txt | wc -l"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "markdown",
"collapsed": false, "metadata": {},
"input": [ "source": [
"!cat * | grep -c \"foo\" folder/part*" "Check number of MapReduce records processed, outputting the results to the terminal:"
], ]
"language": "python", },
"metadata": {}, {
"outputs": [] "cell_type": "code",
}, "execution_count": null,
{ "metadata": {
"cell_type": "markdown", "collapsed": false
"metadata": {}, },
"source": [ "outputs": [],
"Delete matching lines in place:" "source": [
] "!cat * | grep -c \"foo\" folder/part*"
}, ]
{ },
"cell_type": "code", {
"collapsed": false, "cell_type": "markdown",
"input": [ "metadata": {},
"!sed -i '/Important Lines: /d\u2019 original_file" "source": [
], "Delete matching lines in place:"
"language": "python", ]
"metadata": {}, },
"outputs": [] {
}, "cell_type": "code",
{ "execution_count": null,
"cell_type": "markdown", "metadata": {
"metadata": {}, "collapsed": false
"source": [ },
"## Compression" "outputs": [],
] "source": [
}, "!sed -i '/Important Lines: /d’ original_file"
{ ]
"cell_type": "code", },
"collapsed": false, {
"input": [ "cell_type": "markdown",
"# Compress zip\n", "metadata": {},
"!zip -r archive_name.zip folder_to_compress\n", "source": [
"\n", "## Compression"
"# Compress zip without invisible Mac resources\n", ]
"!zip -r -X archive_name.zip folder_to_compress\n", },
"\n", {
"# Extract zip\n", "cell_type": "code",
"!unzip archive_name.zip\n", "execution_count": null,
"\n", "metadata": {
"# Compress TAR.GZ\n", "collapsed": false
"!tar -zcvf archive_name.tar.gz folder_to_compress\n", },
"\n", "outputs": [],
"# Extract TAR.GZ\n", "source": [
"!tar -zxvf archive_name.tar.gz\n", "# Compress zip\n",
"\n", "!zip -r archive_name.zip folder_to_compress\n",
"# Compress TAR.BZ2\n", "\n",
"!tar -jcvf archive_name.tar.bz2 folder_to_compress\n", "# Compress zip without invisible Mac resources\n",
"\n", "!zip -r -X archive_name.zip folder_to_compress\n",
"# Extract TAR.BZ2\n", "\n",
"!tar -jxvf archive_name.tar.bz2\n", "# Extract zip\n",
"\n", "!unzip archive_name.zip\n",
"# Extract GZ\n", "\n",
"!gunzip archivename.gz\n", "# Compress TAR.GZ\n",
"\n", "!tar -zcvf archive_name.tar.gz folder_to_compress\n",
"# Uncompress all tar.gz in current directory to another directory\n", "\n",
"!for i in *.tar.gz; do echo working on $i; tar xvzf $i -C directory/ ; done" "# Extract TAR.GZ\n",
], "!tar -zxvf archive_name.tar.gz\n",
"language": "python", "\n",
"metadata": {}, "# Compress TAR.BZ2\n",
"outputs": [] "!tar -jcvf archive_name.tar.bz2 folder_to_compress\n",
}, "\n",
{ "# Extract TAR.BZ2\n",
"cell_type": "markdown", "!tar -jxvf archive_name.tar.bz2\n",
"metadata": {}, "\n",
"source": [ "# Extract GZ\n",
"## Curl" "!gunzip archivename.gz\n",
] "\n",
}, "# Uncompress all tar.gz in current directory to another directory\n",
{ "!for i in *.tar.gz; do echo working on $i; tar xvzf $i -C directory/ ; done"
"cell_type": "code", ]
"collapsed": false, },
"input": [ {
"# Display the curl output:\n", "cell_type": "markdown",
"!curl donnemartin.com\n", "metadata": {},
"\n", "source": [
"# Download the curl output to a file:\n", "## Curl"
"!curl donnemartin.com > donnemartin.html\n", ]
"\n", },
"# Download the curl output to a file -o\n", {
"!curl -o image.png http://i1.wp.com/donnemartin.com/wp-content/uploads/2015/02/splunk_cover.png\n", "cell_type": "code",
"\n", "execution_count": null,
"# Download the curl output to a file, keeping the original file name -O\n", "metadata": {
"!curl -O http://i1.wp.com/donnemartin.com/wp-content/uploads/2015/02/splunk_cover.png\n", "collapsed": false
" \n", },
"# Download multiple files, attempting to reuse the same connection\n", "outputs": [],
"!curl -O url1 -O url2\n", "source": [
"\n", "# Display the curl output:\n",
"# Follow redirects -L\n", "!curl donnemartin.com\n",
"!curl -L url\n", "\n",
"\n", "# Download the curl output to a file:\n",
"# Resume a previous download -C -\n", "!curl donnemartin.com > donnemartin.html\n",
"!curl -C - -O url\n", "\n",
"\n", "# Download the curl output to a file -o\n",
"# Authenticate -u\n", "!curl -o image.png http://i1.wp.com/donnemartin.com/wp-content/uploads/2015/02/splunk_cover.png\n",
"!curl -u username:password url" "\n",
], "# Download the curl output to a file, keeping the original file name -O\n",
"language": "python", "!curl -O http://i1.wp.com/donnemartin.com/wp-content/uploads/2015/02/splunk_cover.png\n",
"metadata": {}, " \n",
"outputs": [] "# Download multiple files, attempting to reuse the same connection\n",
}, "!curl -O url1 -O url2\n",
{ "\n",
"cell_type": "markdown", "# Follow redirects -L\n",
"metadata": {}, "!curl -L url\n",
"source": [ "\n",
"## View Running Processes" "# Resume a previous download -C -\n",
] "!curl -C - -O url\n",
}, "\n",
{ "# Authenticate -u\n",
"cell_type": "code", "!curl -u username:password url"
"collapsed": false, ]
"input": [ },
"# Display sorted info about processes\n", {
"!top\n", "cell_type": "markdown",
"\n", "metadata": {},
"# Display all running processes\n", "source": [
"!ps aux | less\n", "## View Running Processes"
"\n", ]
"# Display all matching running processes with full formatting\n", },
"!ps -ef | grep python\n", {
"\n", "cell_type": "code",
"# See processes run by user dmartin\n", "execution_count": null,
"!ps -u dmartin\n", "metadata": {
"\n", "collapsed": false
"# Display running processes as a tree\n", },
"!pstree" "outputs": [],
], "source": [
"language": "python", "# Display sorted info about processes\n",
"metadata": {}, "!top\n",
"outputs": [] "\n",
}, "# Display all running processes\n",
{ "!ps aux | less\n",
"cell_type": "markdown", "\n",
"metadata": {}, "# Display all matching running processes with full formatting\n",
"source": [ "!ps -ef | grep python\n",
"## Terminal Syntax Highlighting" "\n",
] "# See processes run by user dmartin\n",
}, "!ps -u dmartin\n",
{ "\n",
"cell_type": "markdown", "# Display running processes as a tree\n",
"metadata": {}, "!pstree"
"source": [ ]
"Add the following to your ~/.bash_profile:" },
] {
}, "cell_type": "markdown",
{ "metadata": {},
"cell_type": "code", "source": [
"collapsed": false, "## Terminal Syntax Highlighting"
"input": [ ]
"export PS1='\\[\\033[01;32m\\]\\u@\\h\\[\\033[00m\\]:\\[\\033[01;34m\\]\\W\\[\\033[00m\\]\\$ '\n", },
"export CLICOLOR=1\n", {
"export LSCOLORS=ExFxBxDxCxegedabagacad\n", "cell_type": "markdown",
"alias ls='ls -GFh'" "metadata": {},
], "source": [
"language": "python", "Add the following to your ~/.bash_profile:"
"metadata": {}, ]
"outputs": [] },
}, {
{ "cell_type": "code",
"cell_type": "markdown", "execution_count": null,
"metadata": {}, "metadata": {
"source": [ "collapsed": false
"Reload .bash_profile:" },
] "outputs": [],
}, "source": [
{ "export PS1='\\[\\033[01;32m\\]\\u@\\h\\[\\033[00m\\]:\\[\\033[01;34m\\]\\W\\[\\033[00m\\]\\$ '\n",
"cell_type": "code", "export CLICOLOR=1\n",
"collapsed": false, "export LSCOLORS=ExFxBxDxCxegedabagacad\n",
"input": [ "alias ls='ls -GFh'"
"!source ~/.bash_profile" ]
], },
"language": "python", {
"metadata": {}, "cell_type": "markdown",
"outputs": [] "metadata": {},
}, "source": [
{ "Reload .bash_profile:"
"cell_type": "markdown", ]
"metadata": {}, },
"source": [ {
"## Vim" "cell_type": "code",
] "execution_count": null,
}, "metadata": {
{ "collapsed": false
"cell_type": "code", },
"collapsed": false, "outputs": [],
"input": [ "source": [
"Normal mode: esc\n", "!source ~/.bash_profile"
"\n", ]
"Basic movement: h, j, k, l\n", },
"Word movement: w, W, e, E, b, B\n", {
"\n", "cell_type": "markdown",
"Go to matching parenthesis: %\n", "metadata": {},
"Go to start of the line: 0\n", "source": [
"Go to end of the line: $\n", "## Vim"
"\n", ]
"Find character: f\n", },
"\n", {
"Insert mode: i\n", "cell_type": "code",
"Append to line: A\n", "execution_count": null,
"\n", "metadata": {
"Delete character: x\n", "collapsed": false
"Delete command: d\n", },
"Delete line: dd\n", "outputs": [],
"\n", "source": [
"Replace command: r\n", "Normal mode: esc\n",
"Change command: c\n", "\n",
"\n", "Basic movement: h, j, k, l\n",
"Undo: u (U for all changes on a line)\n", "Word movement: w, W, e, E, b, B\n",
"Redo: CTRL-R\n", "\n",
"\n", "Go to matching parenthesis: %\n",
"Copy the current line: yy\n", "Go to start of the line: 0\n",
"Paste the current line: p (P for paste above cursor)\n", "Go to end of the line: $\n",
"\n", "\n",
"Quit without saving changes: q!\n", "Find character: f\n",
"Write the current file and quit: :wq" "\n",
], "Insert mode: i\n",
"language": "python", "Append to line: A\n",
"metadata": {}, "\n",
"outputs": [] "Delete character: x\n",
}, "Delete command: d\n",
{ "Delete line: dd\n",
"cell_type": "markdown", "\n",
"metadata": {}, "Replace command: r\n",
"source": [ "Change command: c\n",
"Run the following command to enable the tutorial:" "\n",
] "Undo: u (U for all changes on a line)\n",
}, "Redo: CTRL-R\n",
{ "\n",
"cell_type": "code", "Copy the current line: yy\n",
"collapsed": false, "Paste the current line: p (P for paste above cursor)\n",
"input": [ "\n",
"!vimtutor" "Quit without saving changes: q!\n",
], "Write the current file and quit: :wq"
"language": "python", ]
"metadata": {}, },
"outputs": [] {
}, "cell_type": "markdown",
{ "metadata": {},
"cell_type": "markdown", "source": [
"metadata": {}, "Run the following command to enable the tutorial:"
"source": [ ]
"Run the following commands to enable syntax colors:" },
] {
}, "cell_type": "code",
{ "execution_count": null,
"cell_type": "code", "metadata": {
"collapsed": false, "collapsed": false
"input": [ },
"!cd ~\n", "outputs": [],
"!vim .vimrc\n", "source": [
"# Add the following to ~/.vimrc\n", "!vimtutor"
"syntax on\n", ]
":wq" },
], {
"language": "python", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"outputs": [] "source": [
} "Run the following commands to enable syntax colors:"
], ]
"metadata": {} },
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!cd ~\n",
"!vim .vimrc\n",
"# Add the following to ~/.vimrc\n",
"syntax on\n",
":wq"
]
} }
] ],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.9"
}
},
"nbformat": 4,
"nbformat_minor": 0
} }