StarMirror/data-science-ipython-notebooks
1
0
Fork 0
mirror of https://github.com/donnemartin/data-science-ipython-notebooks.git synced 2024-03-22 13:30:56 +08:00
Code Issues Projects Releases Wiki Activity

Updated notebook to v3.

Browse Source
This commit is contained in:
Donne Martin 2015-05-24 10:49:21 -04:00
parent d4587d3771
commit 2106accb6d
1 changed files with 245 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
spark/hdfs.ipynb
View File

@ -1,12 +1,4 @@
{
"metadata": {
"name": "",
"signature": "sha256:ff0a8cf7aae0fcbadcd718a2f9164affac4fd22893c629a3755198b4f003d63a"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "markdown",
@ -24,13 +16,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -41,13 +34,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -58,13 +52,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -ls"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -75,13 +70,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -ls /"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -92,13 +88,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -put file.txt file.txt"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -109,13 +106,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -cat file.txt"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -126,13 +124,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -cat file.txt | tail -n 10"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -143,13 +142,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -cat dir/* | less"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -160,13 +160,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -get file.txt file.txt"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -177,13 +178,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -mkdir dir"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -194,13 +196,14 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!hdfs dfs -rm -r dir"
],
"language": "python",
"metadata": {},
"outputs": []
]
},
{
"cell_type": "markdown",
@ -211,16 +214,35 @@
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"data = sc.textFile (\"hdfs://hdfs-host:port/path/file.txt\")"
],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.9"
}
},
"nbformat": 4,
"nbformat_minor": 0
}