StarMirror/data-science-ipython-notebooks
1
0
Fork 0
mirror of https://github.com/donnemartin/data-science-ipython-notebooks.git synced 2024-03-22 13:30:56 +08:00
Code Issues Projects Releases Wiki Activity

Added snippets to run Spark on a cluster.

Browse Source
This commit is contained in:
Donne Martin 2015-03-05 07:26:54 -05:00
parent a5a3da5b28
commit 72cf3af7f1
1 changed files with 115 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
spark/spark.ipynb
View File

@ -1,7 +1,7 @@
{ {
"metadata": { "metadata": {
"name": "", "name": "",
"signature": "sha256:ecb4af31fb2838a9be26c4692a4c2619957209df829895e8486de7eb84b59fa3" "signature": "sha256:0426fb2480e184a6d65b40b69e4601e1abb23c84cc1090d1fe0e2e98803c6220"
}, },
"nbformat": 3, "nbformat": 3,
"nbformat_minor": 0, "nbformat_minor": 0,
@ -16,7 +16,8 @@
"\n", "\n",
"* Python Shell\n", "* Python Shell\n",
"* RDDs\n", "* RDDs\n",
"* Pair RDDs" "* Pair RDDs\n",
"* Running Spark on a Cluster"
] ]
}, },
{ {
@ -351,6 +352,118 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [] "outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Running Spark on a Cluster"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Start the standalone cluster's Master and Worker daemons:"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"sudo service spark-master start\n",
"sudo service spark-worker start"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Stop the standalone cluster's Master and Worker daemons:"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"sudo service spark-master stop\n",
"sudo service spark-worker stop"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Restart the standalone cluster's Master and Worker daemons:"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"sudo service spark-master stop\n",
"sudo service spark-worker stop"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"View the Spark standalone cluster UI:"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"http://localhost:18080//"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Start the Spark shell and connect to the cluster:"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"MASTER=spark://localhost:7077 pyspark"
],
"language": "python",
"metadata": {},
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Confirm you are connected to the correct master:"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"sc.master"
],
"language": "python",
"metadata": {},
"outputs": []
} }
], ],
"metadata": {} "metadata": {}