From 23d3866b8e36ddacf3906d9f111baf246f129836 Mon Sep 17 00:00:00 2001 From: Donne Martin Date: Fri, 13 Mar 2015 07:57:12 -0400 Subject: [PATCH] Prefixed AWS commands with ! so they can be executed within IPython Notebook. --- aws/aws.ipynb | 88 +++++++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/aws/aws.ipynb b/aws/aws.ipynb index 0654242..88ac9c4 100644 --- a/aws/aws.ipynb +++ b/aws/aws.ipynb @@ -1,7 +1,7 @@ { "metadata": { "name": "", - "signature": "sha256:7778453219a7e893528fae53a5bba1a4a5dcc071d59b328be018de36dcd7945b" + "signature": "sha256:760f0227418945ff60ae747898cfec6f9614a279e133bfb3ef96560860b3ce0d" }, "nbformat": 3, "nbformat_minor": 0, @@ -42,7 +42,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "ssh -i key.pem ubuntu@ipaddress" + "!ssh -i key.pem ubuntu@ipaddress" ], "language": "python", "metadata": {}, @@ -59,7 +59,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "ssh -i key.pem ec2-user@ipaddress" + "!ssh -i key.pem ec2-user@ipaddress" ], "language": "python", "metadata": {}, @@ -87,7 +87,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "sudo apt-get install s3cmd" + "!sudo apt-get install s3cmd" ], "language": "python", "metadata": {}, @@ -106,7 +106,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "s3cmd --configure" + "!s3cmd --configure" ], "language": "python", "metadata": {}, @@ -124,49 +124,49 @@ "collapsed": false, "input": [ "# List all buckets\n", - "s3cmd ls\n", + "!s3cmd ls\n", "\n", "# List the contents of the bucket\n", - "s3cmd ls s3://my-bucket-name\n", + "!s3cmd ls s3://my-bucket-name\n", "\n", "# Upload a file into the bucket (private)\n", - "s3cmd put myfile.txt s3://my-bucket-name/myfile.txt\n", + "!s3cmd put myfile.txt s3://my-bucket-name/myfile.txt\n", "\n", "# Upload a file into the bucket (public)\n", - "s3cmd put --acl-public --guess-mime-type myfile.txt s3://my-bucket-name/myfile.txt\n", + "!s3cmd put --acl-public --guess-mime-type myfile.txt s3://my-bucket-name/myfile.txt\n", "\n", "# Recursively upload a directory to s3\n", - "s3cmd put --recursive my-local-folder-path/ s3://my-bucket-name/mydir/\n", + "!s3cmd put --recursive my-local-folder-path/ s3://my-bucket-name/mydir/\n", "\n", "# Download a file\n", - "s3cmd get s3://my-bucket-name/myfile.txt myfile.txt\n", + "!s3cmd get s3://my-bucket-name/myfile.txt myfile.txt\n", "\n", "# Recursively download files that start with myfile\n", - "s3cmd --recursive get s3://my-bucket-name/myfile\n", + "!s3cmd --recursive get s3://my-bucket-name/myfile\n", "\n", "# Delete a file\n", - "s3cmd del s3://my-bucket-name/myfile.txt\n", + "!s3cmd del s3://my-bucket-name/myfile.txt\n", "\n", "# Delete a bucket\n", - "s3cmd del --recursive s3://my-bucket-name/\n", + "!s3cmd del --recursive s3://my-bucket-name/\n", "\n", "# Create a bucket\n", - "s3cmd mb s3://my-bucket-name\n", + "!s3cmd mb s3://my-bucket-name\n", "\n", "# List bucket disk usage (human readable)\n", - "s3cmd du -H s3://my-bucket-name/\n", + "!s3cmd du -H s3://my-bucket-name/\n", "\n", "# Sync local (source) to s3 bucket (destination)\n", - "s3cmd sync my-local-folder-path/ s3://my-bucket-name/\n", + "!s3cmd sync my-local-folder-path/ s3://my-bucket-name/\n", "\n", "# Sync s3 bucket (source) to local (destination)\n", - "s3cmd sync s3://my-bucket-name/ my-local-folder-path/\n", + "!s3cmd sync s3://my-bucket-name/ my-local-folder-path/\n", "\n", "# Do a dry-run (do not perform actual sync, but get information about what would happen)\n", - "s3cmd --dry-run sync s3://my-bucket-name/ my-local-folder-path/\n", + "!s3cmd --dry-run sync s3://my-bucket-name/ my-local-folder-path/\n", "\n", "# Apply a standard shell wildcard include to sync s3 bucket (source) to local (destination)\n", - "s3cmd --include '2014-05-01*' sync s3://my-bucket-name/ my-local-folder-path/" + "!s3cmd --include '2014-05-01*' sync s3://my-bucket-name/ my-local-folder-path/" ], "language": "python", "metadata": {}, @@ -192,8 +192,8 @@ "cell_type": "code", "collapsed": false, "input": [ - "sudo apt-get install boto\n", - "sudo apt-get install git" + "!sudo apt-get install boto\n", + "!sudo apt-get install git" ], "language": "python", "metadata": {}, @@ -210,7 +210,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "git clone https://github.com/twpayne/s3-parallel-put.git" + "!git clone https://github.com/twpayne/s3-parallel-put.git" ], "language": "python", "metadata": {}, @@ -227,8 +227,8 @@ "cell_type": "code", "collapsed": false, "input": [ - "export AWS_ACCESS_KEY_ID=XXX\n", - "export AWS_SECRET_ACCESS_KEY=XXX" + "!export AWS_ACCESS_KEY_ID=XXX\n", + "!export AWS_SECRET_ACCESS_KEY=XXX" ], "language": "python", "metadata": {}, @@ -245,7 +245,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "s3-parallel-put --bucket=bucket --prefix=PREFIX SOURCE" + "!s3-parallel-put --bucket=bucket --prefix=PREFIX SOURCE" ], "language": "python", "metadata": {}, @@ -262,7 +262,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "s3-parallel-put --bucket=bucket --host=s3.amazonaws.com --put=stupid --dry-run --prefix=prefix/ ./" + "!s3-parallel-put --bucket=bucket --host=s3.amazonaws.com --put=stupid --dry-run --prefix=prefix/ ./" ], "language": "python", "metadata": {}, @@ -288,7 +288,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "rvm --default ruby-1.8.7-p374" + "!rvm --default ruby-1.8.7-p374" ], "language": "python", "metadata": {}, @@ -308,7 +308,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "./elastic-mapreduce --create --instance-group master --instance-count 1 \\\n", + "!./elastic-mapreduce --create --instance-group master --instance-count 1 \\\n", "--instance-type m1.small --instance-group core --instance-count 4 \\\n", "--instance-type m1.small --jar /home/hadoop/lib/emr-s3distcp-1.0.jar \\\n", "--args \"--src,s3://my-bucket-source/,--groupBy,.*([0-9]{4}-01).*,\\\n", @@ -361,7 +361,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "python mr-script.py -r emr s3://bucket-source/ --output-dir=s3://bucket-dest/" + "!python mr-script.py -r emr s3://bucket-source/ --output-dir=s3://bucket-dest/" ], "language": "python", "metadata": {}, @@ -378,7 +378,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "python mrjob_script.py input_data.txt > output_data.txt" + "!python mrjob_script.py input_data.txt > output_data.txt" ], "language": "python", "metadata": {}, @@ -680,7 +680,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws kinesis create-stream --stream-name Foo --shard-count 1 --profile adminuser" + "!aws kinesis create-stream --stream-name Foo --shard-count 1 --profile adminuser" ], "language": "python", "metadata": {}, @@ -697,7 +697,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws kinesis list-streams --profile adminuser" + "!aws kinesis list-streams --profile adminuser" ], "language": "python", "metadata": {}, @@ -714,7 +714,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws kinesis describe-stream --stream-name Foo --profile adminuser" + "!aws kinesis describe-stream --stream-name Foo --profile adminuser" ], "language": "python", "metadata": {}, @@ -731,7 +731,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws kinesis put-record --stream-name Foo --data \"SGVsbG8sIHRoaXMgaXMgYSB0ZXN0IDEyMy4=\" --partition-key shardId-000000000000 --region us-east-1 --profile adminuser" + "!aws kinesis put-record --stream-name Foo --data \"SGVsbG8sIHRoaXMgaXMgYSB0ZXN0IDEyMy4=\" --partition-key shardId-000000000000 --region us-east-1 --profile adminuser" ], "language": "python", "metadata": {}, @@ -748,7 +748,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "SHARD_ITERATOR=$(aws kinesis get-shard-iterator --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --stream-name Foo --query 'ShardIterator' --profile adminuser)\n", + "!SHARD_ITERATOR=$(aws kinesis get-shard-iterator --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --stream-name Foo --query 'ShardIterator' --profile adminuser)\n", "aws kinesis get-records --shard-iterator $SHARD_ITERATOR" ], "language": "python", @@ -766,7 +766,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws kinesis delete-stream --stream-name Foo --profile adminuser" + "!aws kinesis delete-stream --stream-name Foo --profile adminuser" ], "language": "python", "metadata": {}, @@ -790,7 +790,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws lambda list-functions \\\n", + "!aws lambda list-functions \\\n", " --region us-east-1 \\\n", " --max-items 10" ], @@ -809,7 +809,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws lambda upload-function \\\n", + "!aws lambda upload-function \\\n", " --region us-east-1 \\\n", " --function-name foo \\\n", " --function-zip file-path/foo.zip \\\n", @@ -834,7 +834,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws lambda invoke-async \\\n", + "!aws lambda invoke-async \\\n", " --function-name foo \\\n", " --region us-east-1 \\\n", " --invoke-args foo.txt \\\n", @@ -855,7 +855,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws lambda get-function-configuration \\\n", + "!aws lambda get-function-configuration \\\n", " --function-name helloworld \\\n", " --region us-east-1 \\\n", " --debug" @@ -875,7 +875,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws lambda get-function \\\n", + "!aws lambda get-function \\\n", " --function-name helloworld \\\n", " --region us-east-1 \\\n", " --debug" @@ -895,7 +895,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws lambda add-event-source \\\n", + "!aws lambda add-event-source \\\n", " --region us-east-1 \\\n", " --function-name ProcessKinesisRecords \\\n", " --role invocation-role-arn \\\n", @@ -918,7 +918,7 @@ "cell_type": "code", "collapsed": false, "input": [ - "aws lambda delete-function \\\n", + "!aws lambda delete-function \\\n", " --function-name helloworld \\\n", " --region us-east-1 \\\n", " --debug"