mirror of
https://github.com/donnemartin/data-science-ipython-notebooks.git
synced 2024-03-22 13:30:56 +08:00
Prefixed AWS commands with ! so they can be executed within IPython Notebook.
This commit is contained in:
Donne Martin
parent
1c4e2157a6
commit
23d3866b8e
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"metadata": {
|
||||
"name": "",
|
||||
"signature": "sha256:7778453219a7e893528fae53a5bba1a4a5dcc071d59b328be018de36dcd7945b"
|
||||
"signature": "sha256:760f0227418945ff60ae747898cfec6f9614a279e133bfb3ef96560860b3ce0d"
|
||||
},
|
||||
"nbformat": 3,
|
||||
"nbformat_minor": 0,
|
||||
|
|
@ -42,7 +42,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"ssh -i key.pem ubuntu@ipaddress"
|
||||
"!ssh -i key.pem ubuntu@ipaddress"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -59,7 +59,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"ssh -i key.pem ec2-user@ipaddress"
|
||||
"!ssh -i key.pem ec2-user@ipaddress"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -87,7 +87,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"sudo apt-get install s3cmd"
|
||||
"!sudo apt-get install s3cmd"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -106,7 +106,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"s3cmd --configure"
|
||||
"!s3cmd --configure"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -124,49 +124,49 @@
|
|||
"collapsed": false,
|
||||
"input": [
|
||||
"# List all buckets\n",
|
||||
"s3cmd ls\n",
|
||||
"!s3cmd ls\n",
|
||||
"\n",
|
||||
"# List the contents of the bucket\n",
|
||||
"s3cmd ls s3://my-bucket-name\n",
|
||||
"!s3cmd ls s3://my-bucket-name\n",
|
||||
"\n",
|
||||
"# Upload a file into the bucket (private)\n",
|
||||
"s3cmd put myfile.txt s3://my-bucket-name/myfile.txt\n",
|
||||
"!s3cmd put myfile.txt s3://my-bucket-name/myfile.txt\n",
|
||||
"\n",
|
||||
"# Upload a file into the bucket (public)\n",
|
||||
"s3cmd put --acl-public --guess-mime-type myfile.txt s3://my-bucket-name/myfile.txt\n",
|
||||
"!s3cmd put --acl-public --guess-mime-type myfile.txt s3://my-bucket-name/myfile.txt\n",
|
||||
"\n",
|
||||
"# Recursively upload a directory to s3\n",
|
||||
"s3cmd put --recursive my-local-folder-path/ s3://my-bucket-name/mydir/\n",
|
||||
"!s3cmd put --recursive my-local-folder-path/ s3://my-bucket-name/mydir/\n",
|
||||
"\n",
|
||||
"# Download a file\n",
|
||||
"s3cmd get s3://my-bucket-name/myfile.txt myfile.txt\n",
|
||||
"!s3cmd get s3://my-bucket-name/myfile.txt myfile.txt\n",
|
||||
"\n",
|
||||
"# Recursively download files that start with myfile\n",
|
||||
"s3cmd --recursive get s3://my-bucket-name/myfile\n",
|
||||
"!s3cmd --recursive get s3://my-bucket-name/myfile\n",
|
||||
"\n",
|
||||
"# Delete a file\n",
|
||||
"s3cmd del s3://my-bucket-name/myfile.txt\n",
|
||||
"!s3cmd del s3://my-bucket-name/myfile.txt\n",
|
||||
"\n",
|
||||
"# Delete a bucket\n",
|
||||
"s3cmd del --recursive s3://my-bucket-name/\n",
|
||||
"!s3cmd del --recursive s3://my-bucket-name/\n",
|
||||
"\n",
|
||||
"# Create a bucket\n",
|
||||
"s3cmd mb s3://my-bucket-name\n",
|
||||
"!s3cmd mb s3://my-bucket-name\n",
|
||||
"\n",
|
||||
"# List bucket disk usage (human readable)\n",
|
||||
"s3cmd du -H s3://my-bucket-name/\n",
|
||||
"!s3cmd du -H s3://my-bucket-name/\n",
|
||||
"\n",
|
||||
"# Sync local (source) to s3 bucket (destination)\n",
|
||||
"s3cmd sync my-local-folder-path/ s3://my-bucket-name/\n",
|
||||
"!s3cmd sync my-local-folder-path/ s3://my-bucket-name/\n",
|
||||
"\n",
|
||||
"# Sync s3 bucket (source) to local (destination)\n",
|
||||
"s3cmd sync s3://my-bucket-name/ my-local-folder-path/\n",
|
||||
"!s3cmd sync s3://my-bucket-name/ my-local-folder-path/\n",
|
||||
"\n",
|
||||
"# Do a dry-run (do not perform actual sync, but get information about what would happen)\n",
|
||||
"s3cmd --dry-run sync s3://my-bucket-name/ my-local-folder-path/\n",
|
||||
"!s3cmd --dry-run sync s3://my-bucket-name/ my-local-folder-path/\n",
|
||||
"\n",
|
||||
"# Apply a standard shell wildcard include to sync s3 bucket (source) to local (destination)\n",
|
||||
"s3cmd --include '2014-05-01*' sync s3://my-bucket-name/ my-local-folder-path/"
|
||||
"!s3cmd --include '2014-05-01*' sync s3://my-bucket-name/ my-local-folder-path/"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -192,8 +192,8 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"sudo apt-get install boto\n",
|
||||
"sudo apt-get install git"
|
||||
"!sudo apt-get install boto\n",
|
||||
"!sudo apt-get install git"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -210,7 +210,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"git clone https://github.com/twpayne/s3-parallel-put.git"
|
||||
"!git clone https://github.com/twpayne/s3-parallel-put.git"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -227,8 +227,8 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"export AWS_ACCESS_KEY_ID=XXX\n",
|
||||
"export AWS_SECRET_ACCESS_KEY=XXX"
|
||||
"!export AWS_ACCESS_KEY_ID=XXX\n",
|
||||
"!export AWS_SECRET_ACCESS_KEY=XXX"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -245,7 +245,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"s3-parallel-put --bucket=bucket --prefix=PREFIX SOURCE"
|
||||
"!s3-parallel-put --bucket=bucket --prefix=PREFIX SOURCE"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -262,7 +262,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"s3-parallel-put --bucket=bucket --host=s3.amazonaws.com --put=stupid --dry-run --prefix=prefix/ ./"
|
||||
"!s3-parallel-put --bucket=bucket --host=s3.amazonaws.com --put=stupid --dry-run --prefix=prefix/ ./"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -288,7 +288,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"rvm --default ruby-1.8.7-p374"
|
||||
"!rvm --default ruby-1.8.7-p374"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -308,7 +308,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"./elastic-mapreduce --create --instance-group master --instance-count 1 \\\n",
|
||||
"!./elastic-mapreduce --create --instance-group master --instance-count 1 \\\n",
|
||||
"--instance-type m1.small --instance-group core --instance-count 4 \\\n",
|
||||
"--instance-type m1.small --jar /home/hadoop/lib/emr-s3distcp-1.0.jar \\\n",
|
||||
"--args \"--src,s3://my-bucket-source/,--groupBy,.*([0-9]{4}-01).*,\\\n",
|
||||
|
|
@ -361,7 +361,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"python mr-script.py -r emr s3://bucket-source/ --output-dir=s3://bucket-dest/"
|
||||
"!python mr-script.py -r emr s3://bucket-source/ --output-dir=s3://bucket-dest/"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -378,7 +378,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"python mrjob_script.py input_data.txt > output_data.txt"
|
||||
"!python mrjob_script.py input_data.txt > output_data.txt"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -680,7 +680,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws kinesis create-stream --stream-name Foo --shard-count 1 --profile adminuser"
|
||||
"!aws kinesis create-stream --stream-name Foo --shard-count 1 --profile adminuser"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -697,7 +697,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws kinesis list-streams --profile adminuser"
|
||||
"!aws kinesis list-streams --profile adminuser"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -714,7 +714,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws kinesis describe-stream --stream-name Foo --profile adminuser"
|
||||
"!aws kinesis describe-stream --stream-name Foo --profile adminuser"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -731,7 +731,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws kinesis put-record --stream-name Foo --data \"SGVsbG8sIHRoaXMgaXMgYSB0ZXN0IDEyMy4=\" --partition-key shardId-000000000000 --region us-east-1 --profile adminuser"
|
||||
"!aws kinesis put-record --stream-name Foo --data \"SGVsbG8sIHRoaXMgaXMgYSB0ZXN0IDEyMy4=\" --partition-key shardId-000000000000 --region us-east-1 --profile adminuser"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -748,7 +748,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"SHARD_ITERATOR=$(aws kinesis get-shard-iterator --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --stream-name Foo --query 'ShardIterator' --profile adminuser)\n",
|
||||
"!SHARD_ITERATOR=$(aws kinesis get-shard-iterator --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --stream-name Foo --query 'ShardIterator' --profile adminuser)\n",
|
||||
"aws kinesis get-records --shard-iterator $SHARD_ITERATOR"
|
||||
],
|
||||
"language": "python",
|
||||
|
|
@ -766,7 +766,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws kinesis delete-stream --stream-name Foo --profile adminuser"
|
||||
"!aws kinesis delete-stream --stream-name Foo --profile adminuser"
|
||||
],
|
||||
"language": "python",
|
||||
"metadata": {},
|
||||
|
|
@ -790,7 +790,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws lambda list-functions \\\n",
|
||||
"!aws lambda list-functions \\\n",
|
||||
" --region us-east-1 \\\n",
|
||||
" --max-items 10"
|
||||
],
|
||||
|
|
@ -809,7 +809,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws lambda upload-function \\\n",
|
||||
"!aws lambda upload-function \\\n",
|
||||
" --region us-east-1 \\\n",
|
||||
" --function-name foo \\\n",
|
||||
" --function-zip file-path/foo.zip \\\n",
|
||||
|
|
@ -834,7 +834,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws lambda invoke-async \\\n",
|
||||
"!aws lambda invoke-async \\\n",
|
||||
" --function-name foo \\\n",
|
||||
" --region us-east-1 \\\n",
|
||||
" --invoke-args foo.txt \\\n",
|
||||
|
|
@ -855,7 +855,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws lambda get-function-configuration \\\n",
|
||||
"!aws lambda get-function-configuration \\\n",
|
||||
" --function-name helloworld \\\n",
|
||||
" --region us-east-1 \\\n",
|
||||
" --debug"
|
||||
|
|
@ -875,7 +875,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws lambda get-function \\\n",
|
||||
"!aws lambda get-function \\\n",
|
||||
" --function-name helloworld \\\n",
|
||||
" --region us-east-1 \\\n",
|
||||
" --debug"
|
||||
|
|
@ -895,7 +895,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws lambda add-event-source \\\n",
|
||||
"!aws lambda add-event-source \\\n",
|
||||
" --region us-east-1 \\\n",
|
||||
" --function-name ProcessKinesisRecords \\\n",
|
||||
" --role invocation-role-arn \\\n",
|
||||
|
|
@ -918,7 +918,7 @@
|
|||
"cell_type": "code",
|
||||
"collapsed": false,
|
||||
"input": [
|
||||
"aws lambda delete-function \\\n",
|
||||
"!aws lambda delete-function \\\n",
|
||||
" --function-name helloworld \\\n",
|
||||
" --region us-east-1 \\\n",
|
||||
" --debug"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user