mirror of
https://github.com/donnemartin/data-science-ipython-notebooks.git
synced 2024-03-22 13:30:56 +08:00
Added snippets for setting up S3cmd and several frequently used commands.
This commit is contained in:
parent
eff1a1fab4
commit
50a79a011c
|
@ -128,6 +128,108 @@
|
||||||
"language": "python",
|
"language": "python",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": []
|
"outputs": []
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Setup S3cmd"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Install s3cmd:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"collapsed": false,
|
||||||
|
"input": [
|
||||||
|
"sudo apt-get install s3cmd"
|
||||||
|
],
|
||||||
|
"language": "python",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Configure s3cmd:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"collapsed": false,
|
||||||
|
"input": [
|
||||||
|
"s3cmd --configure"
|
||||||
|
],
|
||||||
|
"language": "python",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Frequently used S3cmds:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"collapsed": false,
|
||||||
|
"input": [
|
||||||
|
"# List all buckets\n",
|
||||||
|
"s3cmd ls\n",
|
||||||
|
"\n",
|
||||||
|
"# List the contents of the bucket\n",
|
||||||
|
"s3cmd ls s3://my-bucket-name\n",
|
||||||
|
"\n",
|
||||||
|
"# Upload a file into the bucket (private)\n",
|
||||||
|
"s3cmd put myfile.txt s3://my-bucket-name/myfile.txt\n",
|
||||||
|
"\n",
|
||||||
|
"# Upload a file into the bucket (public)\n",
|
||||||
|
"s3cmd put --acl-public --guess-mime-type myfile.txt s3://my-bucket-name/myfile.txt\n",
|
||||||
|
"\n",
|
||||||
|
"# Recursively upload a directory to s3\n",
|
||||||
|
"s3cmd put --recursive my-local-folder-path/ s3://my-bucket-name/mydir/\n",
|
||||||
|
"\n",
|
||||||
|
"# Download a file\n",
|
||||||
|
"s3cmd get s3://my-bucket-name/myfile.txt myfile.txt\n",
|
||||||
|
"\n",
|
||||||
|
"# Recursively download files that start with myfile\n",
|
||||||
|
"s3cmd --recursive get s3://my-bucket-name/myfile\n",
|
||||||
|
"\n",
|
||||||
|
"# Delete a file\n",
|
||||||
|
"s3cmd del s3://my-bucket-name/myfile.txt\n",
|
||||||
|
"\n",
|
||||||
|
"# Delete a bucket\n",
|
||||||
|
"s3cmd del --recursive s3://my-bucket-name/\n",
|
||||||
|
"\n",
|
||||||
|
"# Create a bucket\n",
|
||||||
|
"s3cmd mb s3://my-bucket-name\n",
|
||||||
|
"\n",
|
||||||
|
"# List bucket disk usage (human readable)\n",
|
||||||
|
"s3cmd du -H s3://my-bucket-name/\n",
|
||||||
|
"\n",
|
||||||
|
"# Sync local (source) to s3 bucket (destination)\n",
|
||||||
|
"s3cmd sync my-local-folder-path/ s3://my-bucket-name/\n",
|
||||||
|
"\n",
|
||||||
|
"# Sync s3 bucket (source) to local (destination)\n",
|
||||||
|
"s3cmd sync s3://my-bucket-name/ my-local-folder-path/\n",
|
||||||
|
"\n",
|
||||||
|
"# Do a dry-run (do not perform actual sync, but get information about what would happen)\n",
|
||||||
|
"s3cmd --dry-run sync s3://my-bucket-name/ my-local-folder-path/\n",
|
||||||
|
"\n",
|
||||||
|
"# Apply a standard shell wildcard include to sync s3 bucket (source) to local (destination)\n",
|
||||||
|
"s3cmd --include '2014-05-01*' sync s3://my-bucket-name/ my-local-folder-path/"
|
||||||
|
],
|
||||||
|
"language": "python",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": []
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"metadata": {}
|
"metadata": {}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user