From a0b5eb82ad32dab6a5ce2b9c86b0d2c6fa63e1ae Mon Sep 17 00:00:00 2001 From: Richard Lagerstrom Date: Fri, 8 Apr 2016 11:39:20 +0200 Subject: [PATCH 1/4] [Optional param] set storage class to reduced redundancy --- s4cmd.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/s4cmd.py b/s4cmd.py index d6b6e79..1e3fdc8 100755 --- a/s4cmd.py +++ b/s4cmd.py @@ -1013,7 +1013,7 @@ def upload(self, mpi, source, target, pos = 0, chunk = 0, part = 0): key = boto.s3.key.Key(bucket) key.key = s3url.path key.set_metadata('privilege', self.get_file_privilege(source)) - key.set_contents_from_filename(source) + key.set_contents_from_filename(source, reduced_redundancy=self.opt.reduced_redundancy) message('%s => %s', source, target) return @@ -1413,6 +1413,9 @@ def _totalsize_handler(self, args): '--max-singlepart-upload-size', help = 'files with size (in MB) greater than this will be uploaded in ' 'multipart transfers', type = int, default = 4500 * 1024 * 1024) + parser.add_option( + '--rr', '--reduced-redundancy', help = 'Store object with \'Reduced redundancy\'. Lower per-GB price. [put, cp, mv]', + dest = 'reduced_redundancy', action = 'store_true', default = False) (opt, args) = parser.parse_args() s4cmd_logging.configure(opt) From c35f412eb5109d6a816f3784ca1e895ccf6d226d Mon Sep 17 00:00:00 2001 From: Richard Lagerstrom Date: Fri, 8 Apr 2016 12:53:54 +0200 Subject: [PATCH 2/4] [Optional param] Store objects with public ACL --- s4cmd.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/s4cmd.py b/s4cmd.py index d6b6e79..b40ce98 100755 --- a/s4cmd.py +++ b/s4cmd.py @@ -1014,6 +1014,8 @@ def upload(self, mpi, source, target, pos = 0, chunk = 0, part = 0): key.key = s3url.path key.set_metadata('privilege', self.get_file_privilege(source)) key.set_contents_from_filename(source) + if self.opt.acl_public: + key.set_acl('public-read') message('%s => %s', source, target) return @@ -1413,6 +1415,10 @@ def _totalsize_handler(self, args): '--max-singlepart-upload-size', help = 'files with size (in MB) greater than this will be uploaded in ' 'multipart transfers', type = int, default = 4500 * 1024 * 1024) + parser.add_option( + '-P', '--acl-public', + help = 'Store objects with ACL allowing read for anyone.', + dest = 'acl_public', action = 'store_true', default = False) (opt, args) = parser.parse_args() s4cmd_logging.configure(opt) From cdd35280362232149fb9669a22eb5afa6e31c31a Mon Sep 17 00:00:00 2001 From: Richard Lagerstrom Date: Fri, 8 Apr 2016 12:59:01 +0200 Subject: [PATCH 3/4] Minor adjustment --- s4cmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s4cmd.py b/s4cmd.py index 1e3fdc8..efc4510 100755 --- a/s4cmd.py +++ b/s4cmd.py @@ -1414,8 +1414,8 @@ def _totalsize_handler(self, args): help = 'files with size (in MB) greater than this will be uploaded in ' 'multipart transfers', type = int, default = 4500 * 1024 * 1024) parser.add_option( - '--rr', '--reduced-redundancy', help = 'Store object with \'Reduced redundancy\'. Lower per-GB price. [put, cp, mv]', - dest = 'reduced_redundancy', action = 'store_true', default = False) + '--rr', '--reduced-redundancy', help = 'Store object with \'Reduced redundancy\'. Lower per-GB ' + 'price. [put, cp, mv]', dest = 'reduced_redundancy', action = 'store_true', default = False) (opt, args) = parser.parse_args() s4cmd_logging.configure(opt) From cb3ddc1f31a0d4ad728b0ad9ee3cb532ae12d346 Mon Sep 17 00:00:00 2001 From: Richard Lagerstrom Date: Fri, 8 Apr 2016 13:36:59 +0200 Subject: [PATCH 4/4] [Optional param] Add given HTTP header to upload request. --- s4cmd.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/s4cmd.py b/s4cmd.py index d6b6e79..da1e90b 100755 --- a/s4cmd.py +++ b/s4cmd.py @@ -1008,12 +1008,27 @@ def upload(self, mpi, source, target, pos = 0, chunk = 0, part = 0): elif not self.opt.force and key: raise Failure('File already exists: %s' % target) + # extra headers + extra_headers = {} + if self.opt.add_header: + for hdr in self.opt.add_header: + try: + key, val = hdr.split(":", 1) + except ValueError: + raise Failure("Invalid header format: %s" % hdr) + key_inval = re.sub("[a-zA-Z0-9-.]", "", key) + if key_inval: + key_inval = key_inval.replace(" ", "") + key_inval = key_inval.replace("\t", "") + raise ParameterError("Invalid character(s) in header name '%s': \"%s\"" % (key, key_inval)) + extra_headers[key.strip().lower()] = val.strip() + # Small file optimization. if fsize < self.opt.max_singlepart_upload_size: key = boto.s3.key.Key(bucket) key.key = s3url.path key.set_metadata('privilege', self.get_file_privilege(source)) - key.set_contents_from_filename(source) + key.set_contents_from_filename(source, headers=extra_headers) message('%s => %s', source, target) return @@ -1413,6 +1428,11 @@ def _totalsize_handler(self, args): '--max-singlepart-upload-size', help = 'files with size (in MB) greater than this will be uploaded in ' 'multipart transfers', type = int, default = 4500 * 1024 * 1024) + parser.add_option( + '--add-header', help = 'Add a given HTTP header to the upload request. Can be ' + 'used multiple times. For instance set \'Expires\' or ' + '\'Cache-Control\' headers (or both) using this option.', + dest = 'add_header', action='append', metavar='NAME:VALUE') (opt, args) = parser.parse_args() s4cmd_logging.configure(opt)