diff --git a/.gitignore b/.gitignore index 9ed67ef1..015a9c69 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ MANIFEST .tox .coverage *flymake.py -venv +.vscode/ +venv* venv-2.5 env-2.5 diff --git a/boto/auth.py b/boto/auth.py index b479d126..3be25e2b 100644 --- a/boto/auth.py +++ b/boto/auth.py @@ -39,7 +39,7 @@ import hmac import os import posixpath -from boto.compat import urllib, encodebytes, parse_qs_safe, urlparse +from boto.compat import urllib, encodebytes, parse_qs_safe, urlparse, six from boto.auth_handler import AuthHandler from boto.exception import BotoClientError @@ -378,8 +378,9 @@ class HmacAuthV4Handler(AuthHandler, HmacKeys): pairs = [] for pname in parameter_names: pval = boto.utils.get_utf8_value(http_request.params[pname]) - pairs.append(urllib.parse.quote(pname, safe='') + '=' + - urllib.parse.quote(pval, safe='-_~')) + pairs.append(urllib.parse.quote(pname, safe=''.encode('ascii')) + + '=' + + urllib.parse.quote(pval, safe='-_~'.encode('ascii'))) return '&'.join(pairs) def canonical_query_string(self, http_request): @@ -830,7 +831,7 @@ class STSAnonHandler(AuthHandler): pairs = [] for key in keys: val = boto.utils.get_utf8_value(params[key]) - pairs.append(key + '=' + self._escape_value(val.decode('utf-8'))) + pairs.append(key + '=' + self._escape_value(six.ensure_str(val))) return '&'.join(pairs) def add_auth(self, http_request, **kwargs): diff --git a/boto/gs/bucket.py b/boto/gs/bucket.py index 8e56dedb..01ac0839 100644 --- a/boto/gs/bucket.py +++ b/boto/gs/bucket.py @@ -19,8 +19,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import re -import urllib import xml.sax import boto @@ -38,6 +41,7 @@ from boto.gs.key import Key as GSKey from boto.s3.acl import Policy from boto.s3.bucket import Bucket as S3Bucket from boto.utils import get_utf8_value +from boto.compat import quote from boto.compat import six # constants for http query args @@ -47,7 +51,11 @@ CORS_ARG = 'cors' ENCRYPTION_CONFIG_ARG = 'encryptionConfig' LIFECYCLE_ARG = 'lifecycle' STORAGE_CLASS_ARG='storageClass' -ERROR_DETAILS_REGEX = re.compile(r'
(?P
.*)
') +_ERROR_DETAILS_REGEX_STR = r'
(?P
.*)
' +if six.PY3: + _ERROR_DETAILS_REGEX_STR = _ERROR_DETAILS_REGEX_STR.encode('ascii') +ERROR_DETAILS_REGEX = re.compile(_ERROR_DETAILS_REGEX_STR) + class Bucket(S3Bucket): """Represents a Google Cloud Storage bucket.""" @@ -117,7 +125,7 @@ class Bucket(S3Bucket): query_args_l.append('generation=%s' % generation) if response_headers: for rk, rv in six.iteritems(response_headers): - query_args_l.append('%s=%s' % (rk, urllib.quote(rv))) + query_args_l.append('%s=%s' % (rk, quote(rv))) try: key, resp = self._get_key_internal(key_name, headers, query_args_l=query_args_l) @@ -353,6 +361,9 @@ class Bucket(S3Bucket): details = (('
%s. Note that Full Control access' ' is required to access ACLs.
') % details) + if six.PY3: + # All args to re.sub() must be of same type + details = details.encode('utf-8') body = re.sub(ERROR_DETAILS_REGEX, details, body) raise self.connection.provider.storage_response_error( response.status, response.reason, body) @@ -453,8 +464,8 @@ class Bucket(S3Bucket): headers['x-goog-if-metageneration-match'] = str(if_metageneration) response = self.connection.make_request( - 'PUT', get_utf8_value(self.name), get_utf8_value(key_name), - data=get_utf8_value(data), headers=headers, query_args=query_args) + 'PUT', self.name, key_name, + data=data, headers=headers, query_args=query_args) body = response.read() if response.status != 200: raise self.connection.provider.storage_response_error( @@ -599,7 +610,7 @@ class Bucket(S3Bucket): :param dict headers: Additional headers to send with the request. """ response = self.connection.make_request( - 'PUT', get_utf8_value(self.name), data=get_utf8_value(cors), + 'PUT', self.name, data=cors, query_args=CORS_ARG, headers=headers) body = response.read() if response.status != 200: @@ -1004,7 +1015,7 @@ class Bucket(S3Bucket): """ xml = lifecycle_config.to_xml() response = self.connection.make_request( - 'PUT', get_utf8_value(self.name), data=get_utf8_value(xml), + 'PUT', self.name, data=xml, query_args=LIFECYCLE_ARG, headers=headers) body = response.read() if response.status == 200: @@ -1126,7 +1137,7 @@ class Bucket(S3Bucket): body = self._construct_encryption_config_xml( default_kms_key_name=default_kms_key_name) response = self.connection.make_request( - 'PUT', get_utf8_value(self.name), data=get_utf8_value(body), + 'PUT', self.name, data=body, query_args=ENCRYPTION_CONFIG_ARG, headers=headers) body = response.read() if response.status != 200: diff --git a/boto/gs/key.py b/boto/gs/key.py index f1ea3e16..b1d4189f 100644 --- a/boto/gs/key.py +++ b/boto/gs/key.py @@ -936,9 +936,9 @@ class Key(S3Key): if content_type: headers['Content-Type'] = content_type resp = self.bucket.connection.make_request( - 'PUT', get_utf8_value(self.bucket.name), get_utf8_value(self.name), + 'PUT', self.bucket.name, self.name, headers=headers, query_args='compose', - data=get_utf8_value(compose_req_xml)) + data=compose_req_xml) if resp.status < 200 or resp.status > 299: raise self.bucket.connection.provider.storage_response_error( resp.status, resp.reason, resp.read()) diff --git a/boto/s3/bucket.py b/boto/s3/bucket.py index 1adae4b1..fdd040c8 100644 --- a/boto/s3/bucket.py +++ b/boto/s3/bucket.py @@ -21,6 +21,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. +from __future__ import division + import boto from boto import handler from boto.resultset import ResultSet @@ -201,7 +203,7 @@ class Bucket(object): response.read() # Allow any success status (2xx) - for example this lets us # support Range gets, which return status 206: - if response.status / 100 == 2: + if response.status // 100 == 2: k = self.key_class(self) provider = self.connection.provider k.metadata = boto.utils.get_aws_metadata(response.msg, provider) @@ -846,7 +848,11 @@ class Bucket(object): """ headers = headers or {} provider = self.connection.provider - src_key_name = boto.utils.get_utf8_value(src_key_name) + if six.PY3: + if isinstance(src_key_name, bytes): + src_key_name = src_key_name.decode('utf-8') + else: + src_key_name = boto.utils.get_utf8_value(src_key_name) if preserve_acl: if self.name == src_bucket_name: src_bucket = self @@ -875,8 +881,6 @@ class Bucket(object): if response.status == 200: key = self.new_key(new_key_name) h = handler.XmlHandler(key, self) - if not isinstance(body, bytes): - body = body.encode('utf-8') xml.sax.parseString(body, h) if hasattr(key, 'Error'): raise provider.storage_copy_error(key.Code, key.Message, body) diff --git a/boto/s3/connection.py b/boto/s3/connection.py index fa3fbd72..54980ef5 100644 --- a/boto/s3/connection.py +++ b/boto/s3/connection.py @@ -24,9 +24,9 @@ import xml.sax import base64 -from boto.compat import six, urllib import time +from boto.compat import six, urllib from boto.auth import detect_potential_s3sigv4 import boto.utils from boto.connection import AWSAuthConnection @@ -89,6 +89,8 @@ class _CallingFormat(object): def build_auth_path(self, bucket, key=''): key = boto.utils.get_utf8_value(key) + if isinstance(bucket, bytes): + bucket = bucket.decode('utf-8') path = '' if bucket != '': path = '/' + bucket diff --git a/boto/s3/key.py b/boto/s3/key.py index 90f4497d..35f04c9f 100644 --- a/boto/s3/key.py +++ b/boto/s3/key.py @@ -20,6 +20,8 @@ # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. +from __future__ import print_function + import email.utils import errno import hashlib @@ -40,6 +42,7 @@ from boto.provider import Provider from boto.s3.keyfile import KeyFile from boto.s3.user import User from boto import UserAgent +import boto.utils from boto.utils import compute_md5, compute_hash from boto.utils import find_matching_headers from boto.utils import merge_headers_by_name @@ -389,7 +392,7 @@ class Key(object): By providing a next method, the key object supports use as an iterator. For example, you can now say: - for bytes in key: + for key_bytes in key: write bytes to a file or whatever All of the HTTP connection stuff is handled for you. @@ -849,9 +852,10 @@ class Key(object): chunk_len = len(chunk) data_len += chunk_len if chunked_transfer: - http_conn.send('%x;\r\n' % chunk_len) + chunk_len_bytes = ('%x' % chunk_len).encode('utf-8') + http_conn.send(chunk_len_bytes + b';\r\n') http_conn.send(chunk) - http_conn.send('\r\n') + http_conn.send(b'\r\n') else: http_conn.send(chunk) for alg in digesters: @@ -879,9 +883,9 @@ class Key(object): self.local_hashes[alg] = digesters[alg].digest() if chunked_transfer: - http_conn.send('0\r\n') + http_conn.send(b'0\r\n') # http_conn.send("Content-MD5: %s\r\n" % self.base64md5) - http_conn.send('\r\n') + http_conn.send(b'\r\n') if cb and (cb_count <= 1 or i > 0) and data_len > 0: cb(data_len, cb_size) @@ -1548,11 +1552,11 @@ class Key(object): i = 0 cb(data_len, cb_size) try: - for bytes in self: - fp.write(bytes) - data_len += len(bytes) + for key_bytes in self: + print(key_bytes, file=fp, end='') + data_len += len(key_bytes) for alg in digesters: - digesters[alg].update(bytes) + digesters[alg].update(key_bytes) if cb: if cb_size > 0 and data_len >= cb_size: break diff --git a/boto/vendored/six.py b/boto/vendored/six.py index a104cb87..f03e4746 100644 --- a/boto/vendored/six.py +++ b/boto/vendored/six.py @@ -1,6 +1,6 @@ """Utilities for writing code that runs on Python 2 and 3""" -# Copyright (c) 2010-2015 Benjamin Peterson +# Copyright (c) 2010-2018 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.9.0" +__version__ = "1.12.0" # Useful for very coarse version differentiation. @@ -165,7 +165,6 @@ class _SixMetaPathImporter(object): """ A meta path importer to import six.moves and its submodules. - This class implements a PEP302 finder and loader. It should be compatible with Python 2.5 and all existing versions of Python3 """ @@ -209,7 +208,6 @@ class _SixMetaPathImporter(object): def is_package(self, fullname): """ Return true, if the named module is a package. - We need this method to get correct spec objects with Python 3.4 (see PEP451) """ @@ -217,7 +215,6 @@ class _SixMetaPathImporter(object): def get_code(self, fullname): """Return None - Required, if is_package is implemented""" self.__get_module(fullname) # eventually raises ImportError return None @@ -241,6 +238,7 @@ _moved_attributes = [ MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), MovedAttribute("reduce", "__builtin__", "functools"), @@ -262,10 +260,11 @@ _moved_attributes = [ MovedModule("html_entities", "htmlentitydefs", "html.entities"), MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), @@ -337,10 +336,12 @@ _urllib_parse_moved_attributes = [ MovedAttribute("quote_plus", "urllib", "urllib.parse"), MovedAttribute("unquote", "urllib", "urllib.parse"), MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), MovedAttribute("urlencode", "urllib", "urllib.parse"), MovedAttribute("splitquery", "urllib", "urllib.parse"), MovedAttribute("splittag", "urllib", "urllib.parse"), MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), MovedAttribute("uses_params", "urlparse", "urllib.parse"), @@ -416,6 +417,8 @@ _urllib_request_moved_attributes = [ MovedAttribute("URLopener", "urllib", "urllib.request"), MovedAttribute("FancyURLopener", "urllib", "urllib.request"), MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), ] for attr in _urllib_request_moved_attributes: setattr(Module_six_moves_urllib_request, attr.name, attr) @@ -679,11 +682,15 @@ if PY3: exec_ = getattr(moves.builtins, "exec") def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None else: def exec_(_code_, _globs_=None, _locs_=None): @@ -699,19 +706,28 @@ else: exec("""exec _code_ in _globs_, _locs_""") exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb + try: + raise tp, value, tb + finally: + tb = None """) if sys.version_info[:2] == (3, 2): exec_("""def raise_from(value, from_value): - if from_value is None: - raise value - raise value from from_value + try: + if from_value is None: + raise value + raise value from from_value + finally: + value = None """) elif sys.version_info[:2] > (3, 2): exec_("""def raise_from(value, from_value): - raise value from from_value + try: + raise value from from_value + finally: + value = None """) else: def raise_from(value, from_value): @@ -802,10 +818,14 @@ def with_metaclass(meta, *bases): # This requires a bit of explanation: the basic idea is to make a dummy # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. - class metaclass(meta): + class metaclass(type): def __new__(cls, name, this_bases, d): return meta(name, bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) return type.__new__(metaclass, 'temporary_class', (), {}) @@ -821,15 +841,69 @@ def add_metaclass(metaclass): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3 it does nothing. - To support Python 2 and 3 with a single code base, define a __str__ method returning text and apply this decorator to the class. """ @@ -865,4 +939,4 @@ if sys.meta_path: break del i, importer # Finally, add the importer to the meta path import hook. -sys.meta_path.append(_importer) +sys.meta_path.append(_importer) \ No newline at end of file