464 lines
18 KiB
Python
464 lines
18 KiB
Python
|
# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
|
||
|
|
# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||
|
|
#
|
||
|
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
||
|
|
# copy of this software and associated documentation files (the
|
||
|
|
# "Software"), to deal in the Software without restriction, including
|
||
|
|
# without limitation the rights to use, copy, modify, merge, publish, dis-
|
||
|
|
# tribute, sublicense, and/or sell copies of the Software, and to permit
|
||
|
|
# persons to whom the Software is furnished to do so, subject to the fol-
|
||
|
|
# lowing conditions:
|
||
|
|
#
|
||
|
|
# The above copyright notice and this permission notice shall be included
|
||
|
|
# in all copies or substantial portions of the Software.
|
||
|
|
#
|
||
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||
|
|
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
||
|
|
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
||
|
|
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||
|
|
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||
|
|
# IN THE SOFTWARE.
|
||
|
|
#
|
||
|
|
import base64
|
||
|
|
import datetime
|
||
|
|
from hashlib import sha256
|
||
|
|
from hashlib import sha1
|
||
|
|
import hmac
|
||
|
|
import logging
|
||
|
|
from email.utils import formatdate
|
||
|
|
from operator import itemgetter
|
||
|
|
import functools
|
||
|
|
|
||
|
|
from botocore.exceptions import NoCredentialsError
|
||
|
|
from botocore.utils import normalize_url_path
|
||
|
|
from botocore.compat import HTTPHeaders
|
||
|
|
from botocore.compat import quote, unquote, urlsplit
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
EMPTY_SHA256_HASH = (
|
||
|
|
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
|
||
|
|
# This is the buffer size used when calculating sha256 checksums.
|
||
|
|
# Experimenting with various buffer sizes showed that this value generally
|
||
|
|
# gave the best result (in terms of performance).
|
||
|
|
PAYLOAD_BUFFER = 1024 * 1024
|
||
|
|
|
||
|
|
|
||
|
|
class BaseSigner(object):
|
||
|
|
REQUIRES_REGION = False
|
||
|
|
|
||
|
|
def add_auth(self, request):
|
||
|
|
raise NotImplementedError("add_auth")
|
||
|
|
|
||
|
|
|
||
|
|
class SigV2Auth(BaseSigner):
|
||
|
|
"""
|
||
|
|
Sign a request with Signature V2.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, credentials):
|
||
|
|
self.credentials = credentials
|
||
|
|
if self.credentials is None:
|
||
|
|
raise NoCredentialsError
|
||
|
|
|
||
|
|
def calc_signature(self, request, params):
|
||
|
|
logger.debug("Calculating signature using v2 auth.")
|
||
|
|
split = urlsplit(request.url)
|
||
|
|
path = split.path
|
||
|
|
if len(path) == 0:
|
||
|
|
path = '/'
|
||
|
|
string_to_sign = '%s\n%s\n%s\n' % (request.method,
|
||
|
|
split.netloc,
|
||
|
|
path)
|
||
|
|
lhmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
|
||
|
|
digestmod=sha256)
|
||
|
|
pairs = []
|
||
|
|
for key in sorted(params):
|
||
|
|
value = params[key]
|
||
|
|
pairs.append(quote(key, safe='') + '=' +
|
||
|
|
quote(value, safe='-_~'))
|
||
|
|
qs = '&'.join(pairs)
|
||
|
|
string_to_sign += qs
|
||
|
|
logger.debug('String to sign: %s', string_to_sign)
|
||
|
|
lhmac.update(string_to_sign.encode('utf-8'))
|
||
|
|
b64 = base64.b64encode(lhmac.digest()).strip().decode('utf-8')
|
||
|
|
return (qs, b64)
|
||
|
|
|
||
|
|
def add_auth(self, request):
|
||
|
|
# The auth handler is the last thing called in the
|
||
|
|
# preparation phase of a prepared request.
|
||
|
|
# Because of this we have to parse the query params
|
||
|
|
# from the request body so we can update them with
|
||
|
|
# the sigv2 auth params.
|
||
|
|
if request.data:
|
||
|
|
# POST
|
||
|
|
params = request.data
|
||
|
|
else:
|
||
|
|
# GET
|
||
|
|
params = request.param
|
||
|
|
params['AWSAccessKeyId'] = self.credentials.access_key
|
||
|
|
params['SignatureVersion'] = '2'
|
||
|
|
params['SignatureMethod'] = 'HmacSHA256'
|
||
|
|
params['Timestamp'] = datetime.datetime.utcnow().isoformat()
|
||
|
|
if self.credentials.token:
|
||
|
|
params['SecurityToken'] = self.credentials.token
|
||
|
|
qs, signature = self.calc_signature(request, params)
|
||
|
|
params['Signature'] = signature
|
||
|
|
return request
|
||
|
|
|
||
|
|
|
||
|
|
class SigV3Auth(BaseSigner):
|
||
|
|
def __init__(self, credentials):
|
||
|
|
self.credentials = credentials
|
||
|
|
if self.credentials is None:
|
||
|
|
raise NoCredentialsError
|
||
|
|
|
||
|
|
def add_auth(self, request):
|
||
|
|
if 'Date' not in request.headers:
|
||
|
|
request.headers['Date'] = formatdate(usegmt=True)
|
||
|
|
if self.credentials.token:
|
||
|
|
request.headers['X-Amz-Security-Token'] = self.credentials.token
|
||
|
|
new_hmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
|
||
|
|
digestmod=sha256)
|
||
|
|
new_hmac.update(request.headers['Date'].encode('utf-8'))
|
||
|
|
encoded_signature = base64.encodestring(new_hmac.digest()).strip()
|
||
|
|
signature = ('AWS3-HTTPS AWSAccessKeyId=%s,Algorithm=%s,Signature=%s' %
|
||
|
|
(self.credentials.access_key, 'HmacSHA256',
|
||
|
|
encoded_signature.decode('utf-8')))
|
||
|
|
request.headers['X-Amzn-Authorization'] = signature
|
||
|
|
|
||
|
|
|
||
|
|
class SigV4Auth(BaseSigner):
|
||
|
|
"""
|
||
|
|
Sign a request with Signature V4.
|
||
|
|
"""
|
||
|
|
REQUIRES_REGION = True
|
||
|
|
|
||
|
|
def __init__(self, credentials, service_name, region_name):
|
||
|
|
self.credentials = credentials
|
||
|
|
if self.credentials is None:
|
||
|
|
raise NoCredentialsError
|
||
|
|
# We initialize these value here so the unit tests can have
|
||
|
|
# valid values. But these will get overriden in ``add_auth``
|
||
|
|
# later for real requests.
|
||
|
|
now = datetime.datetime.utcnow()
|
||
|
|
self.timestamp = now.strftime('%Y%m%dT%H%M%SZ')
|
||
|
|
self._region_name = region_name
|
||
|
|
self._service_name = service_name
|
||
|
|
|
||
|
|
def _sign(self, key, msg, hex=False):
|
||
|
|
if hex:
|
||
|
|
sig = hmac.new(key, msg.encode('utf-8'), sha256).hexdigest()
|
||
|
|
else:
|
||
|
|
sig = hmac.new(key, msg.encode('utf-8'), sha256).digest()
|
||
|
|
return sig
|
||
|
|
|
||
|
|
def headers_to_sign(self, request):
|
||
|
|
"""
|
||
|
|
Select the headers from the request that need to be included
|
||
|
|
in the StringToSign.
|
||
|
|
"""
|
||
|
|
header_map = HTTPHeaders()
|
||
|
|
split = urlsplit(request.url)
|
||
|
|
for name, value in request.headers.items():
|
||
|
|
lname = name.lower()
|
||
|
|
header_map[lname] = value
|
||
|
|
if 'host' not in header_map:
|
||
|
|
header_map['host'] = split.netloc
|
||
|
|
return header_map
|
||
|
|
|
||
|
|
def canonical_query_string(self, request):
|
||
|
|
cqs = ''
|
||
|
|
if request.params:
|
||
|
|
params = request.params
|
||
|
|
l = []
|
||
|
|
for param in params:
|
||
|
|
value = str(params[param])
|
||
|
|
l.append('%s=%s' % (quote(param, safe='-_.~'),
|
||
|
|
quote(value, safe='-_.~')))
|
||
|
|
l = sorted(l)
|
||
|
|
cqs = '&'.join(l)
|
||
|
|
return cqs
|
||
|
|
|
||
|
|
def canonical_headers(self, headers_to_sign):
|
||
|
|
"""
|
||
|
|
Return the headers that need to be included in the StringToSign
|
||
|
|
in their canonical form by converting all header keys to lower
|
||
|
|
case, sorting them in alphabetical order and then joining
|
||
|
|
them into a string, separated by newlines.
|
||
|
|
"""
|
||
|
|
headers = []
|
||
|
|
sorted_header_names = sorted(set(headers_to_sign))
|
||
|
|
for key in sorted_header_names:
|
||
|
|
value = ','.join(v.strip() for v in
|
||
|
|
sorted(headers_to_sign.get_all(key)))
|
||
|
|
headers.append('%s:%s' % (key, value))
|
||
|
|
return '\n'.join(headers)
|
||
|
|
|
||
|
|
def signed_headers(self, headers_to_sign):
|
||
|
|
l = ['%s' % n.lower().strip() for n in set(headers_to_sign)]
|
||
|
|
l = sorted(l)
|
||
|
|
return ';'.join(l)
|
||
|
|
|
||
|
|
def payload(self, request):
|
||
|
|
if request.body and hasattr(request.body, 'seek'):
|
||
|
|
position = request.body.tell()
|
||
|
|
read_chunksize = functools.partial(request.body.read,
|
||
|
|
PAYLOAD_BUFFER)
|
||
|
|
checksum = sha256()
|
||
|
|
for chunk in iter(read_chunksize, b''):
|
||
|
|
checksum.update(chunk)
|
||
|
|
hex_checksum = checksum.hexdigest()
|
||
|
|
request.body.seek(position)
|
||
|
|
return hex_checksum
|
||
|
|
elif request.body:
|
||
|
|
return sha256(request.body.encode('utf-8')).hexdigest()
|
||
|
|
else:
|
||
|
|
return EMPTY_SHA256_HASH
|
||
|
|
|
||
|
|
def canonical_request(self, request):
|
||
|
|
cr = [request.method.upper()]
|
||
|
|
path = self._normalize_url_path(urlsplit(request.url).path)
|
||
|
|
cr.append(path)
|
||
|
|
cr.append(self.canonical_query_string(request))
|
||
|
|
headers_to_sign = self.headers_to_sign(request)
|
||
|
|
cr.append(self.canonical_headers(headers_to_sign) + '\n')
|
||
|
|
cr.append(self.signed_headers(headers_to_sign))
|
||
|
|
if 'X-Amz-Content-SHA256' in request.headers:
|
||
|
|
body_checksum = request.headers['X-Amz-Content-SHA256']
|
||
|
|
else:
|
||
|
|
body_checksum = self.payload(request)
|
||
|
|
cr.append(body_checksum)
|
||
|
|
return '\n'.join(cr)
|
||
|
|
|
||
|
|
def _normalize_url_path(self, path):
|
||
|
|
return normalize_url_path(path)
|
||
|
|
|
||
|
|
def scope(self, args):
|
||
|
|
scope = [self.credentials.access_key]
|
||
|
|
scope.append(self.timestamp[0:8])
|
||
|
|
scope.append(self._region_name)
|
||
|
|
scope.append(self._service_name)
|
||
|
|
scope.append('aws4_request')
|
||
|
|
return '/'.join(scope)
|
||
|
|
|
||
|
|
def credential_scope(self, args):
|
||
|
|
scope = []
|
||
|
|
scope.append(self.timestamp[0:8])
|
||
|
|
scope.append(self._region_name)
|
||
|
|
scope.append(self._service_name)
|
||
|
|
scope.append('aws4_request')
|
||
|
|
return '/'.join(scope)
|
||
|
|
|
||
|
|
def string_to_sign(self, request, canonical_request):
|
||
|
|
"""
|
||
|
|
Return the canonical StringToSign as well as a dict
|
||
|
|
containing the original version of all headers that
|
||
|
|
were included in the StringToSign.
|
||
|
|
"""
|
||
|
|
sts = ['AWS4-HMAC-SHA256']
|
||
|
|
sts.append(self.timestamp)
|
||
|
|
sts.append(self.credential_scope(request))
|
||
|
|
sts.append(sha256(canonical_request.encode('utf-8')).hexdigest())
|
||
|
|
return '\n'.join(sts)
|
||
|
|
|
||
|
|
def signature(self, string_to_sign):
|
||
|
|
key = self.credentials.secret_key
|
||
|
|
k_date = self._sign(('AWS4' + key).encode('utf-8'),
|
||
|
|
self.timestamp[0:8])
|
||
|
|
k_region = self._sign(k_date, self._region_name)
|
||
|
|
k_service = self._sign(k_region, self._service_name)
|
||
|
|
k_signing = self._sign(k_service, 'aws4_request')
|
||
|
|
return self._sign(k_signing, string_to_sign, hex=True)
|
||
|
|
|
||
|
|
def add_auth(self, request):
|
||
|
|
# Create a new timestamp for each signing event
|
||
|
|
now = datetime.datetime.utcnow()
|
||
|
|
self.timestamp = now.strftime('%Y%m%dT%H%M%SZ')
|
||
|
|
# This could be a retry. Make sure the previous
|
||
|
|
# authorization header is removed first.
|
||
|
|
self._add_headers_before_signing(request)
|
||
|
|
canonical_request = self.canonical_request(request)
|
||
|
|
logger.debug("Calculating signature using v4 auth.")
|
||
|
|
logger.debug('CanonicalRequest:\n%s', canonical_request)
|
||
|
|
string_to_sign = self.string_to_sign(request, canonical_request)
|
||
|
|
logger.debug('StringToSign:\n%s', string_to_sign)
|
||
|
|
signature = self.signature(string_to_sign)
|
||
|
|
logger.debug('Signature:\n%s', signature)
|
||
|
|
|
||
|
|
l = ['AWS4-HMAC-SHA256 Credential=%s' % self.scope(request)]
|
||
|
|
headers_to_sign = self.headers_to_sign(request)
|
||
|
|
l.append('SignedHeaders=%s' % self.signed_headers(headers_to_sign))
|
||
|
|
l.append('Signature=%s' % signature)
|
||
|
|
request.headers['Authorization'] = ', '.join(l)
|
||
|
|
return request
|
||
|
|
|
||
|
|
def _add_headers_before_signing(self, request):
|
||
|
|
if 'Authorization' in request.headers:
|
||
|
|
del request.headers['Authorization']
|
||
|
|
if 'Date' not in request.headers:
|
||
|
|
request.headers['X-Amz-Date'] = self.timestamp
|
||
|
|
if self.credentials.token:
|
||
|
|
request.headers['X-Amz-Security-Token'] = self.credentials.token
|
||
|
|
|
||
|
|
|
||
|
|
class S3SigV4Auth(SigV4Auth):
|
||
|
|
|
||
|
|
def canonical_query_string(self, request):
|
||
|
|
split = urlsplit(request.url)
|
||
|
|
buf = ''
|
||
|
|
if split.query:
|
||
|
|
qsa = split.query.split('&')
|
||
|
|
qsa = [a.split('=', 1) for a in qsa]
|
||
|
|
quoted_qsa = []
|
||
|
|
for q in qsa:
|
||
|
|
if len(q) == 2:
|
||
|
|
quoted_qsa.append(
|
||
|
|
'%s=%s' % (quote(q[0], safe='-_.~'),
|
||
|
|
quote(unquote(q[1]), safe='-_.~')))
|
||
|
|
elif len(q) == 1:
|
||
|
|
quoted_qsa.append('%s=' % quote(q[0], safe='-_.~'))
|
||
|
|
if len(quoted_qsa) > 0:
|
||
|
|
quoted_qsa.sort(key=itemgetter(0))
|
||
|
|
buf += '&'.join(quoted_qsa)
|
||
|
|
return buf
|
||
|
|
|
||
|
|
def _add_headers_before_signing(self, request):
|
||
|
|
super(S3SigV4Auth, self)._add_headers_before_signing(request)
|
||
|
|
request.headers['X-Amz-Content-SHA256'] = self.payload(request)
|
||
|
|
|
||
|
|
def _normalize_url_path(self, path):
|
||
|
|
# For S3, we do not normalize the path.
|
||
|
|
return path
|
||
|
|
|
||
|
|
|
||
|
|
class HmacV1Auth(BaseSigner):
|
||
|
|
|
||
|
|
# List of Query String Arguments of Interest
|
||
|
|
QSAOfInterest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging',
|
||
|
|
'partNumber', 'policy', 'requestPayment', 'torrent',
|
||
|
|
'versioning', 'versionId', 'versions', 'website',
|
||
|
|
'uploads', 'uploadId', 'response-content-type',
|
||
|
|
'response-content-language', 'response-expires',
|
||
|
|
'response-cache-control', 'response-content-disposition',
|
||
|
|
'response-content-encoding', 'delete', 'lifecycle',
|
||
|
|
'tagging', 'restore', 'storageClass', 'notification']
|
||
|
|
|
||
|
|
def __init__(self, credentials, service_name=None, region_name=None):
|
||
|
|
self.credentials = credentials
|
||
|
|
if self.credentials is None:
|
||
|
|
raise NoCredentialsError
|
||
|
|
self.auth_path = None # see comment in canonical_resource below
|
||
|
|
|
||
|
|
def sign_string(self, string_to_sign):
|
||
|
|
new_hmac = hmac.new(self.credentials.secret_key.encode('utf-8'),
|
||
|
|
digestmod=sha1)
|
||
|
|
new_hmac.update(string_to_sign.encode('utf-8'))
|
||
|
|
return base64.encodestring(new_hmac.digest()).strip().decode('utf-8')
|
||
|
|
|
||
|
|
def canonical_standard_headers(self, headers):
|
||
|
|
interesting_headers = ['content-md5', 'content-type', 'date']
|
||
|
|
hoi = []
|
||
|
|
if 'Date' not in headers:
|
||
|
|
headers['Date'] = formatdate(usegmt=True)
|
||
|
|
for ih in interesting_headers:
|
||
|
|
found = False
|
||
|
|
for key in headers:
|
||
|
|
lk = key.lower()
|
||
|
|
if headers[key] is not None and lk == ih:
|
||
|
|
hoi.append(headers[key].strip())
|
||
|
|
found = True
|
||
|
|
if not found:
|
||
|
|
hoi.append('')
|
||
|
|
return '\n'.join(hoi)
|
||
|
|
|
||
|
|
def canonical_custom_headers(self, headers):
|
||
|
|
custom_headers = {}
|
||
|
|
for key in headers:
|
||
|
|
lk = key.lower()
|
||
|
|
if headers[key] is not None:
|
||
|
|
# TODO: move hardcoded prefix to provider
|
||
|
|
if lk.startswith('x-amz-'):
|
||
|
|
custom_headers[lk] = ','.join(v.strip() for v in
|
||
|
|
headers.get_all(key))
|
||
|
|
sorted_header_keys = sorted(custom_headers.keys())
|
||
|
|
hoi = []
|
||
|
|
for key in sorted_header_keys:
|
||
|
|
hoi.append("%s:%s" % (key, custom_headers[key]))
|
||
|
|
return '\n'.join(hoi)
|
||
|
|
|
||
|
|
def unquote_v(self, nv):
|
||
|
|
"""
|
||
|
|
TODO: Do we need this?
|
||
|
|
"""
|
||
|
|
if len(nv) == 1:
|
||
|
|
return nv
|
||
|
|
else:
|
||
|
|
return (nv[0], unquote(nv[1]))
|
||
|
|
|
||
|
|
def canonical_resource(self, split):
|
||
|
|
# don't include anything after the first ? in the resource...
|
||
|
|
# unless it is one of the QSA of interest, defined above
|
||
|
|
# NOTE:
|
||
|
|
# The path in the canonical resource should always be the
|
||
|
|
# full path including the bucket name, even for virtual-hosting
|
||
|
|
# style addressing. The ``auth_path`` keeps track of the full
|
||
|
|
# path for the canonical resource and would be passed in if
|
||
|
|
# the client was using virtual-hosting style.
|
||
|
|
if self.auth_path:
|
||
|
|
buf = self.auth_path
|
||
|
|
else:
|
||
|
|
buf = split.path
|
||
|
|
if split.query:
|
||
|
|
qsa = split.query.split('&')
|
||
|
|
qsa = [a.split('=', 1) for a in qsa]
|
||
|
|
qsa = [self.unquote_v(a) for a in qsa if a[0] in self.QSAOfInterest]
|
||
|
|
if len(qsa) > 0:
|
||
|
|
qsa.sort(key=itemgetter(0))
|
||
|
|
qsa = ['='.join(a) for a in qsa]
|
||
|
|
buf += '?'
|
||
|
|
buf += '&'.join(qsa)
|
||
|
|
return buf
|
||
|
|
|
||
|
|
def canonical_string(self, method, split, headers, expires=None):
|
||
|
|
cs = method.upper() + '\n'
|
||
|
|
cs += self.canonical_standard_headers(headers) + '\n'
|
||
|
|
custom_headers = self.canonical_custom_headers(headers)
|
||
|
|
if custom_headers:
|
||
|
|
cs += custom_headers + '\n'
|
||
|
|
cs += self.canonical_resource(split)
|
||
|
|
return cs
|
||
|
|
|
||
|
|
def get_signature(self, method, split, headers, expires=None):
|
||
|
|
if self.credentials.token:
|
||
|
|
#TODO: remove hardcoded header name
|
||
|
|
headers['x-amz-security-token'] = self.credentials.token
|
||
|
|
string_to_sign = self.canonical_string(method,
|
||
|
|
split,
|
||
|
|
headers)
|
||
|
|
logger.debug('StringToSign:\n%s', string_to_sign)
|
||
|
|
return self.sign_string(string_to_sign)
|
||
|
|
|
||
|
|
def add_auth(self, request):
|
||
|
|
logger.debug("Calculating signature using hmacv1 auth.")
|
||
|
|
split = urlsplit(request.url)
|
||
|
|
logger.debug('HTTP request method: %s', request.method)
|
||
|
|
signature = self.get_signature(request.method, split,
|
||
|
|
request.headers)
|
||
|
|
request.headers['Authorization'] = ("AWS %s:%s" % (self.credentials.access_key,
|
||
|
|
signature))
|
||
|
|
|
||
|
|
|
||
|
|
# Defined at the bottom instead of the top of the module because the Auth
|
||
|
|
# classes weren't defined yet.
|
||
|
|
AUTH_TYPE_MAPS = {
|
||
|
|
'v2': SigV2Auth,
|
||
|
|
'v4': SigV4Auth,
|
||
|
|
'v3': SigV3Auth,
|
||
|
|
'v3https': SigV3Auth,
|
||
|
|
's3': HmacV1Auth,
|
||
|
|
's3v4': S3SigV4Auth,
|
||
|
|
}
|