190 lines
8 KiB
Python
190 lines
8 KiB
Python
# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
# may not use this file except in compliance with the License. A copy of
|
|
# the License is located at
|
|
#
|
|
# http://aws.amazon.com/apache2.0/
|
|
#
|
|
# or in the "license" file accompanying this file. This file is
|
|
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
# ANY KIND, either express or implied. See the License for the specific
|
|
# language governing permissions and limitations under the License.
|
|
import contextlib
|
|
import json
|
|
from tests import BaseSessionTest, mock, ClientHTTPStubber
|
|
|
|
from botocore.exceptions import ClientError
|
|
from botocore.config import Config
|
|
|
|
|
|
class BaseRetryTest(BaseSessionTest):
|
|
def setUp(self):
|
|
super(BaseRetryTest, self).setUp()
|
|
self.region = 'us-west-2'
|
|
self.sleep_patch = mock.patch('time.sleep')
|
|
self.sleep_patch.start()
|
|
|
|
def tearDown(self):
|
|
super(BaseRetryTest, self).tearDown()
|
|
self.sleep_patch.stop()
|
|
|
|
@contextlib.contextmanager
|
|
def assert_will_retry_n_times(self, client, num_retries,
|
|
status=500, body=b'{}'):
|
|
num_responses = num_retries + 1
|
|
if not isinstance(body, bytes):
|
|
body = json.dumps(body).encode()
|
|
with ClientHTTPStubber(client) as http_stubber:
|
|
for _ in range(num_responses):
|
|
http_stubber.add_response(status=status, body=body)
|
|
with self.assertRaisesRegexp(
|
|
ClientError, 'reached max retries: %s' % num_retries):
|
|
yield
|
|
self.assertEqual(len(http_stubber.requests), num_responses)
|
|
|
|
|
|
class TestLegacyRetry(BaseRetryTest):
|
|
def test_can_override_max_attempts(self):
|
|
client = self.session.create_client(
|
|
'dynamodb', self.region, config=Config(
|
|
retries={'max_attempts': 1}))
|
|
with self.assert_will_retry_n_times(client, 1):
|
|
client.list_tables()
|
|
|
|
def test_do_not_attempt_retries(self):
|
|
client = self.session.create_client(
|
|
'dynamodb', self.region, config=Config(
|
|
retries={'max_attempts': 0}))
|
|
with self.assert_will_retry_n_times(client, 0):
|
|
client.list_tables()
|
|
|
|
def test_setting_max_attempts_does_not_set_for_other_clients(self):
|
|
# Make one client with max attempts configured.
|
|
self.session.create_client(
|
|
'codecommit', self.region, config=Config(
|
|
retries={'max_attempts': 1}))
|
|
|
|
# Make another client that has no custom retry configured.
|
|
client = self.session.create_client('codecommit', self.region)
|
|
# It should use the default max retries, which should be four retries
|
|
# for this service.
|
|
with self.assert_will_retry_n_times(client, 4):
|
|
client.list_repositories()
|
|
|
|
def test_service_specific_defaults_do_not_mutate_general_defaults(self):
|
|
# This tests for a bug where if you created a client for a service
|
|
# with specific retry configurations and then created a client for
|
|
# a service whose retry configurations fallback to the general
|
|
# defaults, the second client would actually use the defaults of
|
|
# the first client.
|
|
|
|
# Make a dynamodb client. It's a special case client that is
|
|
# configured to a make a maximum of 10 requests (9 retries).
|
|
client = self.session.create_client('dynamodb', self.region)
|
|
with self.assert_will_retry_n_times(client, 9):
|
|
client.list_tables()
|
|
|
|
# A codecommit client is not a special case for retries. It will at
|
|
# most make 5 requests (4 retries) for its default.
|
|
client = self.session.create_client('codecommit', self.region)
|
|
with self.assert_will_retry_n_times(client, 4):
|
|
client.list_repositories()
|
|
|
|
def test_set_max_attempts_on_session(self):
|
|
self.session.set_default_client_config(
|
|
Config(retries={'max_attempts': 1}))
|
|
# Max attempts should be inherited from the session.
|
|
client = self.session.create_client('codecommit', self.region)
|
|
with self.assert_will_retry_n_times(client, 1):
|
|
client.list_repositories()
|
|
|
|
def test_can_clobber_max_attempts_on_session(self):
|
|
self.session.set_default_client_config(
|
|
Config(retries={'max_attempts': 1}))
|
|
# Max attempts should override the session's configured max attempts.
|
|
client = self.session.create_client(
|
|
'codecommit', self.region, config=Config(
|
|
retries={'max_attempts': 0}))
|
|
with self.assert_will_retry_n_times(client, 0):
|
|
client.list_repositories()
|
|
|
|
|
|
class TestRetriesV2(BaseRetryTest):
|
|
def create_client_with_retry_mode(self, service, retry_mode,
|
|
max_attempts=None):
|
|
retries = {'mode': retry_mode}
|
|
if max_attempts is not None:
|
|
retries['total_max_attempts'] = max_attempts
|
|
client = self.session.create_client(
|
|
service, self.region, config=Config(retries=retries))
|
|
return client
|
|
|
|
def test_standard_mode_has_default_3_retries(self):
|
|
client = self.create_client_with_retry_mode(
|
|
'dynamodb', retry_mode='standard')
|
|
with self.assert_will_retry_n_times(client, 2):
|
|
client.list_tables()
|
|
|
|
def test_standard_mode_can_configure_max_attempts(self):
|
|
client = self.create_client_with_retry_mode(
|
|
'dynamodb', retry_mode='standard', max_attempts=5)
|
|
with self.assert_will_retry_n_times(client, 4):
|
|
client.list_tables()
|
|
|
|
def test_no_retry_needed_standard_mode(self):
|
|
client = self.create_client_with_retry_mode(
|
|
'dynamodb', retry_mode='standard')
|
|
with ClientHTTPStubber(client) as http_stubber:
|
|
http_stubber.add_response(status=200, body=b'{}')
|
|
client.list_tables()
|
|
|
|
def test_standard_mode_retry_throttling_error(self):
|
|
client = self.create_client_with_retry_mode(
|
|
'dynamodb', retry_mode='standard')
|
|
error_body = {
|
|
"__type": "ThrottlingException",
|
|
"message": "Error"
|
|
}
|
|
with self.assert_will_retry_n_times(client, 2,
|
|
status=400,
|
|
body=error_body):
|
|
client.list_tables()
|
|
|
|
def test_standard_mode_retry_transient_error(self):
|
|
client = self.create_client_with_retry_mode(
|
|
'dynamodb', retry_mode='standard')
|
|
with self.assert_will_retry_n_times(client, 2, status=502):
|
|
client.list_tables()
|
|
|
|
def test_adaptive_mode_still_retries_errors(self):
|
|
# Verify that adaptive mode is just adding on to standard mode.
|
|
client = self.create_client_with_retry_mode(
|
|
'dynamodb', retry_mode='adaptive')
|
|
with self.assert_will_retry_n_times(client, 2):
|
|
client.list_tables()
|
|
|
|
def test_adaptive_mode_retry_transient_error(self):
|
|
client = self.create_client_with_retry_mode(
|
|
'dynamodb', retry_mode='adaptive')
|
|
with self.assert_will_retry_n_times(client, 2, status=502):
|
|
client.list_tables()
|
|
|
|
def test_can_exhaust_default_retry_quota(self):
|
|
# Quota of 500 / 5 retry costs == 100 retry attempts
|
|
# 100 retry attempts / 2 retries per API call == 50 client calls
|
|
client = self.create_client_with_retry_mode(
|
|
'dynamodb', retry_mode='standard')
|
|
for i in range(50):
|
|
with self.assert_will_retry_n_times(client, 2, status=502):
|
|
client.list_tables()
|
|
# Now on the 51th attempt we should see quota errors, which we can
|
|
# verify by looking at the request metadata.
|
|
with ClientHTTPStubber(client) as http_stubber:
|
|
http_stubber.add_response(status=502, body=b'{}')
|
|
with self.assertRaises(ClientError) as e:
|
|
client.list_tables()
|
|
self.assertTrue(
|
|
e.exception.response['ResponseMetadata'].get('RetryQuotaReached')
|
|
)
|