Skip to content

Commit cf7fd2e

Browse files
committed
Support the adaptive retry mode in the CRT transfer client
1 parent 5ab71b0 commit cf7fd2e

4 files changed

Lines changed: 67 additions & 22 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"type": "enhancement",
3+
"category": "crt",
4+
"description": "Support ``retry_mode = adaptive`` in the CRT transfer client."
5+
}

‎awscli/customizations/s3/factory.py‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -243,8 +243,6 @@ def _is_eligible_for_crt_client(self, params, runtime_config):
243243

244244
def _get_unsupported_settings(self, params, runtime_config):
245245
unsupported = self._get_classic_only_settings(runtime_config)
246-
if self._is_adaptive_retry_mode():
247-
unsupported.append(f'retry_mode = {ADAPTIVE_RETRY_MODE}')
248246
if self._is_non_seekable_stream_upload(params):
249247
unsupported.append('uploads from a non-seekable stream')
250248
return unsupported
@@ -303,9 +301,6 @@ def warn_unsupported_settings(self, client_type, runtime_config):
303301
unsupported = self._get_unsupported_options(
304302
client_type, runtime_config
305303
)
306-
if client_type == constants.CRT_TRANSFER_CLIENT:
307-
if self._is_adaptive_retry_mode():
308-
unsupported.append(f'retry_mode = {ADAPTIVE_RETRY_MODE}')
309304
if not unsupported:
310305
return
311306
uni_print(
@@ -430,14 +425,17 @@ def _resolve_crt_client_config_kwargs(self, runtime_config):
430425
# passed to opt into the CRT's dynamic part size calculation.
431426
kwargs['part_size'] = None
432427
max_attempts = self._resolve_max_attempts(runtime_config)
428+
retry_options = {}
429+
if self._is_adaptive_retry_mode():
430+
retry_options['retry_mode'] = awscrt.s3.S3RetryMode.ADAPTIVE
433431
if max_attempts == 1:
434-
kwargs['retry_options'] = {
435-
'retry_mode': awscrt.s3.S3RetryMode.NONE
436-
}
432+
retry_options['retry_mode'] = awscrt.s3.S3RetryMode.NONE
437433
elif max_attempts is not None:
438-
kwargs['retry_options'] = {
439-
'max_retries': min(max_attempts - 1, MAX_CRT_MAX_RETRIES)
440-
}
434+
retry_options['max_retries'] = min(
435+
max_attempts - 1, MAX_CRT_MAX_RETRIES
436+
)
437+
if retry_options:
438+
kwargs['retry_options'] = retry_options
441439
return kwargs
442440

443441
def _resolve_target_throughput(self, runtime_config):

‎awscli/topics/s3-config.rst‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,6 @@ files to and from S3. Valid choices are:
344344
``classic`` transfer client when any of the following are in use:
345345

346346
* The ``max_bandwidth`` configuration value.
347-
* A ``retry_mode`` configuration value of ``adaptive``.
348347
* Uploads whose source is a non-seekable stream, such as
349348
``aws s3 cp - s3://bucket/key``.
350349

‎tests/unit/customizations/s3/test_factory.py‎

Lines changed: 53 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626

2727
from awscli.customizations.s3 import constants
2828
from awscli.customizations.s3.factory import (
29-
ADAPTIVE_RETRY_MODE,
3029
CRT_PART_SIZE_EXCEEDS_MEMORY_LIMIT,
3130
MAX_CRT_CONNECT_TIMEOUT_MS,
3231
MAX_CRT_MAX_RETRIES,
@@ -910,11 +909,11 @@ def test_resolves_to_classic_when_max_bandwidth_configured(
910909
== constants.CLASSIC_TRANSFER_CLIENT
911910
)
912911

913-
def test_resolves_to_classic_for_adaptive_retry_mode(
912+
def test_resolves_to_crt_for_adaptive_retry_mode(
914913
self, resolve_client_type, auto_resolve_session
915914
):
916915
stub_config_variables(auto_resolve_session, retry_mode='adaptive')
917-
assert resolve_client_type() == constants.CLASSIC_TRANSFER_CLIENT
916+
assert resolve_client_type() == constants.CRT_TRANSFER_CLIENT
918917

919918
def test_resolves_to_crt_for_standard_retry_mode(
920919
self, resolve_client_type, auto_resolve_session
@@ -983,8 +982,6 @@ def test_warns_when_routed_away_for_max_bandwidth(
983982
def test_does_not_warn_for_adaptive_retry_mode(
984983
self, resolve_client_type, auto_resolve_session, capsys
985984
):
986-
# The crt transfer client will eventually support adaptive retries, so
987-
# there is nothing for the user to act on.
988985
stub_config_variables(auto_resolve_session, retry_mode='adaptive')
989986
resolve_client_type()
990987
assert capsys.readouterr().err == ''
@@ -1124,12 +1121,11 @@ def test_does_not_warn_for_max_bandwidth_when_classic_resolved(
11241121
== ''
11251122
)
11261123

1127-
def test_warns_for_adaptive_retry_mode_under_crt(
1124+
def test_does_not_warn_for_adaptive_retry_mode_under_crt(
11281125
self, warn_unsupported_settings, auto_resolve_session
11291126
):
11301127
stub_config_variables(auto_resolve_session, retry_mode='adaptive')
1131-
warning = warn_unsupported_settings(constants.CRT_TRANSFER_CLIENT)
1132-
assert f'retry_mode = {ADAPTIVE_RETRY_MODE}' in warning
1128+
assert warn_unsupported_settings(constants.CRT_TRANSFER_CLIENT) == ''
11331129

11341130
def test_does_not_warn_for_supported_retry_mode(
11351131
self, warn_unsupported_settings, auto_resolve_session
@@ -1372,6 +1368,49 @@ def test_does_not_warn_when_crt_explicitly_preferred_with_retries_disabled(
13721368
)
13731369
assert warning == ''
13741370

1371+
def test_maps_adaptive_retry_mode_to_crt_client(
1372+
self, crt_client_kwargs, auto_resolve_session
1373+
):
1374+
stub_config_variables(auto_resolve_session, retry_mode='adaptive')
1375+
kwargs = crt_client_kwargs(
1376+
preferred_transfer_client=constants.CRT_TRANSFER_CLIENT
1377+
)
1378+
assert kwargs['retry_options'] == {
1379+
'retry_mode': awscrt.s3.S3RetryMode.ADAPTIVE
1380+
}
1381+
1382+
def test_maps_adaptive_retry_mode_with_max_attempts(
1383+
self, crt_client_kwargs, auto_resolve_session
1384+
):
1385+
stub_config_variables(
1386+
auto_resolve_session, retry_mode='adaptive', max_attempts=5
1387+
)
1388+
stub_configured_variables(auto_resolve_session, 'max_attempts')
1389+
assert crt_client_kwargs()['retry_options'] == {
1390+
'retry_mode': awscrt.s3.S3RetryMode.ADAPTIVE,
1391+
'max_retries': 4,
1392+
}
1393+
1394+
def test_disabled_retries_take_precedence_over_adaptive(
1395+
self, crt_client_kwargs, auto_resolve_session
1396+
):
1397+
stub_config_variables(
1398+
auto_resolve_session, retry_mode='adaptive', max_attempts=1
1399+
)
1400+
stub_configured_variables(auto_resolve_session, 'max_attempts')
1401+
assert crt_client_kwargs()['retry_options'] == {
1402+
'retry_mode': awscrt.s3.S3RetryMode.NONE
1403+
}
1404+
1405+
def test_does_not_map_standard_retry_mode(
1406+
self, crt_client_kwargs, auto_resolve_session
1407+
):
1408+
stub_config_variables(auto_resolve_session, retry_mode='standard')
1409+
kwargs = crt_client_kwargs(
1410+
preferred_transfer_client=constants.CRT_TRANSFER_CLIENT
1411+
)
1412+
assert 'retry_options' not in kwargs
1413+
13751414
def test_maps_configured_max_attempts_to_crt_client(
13761415
self, crt_client_kwargs, auto_resolve_session
13771416
):
@@ -1699,8 +1738,12 @@ def test_awscrt_accepts_max_retries(self, max_retries):
16991738
retry_options={'max_retries': max_retries},
17001739
)
17011740

1702-
def test_awscrt_accepts_none_retry_mode(self):
1741+
@pytest.mark.parametrize(
1742+
'retry_mode',
1743+
[awscrt.s3.S3RetryMode.NONE, awscrt.s3.S3RetryMode.ADAPTIVE],
1744+
)
1745+
def test_awscrt_accepts_retry_mode(self, retry_mode):
17031746
create_s3_crt_client(
17041747
region='us-west-2',
1705-
retry_options={'retry_mode': awscrt.s3.S3RetryMode.NONE},
1748+
retry_options={'retry_mode': retry_mode, 'max_retries': 4},
17061749
)

0 commit comments

Comments
 (0)