Merge pull request #282 from kyleknap/crt-throughput

Update throughput target defaults

.changes/next-release/enhancement-crt-28261.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "``crt``",
+  "description": "Update ``target_throughput`` defaults. If not configured, s3transfer will use the AWS CRT to attempt to determine a recommended target throughput to use based on the system. If there is no recommended throughput, s3transfer now falls back to ten gigabits per second."
+}

s3transfer/crt.py

@@ -25,13 +25,18 @@
     EventLoopGroup,
     TlsContextOptions,
 )
-from awscrt.s3 import S3Client, S3RequestTlsMode, S3RequestType
+from awscrt.s3 import (
+    S3Client,
+    S3RequestTlsMode,
+    S3RequestType,
+    get_recommended_throughput_target_gbps,
+)
 from botocore import UNSIGNED
 from botocore.compat import urlsplit
 from botocore.config import Config
 from botocore.exceptions import NoCredentialsError
 
-from s3transfer.constants import GB, MB
+from s3transfer.constants import MB
 from s3transfer.exceptions import TransferNotDoneError
 from s3transfer.futures import BaseTransferFuture, BaseTransferMeta
 from s3transfer.utils import CallArgs, OSUtils, get_callbacks
@@ -94,7 +99,7 @@ def create_s3_crt_client(
     region,
     botocore_credential_provider=None,
     num_threads=None,
-    target_throughput=5 * GB / 8,
+    target_throughput=None,
     part_size=8 * MB,
     use_ssl=True,
     verify=None,
@@ -113,8 +118,14 @@ def create_s3_crt_client(
         is the number of processors in the machine.
 
     :type target_throughput: Optional[int]
-    :param target_throughput: Throughput target in Bytes.
-        Default is 0.625 GB/s (which translates to 5 Gb/s).
+    :param target_throughput: Throughput target in bytes per second.
+        By default, CRT will automatically attempt to choose a target
+        throughput that matches the system's maximum network throughput.
+        Currently, if CRT is unable to determine the maximum network
+        throughput, a fallback target throughput of ``1_250_000_000`` bytes
+        per second (which translates to 10 gigabits per second, or 1.16
+        gibibytes per second) is used. To set a specific target
+        throughput, set a value for this parameter.
 
     :type part_size: Optional[int]
     :param part_size: Size, in Bytes, of parts that files will be downloaded
@@ -163,8 +174,9 @@ def create_s3_crt_client(
         provider = AwsCredentialsProvider.new_delegate(
             credentails_provider_adapter
         )
-
-    target_gbps = target_throughput * 8 / GB
+    target_gbps = _get_crt_throughput_target_gbps(
+        provided_throughput_target_bytes=target_throughput
+    )
     return S3Client(
         bootstrap=bootstrap,
         region=region,
@@ -176,6 +188,24 @@ def create_s3_crt_client(
     )
 
 
+def _get_crt_throughput_target_gbps(provided_throughput_target_bytes=None):
+    if provided_throughput_target_bytes is None:
+        target_gbps = get_recommended_throughput_target_gbps()
+        logger.debug(
+            'Recommended CRT throughput target in gbps: %s', target_gbps
+        )
+        if target_gbps is None:
+            target_gbps = 10.0
+    else:
+        # NOTE: The GB constant in s3transfer is technically a gibibyte. The
+        # GB constant is not used here because the CRT interprets gigabits
+        # for networking as a base power of 10
+        # (i.e. 1000 ** 3 instead of 1024 ** 3).
+        target_gbps = provided_throughput_target_bytes * 8 / 1_000_000_000
+    logger.debug('Using CRT throughput target in gbps: %s', target_gbps)
+    return target_gbps
+
+
 class CRTTransferManager:
     def __init__(self, crt_s3_client, crt_request_serializer, osutil=None):
         """A transfer manager interface for Amazon S3 on CRT s3 client.

tests/unit/test_crt.py

@@ -26,6 +26,11 @@
     import s3transfer.crt
 
 
+requires_crt_pytest = pytest.mark.skipif(
+    not HAS_CRT, reason="Test requires awscrt to be installed."
+)
+
+
 @pytest.fixture
 def mock_crt_process_lock(monkeypatch):
     # The process lock is cached at the module layer whenever the
@@ -38,13 +43,25 @@ def mock_crt_process_lock(monkeypatch):
         yield mock_lock
 
 
+@pytest.fixture
+def mock_s3_crt_client():
+    with mock.patch('s3transfer.crt.S3Client', spec=True) as mock_client:
+        yield mock_client
+
+
+@pytest.fixture
+def mock_get_recommended_throughput_target_gbps():
+    with mock.patch(
+        's3transfer.crt.get_recommended_throughput_target_gbps'
+    ) as mock_get_target_gbps:
+        yield mock_get_target_gbps
+
+
 class CustomFutureException(Exception):
     pass
 
 
-@pytest.mark.skipif(
-    not HAS_CRT, reason="Test requires awscrt to be installed."
-)
+@requires_crt_pytest
 class TestCRTProcessLock:
     def test_acquire_crt_s3_process_lock(self, mock_crt_process_lock):
         lock = s3transfer.crt.acquire_crt_s3_process_lock('app-name')
@@ -223,3 +240,38 @@ def test_call(self):
         writer = s3transfer.crt.OnBodyFileObjWriter(fileobj)
         writer(chunk=b'content')
         self.assertEqual(fileobj.getvalue(), b'content')
+
+
+@requires_crt_pytest
+class TestCreateS3CRTClient:
+    @pytest.mark.parametrize(
+        'provided_bytes_per_sec,recommended_gbps,expected_gbps',
+        [
+            (None, 100.0, 100.0),
+            (None, None, 10.0),
+            # NOTE: create_s3_crt_client() accepts target throughput as bytes
+            # per second and it is converted to gigabits per second for the
+            # CRT client instantiation.
+            (1_000_000_000, None, 8.0),
+            (1_000_000_000, 100.0, 8.0),
+        ],
+    )
+    def test_target_throughput(
+        self,
+        provided_bytes_per_sec,
+        recommended_gbps,
+        expected_gbps,
+        mock_s3_crt_client,
+        mock_get_recommended_throughput_target_gbps,
+    ):
+        mock_get_recommended_throughput_target_gbps.return_value = (
+            recommended_gbps
+        )
+        s3transfer.crt.create_s3_crt_client(
+            'us-west-2',
+            target_throughput=provided_bytes_per_sec,
+        )
+        assert (
+            mock_s3_crt_client.call_args[1]['throughput_target_gbps']
+            == expected_gbps
+        )