From 5a266926e5dfdb57c07e6eb1365a6ccb73331d71 Mon Sep 17 00:00:00 2001 From: Ernest Zaslavsky Date: Thu, 6 Feb 2025 12:52:33 +0200 Subject: [PATCH] s3_client: Increase default part size for optimal performance Set the `upload_file` part size to 50MiB, as this value provides the best performance based on tests conducted using `perf_s3_client` on an i4i.4xlarge instance. ./perf_s3_client --smp 1 --upload --object_name ./1G-test-file --sockets 1 --part_size_mb 5 INFO 2025-02-06 10:34:08,007 [shard 0:main] perf - Uploaded 1024MB in 27.768863962s, speed 36.87583335786734MB/s ./perf_s3_client --smp 1 --upload --object_name ./1G-test-file --sockets 1 --part_size_mb 10 INFO 2025-02-06 10:35:07,161 [shard 0:main] perf - Uploaded 1024MB in 28.175412552s, speed 36.34374467845414MB/s ./perf_s3_client --smp 1 --upload --object_name ./1G-test-file --sockets 1 --part_size_mb 20 INFO 2025-02-06 10:35:55,530 [shard 0:main] perf - Uploaded 1024MB in 14.483539631s, speed 70.700949221575MB/s ./perf_s3_client --smp 1 --upload --object_name ./1G-test-file --sockets 1 --part_size_mb 30 INFO 2025-02-06 10:36:35,466 [shard 0:main] perf - Uploaded 1024MB in 11.486155799s, speed 89.15080188004683MB/s ./perf_s3_client --smp 1 --upload --object_name ./1G-test-file --sockets 1 --part_size_mb 40 INFO 2025-02-06 10:37:46,642 [shard 0:main] perf - Uploaded 1024MB in 10.236196424s, speed 100.03715809898961MB/s /perf_s3_client --smp 1 --upload --object_name ./1G-test-file --sockets 1 --part_size_mb 50 INFO 2025-02-06 10:38:34,777 [shard 0:main] perf - Uploaded 1024MB in 9.490644522s, speed 107.895728011548MB/s ./perf_s3_client --smp 1 --upload --object_name ./1G-test-file --sockets 1 --part_size_mb 60 INFO 2025-02-06 10:39:08,832 [shard 0:main] perf - Uploaded 1024MB in 9.767783693s, speed 104.83442633295012MB/s ./perf_s3_client --smp 1 --upload --object_name ./1G-test-file --sockets 1 --part_size_mb 70 INFO 2025-02-06 10:39:47,916 [shard 0:main] perf - Uploaded 1024MB in 10.166116742s, speed 100.72675988162482MB/s Closes scylladb/scylladb#22732 --- utils/s3/client.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/s3/client.cc b/utils/s3/client.cc index 9a7221b39b..7489fb0797 100644 --- a/utils/s3/client.cc +++ b/utils/s3/client.cc @@ -1133,7 +1133,11 @@ class client::do_upload_file : private multipart_upload { // a giant chunk of buffer if a certain part fails to upload, we prefer // small parts, let's make it a multiple of MiB. part_size = div_ceil(total_size / aws_maximum_parts_in_piece, 1_MiB); - part_size = std::max(part_size, aws_minimum_part_size); + // The default part size for multipart upload is set to 50MiB. + // This value was determined empirically by running `perf_s3_client` with various part sizes to find the optimal one. + static constexpr size_t default_part_size = 50_MiB; + + part_size = std::max(part_size, default_part_size); return {div_ceil(total_size, part_size), part_size}; }