From 06eac50dd159ce0662c129a7013dbdfde509aa31 Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 29 Jun 2026 09:53:13 +0800 Subject: [PATCH 1/7] perf: prefetch next ListObjects page while consumer iterates current page perf: prefetch next ListObjects page while consumer iterates current page --- include/miniocpp/client.h | 12 +++++- include/miniocpp/response.h | 4 ++ src/client.cc | 75 ++++++++++++++++++++++++------------- 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/include/miniocpp/client.h b/include/miniocpp/client.h index dd4e3e3a..ae4f50b5 100644 --- a/include/miniocpp/client.h +++ b/include/miniocpp/client.h @@ -40,17 +40,27 @@ class ListObjectsResult { bool failed_ = false; ListObjectsResponse resp_; std::list::iterator itr_; + std::shared_ptr> prefetch_future_; void Populate(); + void StartPrefetch(); + void UpdatePaginationArgs(); public: explicit ListObjectsResult(error::Error err); ListObjectsResult(Client* const client, const ListObjectsArgs& args); ListObjectsResult(Client* const client, ListObjectsArgs&& args); ~ListObjectsResult() = default; + ListObjectsResult(const ListObjectsResult&) = default; + ListObjectsResult& operator=(const ListObjectsResult&) = default; + ListObjectsResult(ListObjectsResult&&) = default; + ListObjectsResult& operator=(ListObjectsResult&&) = default; Item& operator*() const { return *itr_; } - explicit operator bool() const { return itr_ != resp_.contents.end(); } + explicit operator bool() { + if (prefetch_future_ && resp_.contents.empty()) Populate(); + return itr_ != resp_.contents.end(); + } ListObjectsResult& operator++() { itr_++; diff --git a/include/miniocpp/response.h b/include/miniocpp/response.h index bf891660..ec9fcaab 100644 --- a/include/miniocpp/response.h +++ b/include/miniocpp/response.h @@ -298,6 +298,10 @@ struct ListObjectsResponse : public Response { std::string_view next_version_id_marker; ListObjectsResponse() = default; + ListObjectsResponse(const ListObjectsResponse&) = default; + ListObjectsResponse& operator=(const ListObjectsResponse&) = default; + ListObjectsResponse(ListObjectsResponse&&) = default; + ListObjectsResponse& operator=(ListObjectsResponse&&) = default; explicit ListObjectsResponse(error::Error err) : Response(std::move(err)) {} diff --git a/src/client.cc b/src/client.cc index 82388b0c..1311861a 100644 --- a/src/client.cc +++ b/src/client.cc @@ -141,16 +141,16 @@ ListObjectsResult::ListObjectsResult(error::Error err) : failed_(true) { ListObjectsResult::ListObjectsResult(Client* const client, const ListObjectsArgs& args) : client_(client), args_(args) { - Populate(); + StartPrefetch(); } ListObjectsResult::ListObjectsResult(Client* const client, ListObjectsArgs&& args) : client_(client), args_(std::move(args)) { - Populate(); + StartPrefetch(); } -void ListObjectsResult::Populate() { +void ListObjectsResult::UpdatePaginationArgs() { if (args_.include_versions) { args_.key_marker = resp_.next_key_marker; args_.version_id_marker = resp_.next_version_id_marker; @@ -160,34 +160,57 @@ void ListObjectsResult::Populate() { args_.start_after = resp_.start_after; args_.continuation_token = resp_.next_continuation_token; } +} - std::string region; - if (GetRegionResponse resp = client_->GetRegion(args_.bucket, args_.region)) { - region = resp.region; - if (args_.recursive) { - args_.delimiter = ""; - } else if (args_.delimiter.empty()) { - args_.delimiter = "/"; - } - - if (args_.include_versions || !args_.version_id_marker.empty()) { - resp_ = client_->ListObjectVersions(ListObjectVersionsArgs(args_)); - } else if (args_.use_api_v1) { - resp_ = client_->ListObjectsV1(ListObjectsV1Args(args_)); - } else { - resp_ = client_->ListObjectsV2(ListObjectsV2Args(args_)); - } +void ListObjectsResult::StartPrefetch() { + ListObjectsArgs next_args = args_; + prefetch_future_ = + std::make_shared>(std::async( + std::launch::async, + [this, next_args = std::move(next_args)]() mutable { + std::string region; + if (GetRegionResponse resp = + client_->GetRegion(next_args.bucket, next_args.region)) { + region = resp.region; + if (next_args.recursive) { + next_args.delimiter = ""; + } else if (next_args.delimiter.empty()) { + next_args.delimiter = "/"; + } + + if (next_args.include_versions || + !next_args.version_id_marker.empty()) { + return client_->ListObjectVersions( + ListObjectVersionsArgs(next_args)); + } else if (next_args.use_api_v1) { + return client_->ListObjectsV1(ListObjectsV1Args(next_args)); + } else { + return client_->ListObjectsV2(ListObjectsV2Args(next_args)); + } + } + return ListObjectsResponse( + error::make("unable to get region")); + })); +} - if (!resp_) { - failed_ = true; - resp_.contents.push_back(Item(resp_)); - } - } else { +void ListObjectsResult::Populate() { + // Always consume from the prefetch future. + if (!prefetch_future_ || !prefetch_future_->valid()) { + return; + } + resp_ = prefetch_future_->get(); + prefetch_future_.reset(); + if (!resp_) { failed_ = true; - resp_.contents.push_back(Item(resp)); + resp_.contents.push_back(Item(resp_)); } - itr_ = resp_.contents.begin(); + + // Start next prefetch if there are more pages. + if (resp_ && resp_.is_truncated) { + UpdatePaginationArgs(); + StartPrefetch(); + } } RemoveObjectsResult::RemoveObjectsResult(error::Error err) { From 53ca78454a83252f7e050090db8e808ffa7c6b48 Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 29 Jun 2026 10:13:29 +0800 Subject: [PATCH 2/7] apply suggestion apply suggestion --- include/miniocpp/client.h | 10 +++------- include/miniocpp/response.h | 4 ++-- src/client.cc | 14 ++++++-------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/include/miniocpp/client.h b/include/miniocpp/client.h index ae4f50b5..01a9c0e7 100644 --- a/include/miniocpp/client.h +++ b/include/miniocpp/client.h @@ -51,8 +51,8 @@ class ListObjectsResult { ListObjectsResult(Client* const client, const ListObjectsArgs& args); ListObjectsResult(Client* const client, ListObjectsArgs&& args); ~ListObjectsResult() = default; - ListObjectsResult(const ListObjectsResult&) = default; - ListObjectsResult& operator=(const ListObjectsResult&) = default; + ListObjectsResult(const ListObjectsResult&) = delete; + ListObjectsResult& operator=(const ListObjectsResult&) = delete; ListObjectsResult(ListObjectsResult&&) = default; ListObjectsResult& operator=(ListObjectsResult&&) = default; @@ -70,11 +70,7 @@ class ListObjectsResult { return *this; } - ListObjectsResult operator++(int) { - ListObjectsResult curr = *this; - ++(*this); - return curr; - } + void operator++(int) { ++(*this); } }; // class ListObjectsResult class RemoveObjectsResult { diff --git a/include/miniocpp/response.h b/include/miniocpp/response.h index ec9fcaab..44ade874 100644 --- a/include/miniocpp/response.h +++ b/include/miniocpp/response.h @@ -298,8 +298,8 @@ struct ListObjectsResponse : public Response { std::string_view next_version_id_marker; ListObjectsResponse() = default; - ListObjectsResponse(const ListObjectsResponse&) = default; - ListObjectsResponse& operator=(const ListObjectsResponse&) = default; + ListObjectsResponse(const ListObjectsResponse&) = delete; + ListObjectsResponse& operator=(const ListObjectsResponse&) = delete; ListObjectsResponse(ListObjectsResponse&&) = default; ListObjectsResponse& operator=(ListObjectsResponse&&) = default; diff --git a/src/client.cc b/src/client.cc index 1311861a..cf926e1e 100644 --- a/src/client.cc +++ b/src/client.cc @@ -167,11 +167,11 @@ void ListObjectsResult::StartPrefetch() { prefetch_future_ = std::make_shared>(std::async( std::launch::async, - [this, next_args = std::move(next_args)]() mutable { + [client = client_, next_args = std::move(next_args)]() mutable { std::string region; if (GetRegionResponse resp = - client_->GetRegion(next_args.bucket, next_args.region)) { - region = resp.region; + client->GetRegion(next_args.bucket, next_args.region)) { + next_args.region = resp.region; if (next_args.recursive) { next_args.delimiter = ""; } else if (next_args.delimiter.empty()) { @@ -180,12 +180,12 @@ void ListObjectsResult::StartPrefetch() { if (next_args.include_versions || !next_args.version_id_marker.empty()) { - return client_->ListObjectVersions( + return client->ListObjectVersions( ListObjectVersionsArgs(next_args)); } else if (next_args.use_api_v1) { - return client_->ListObjectsV1(ListObjectsV1Args(next_args)); + return client->ListObjectsV1(ListObjectsV1Args(next_args)); } else { - return client_->ListObjectsV2(ListObjectsV2Args(next_args)); + return client->ListObjectsV2(ListObjectsV2Args(next_args)); } } return ListObjectsResponse( @@ -194,7 +194,6 @@ void ListObjectsResult::StartPrefetch() { } void ListObjectsResult::Populate() { - // Always consume from the prefetch future. if (!prefetch_future_ || !prefetch_future_->valid()) { return; } @@ -206,7 +205,6 @@ void ListObjectsResult::Populate() { } itr_ = resp_.contents.begin(); - // Start next prefetch if there are more pages. if (resp_ && resp_.is_truncated) { UpdatePaginationArgs(); StartPrefetch(); From d2ac8030eaeb975591e0d292f30debc717fd0398 Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 29 Jun 2026 10:22:51 +0800 Subject: [PATCH 3/7] apply suggestoin apply suggestoin --- include/miniocpp/client.h | 1 + src/client.cc | 71 ++++++++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/include/miniocpp/client.h b/include/miniocpp/client.h index 01a9c0e7..333a5065 100644 --- a/include/miniocpp/client.h +++ b/include/miniocpp/client.h @@ -61,6 +61,7 @@ class ListObjectsResult { if (prefetch_future_ && resp_.contents.empty()) Populate(); return itr_ != resp_.contents.end(); } + explicit operator bool() const { return itr_ != resp_.contents.end(); } ListObjectsResult& operator++() { itr_++; diff --git a/src/client.cc b/src/client.cc index cf926e1e..80e13292 100644 --- a/src/client.cc +++ b/src/client.cc @@ -164,40 +164,57 @@ void ListObjectsResult::UpdatePaginationArgs() { void ListObjectsResult::StartPrefetch() { ListObjectsArgs next_args = args_; - prefetch_future_ = - std::make_shared>(std::async( - std::launch::async, - [client = client_, next_args = std::move(next_args)]() mutable { - std::string region; - if (GetRegionResponse resp = - client->GetRegion(next_args.bucket, next_args.region)) { - next_args.region = resp.region; - if (next_args.recursive) { - next_args.delimiter = ""; - } else if (next_args.delimiter.empty()) { - next_args.delimiter = "/"; - } - - if (next_args.include_versions || - !next_args.version_id_marker.empty()) { - return client->ListObjectVersions( - ListObjectVersionsArgs(next_args)); - } else if (next_args.use_api_v1) { - return client->ListObjectsV1(ListObjectsV1Args(next_args)); - } else { - return client->ListObjectsV2(ListObjectsV2Args(next_args)); + try { + prefetch_future_ = + std::make_shared>(std::async( + std::launch::async, + [client = client_, next_args = std::move(next_args)]() mutable { + try { + GetRegionResponse resp = + client->GetRegion(next_args.bucket, next_args.region); + if (resp) { + next_args.region = resp.region; + if (next_args.recursive) { + next_args.delimiter = ""; + } else if (next_args.delimiter.empty()) { + next_args.delimiter = "/"; + } + + if (next_args.include_versions || + !next_args.version_id_marker.empty()) { + return client->ListObjectVersions( + ListObjectVersionsArgs(next_args)); + } else if (next_args.use_api_v1) { + return client->ListObjectsV1(ListObjectsV1Args(next_args)); + } else { + return client->ListObjectsV2(ListObjectsV2Args(next_args)); + } + } + return ListObjectsResponse(resp); + } catch (const std::exception& e) { + return ListObjectsResponse( + error::Error(std::string("prefetch failed: ") + e.what())); } - } - return ListObjectsResponse( - error::make("unable to get region")); - })); + })); + } catch (const std::exception& e) { + std::promise p; + p.set_value(ListObjectsResponse( + error::Error(std::string("failed to launch prefetch: ") + e.what()))); + prefetch_future_ = + std::make_shared>(p.get_future()); + } } void ListObjectsResult::Populate() { if (!prefetch_future_ || !prefetch_future_->valid()) { return; } - resp_ = prefetch_future_->get(); + try { + resp_ = prefetch_future_->get(); + } catch (const std::exception& e) { + resp_ = ListObjectsResponse( + error::Error(std::string("prefetch result failed: ") + e.what())); + } prefetch_future_.reset(); if (!resp_) { failed_ = true; From b1d8affc1815296cf9d24e3bc60148425e093ce2 Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 29 Jun 2026 10:29:43 +0800 Subject: [PATCH 4/7] apply suggestion apply suggestion --- include/miniocpp/client.h | 20 ++++++++++++-------- src/client.cc | 29 ++++++++++++++++------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/include/miniocpp/client.h b/include/miniocpp/client.h index 333a5065..9263e785 100644 --- a/include/miniocpp/client.h +++ b/include/miniocpp/client.h @@ -38,7 +38,7 @@ class ListObjectsResult { Client* client_ = nullptr; ListObjectsArgs args_; bool failed_ = false; - ListObjectsResponse resp_; + std::shared_ptr resp_; std::list::iterator itr_; std::shared_ptr> prefetch_future_; @@ -51,27 +51,31 @@ class ListObjectsResult { ListObjectsResult(Client* const client, const ListObjectsArgs& args); ListObjectsResult(Client* const client, ListObjectsArgs&& args); ~ListObjectsResult() = default; - ListObjectsResult(const ListObjectsResult&) = delete; - ListObjectsResult& operator=(const ListObjectsResult&) = delete; + ListObjectsResult(const ListObjectsResult&) = default; + ListObjectsResult& operator=(const ListObjectsResult&) = default; ListObjectsResult(ListObjectsResult&&) = default; ListObjectsResult& operator=(ListObjectsResult&&) = default; Item& operator*() const { return *itr_; } explicit operator bool() { - if (prefetch_future_ && resp_.contents.empty()) Populate(); - return itr_ != resp_.contents.end(); + if (prefetch_future_ && (!resp_ || resp_->contents.empty())) Populate(); + return itr_ != resp_->contents.end(); } - explicit operator bool() const { return itr_ != resp_.contents.end(); } + explicit operator bool() const { return itr_ != resp_->contents.end(); } ListObjectsResult& operator++() { itr_++; - if (!failed_ && itr_ == resp_.contents.end() && resp_.is_truncated) { + if (!failed_ && itr_ == resp_->contents.end() && resp_->is_truncated) { Populate(); } return *this; } - void operator++(int) { ++(*this); } + ListObjectsResult operator++(int) { + ListObjectsResult old = *this; + ++(*this); + return old; + } }; // class ListObjectsResult class RemoveObjectsResult { diff --git a/src/client.cc b/src/client.cc index 80e13292..8993f2e4 100644 --- a/src/client.cc +++ b/src/client.cc @@ -134,31 +134,34 @@ struct ScopedRDMARegistration { } // namespace ListObjectsResult::ListObjectsResult(error::Error err) : failed_(true) { - this->resp_.contents.push_back(Item(std::move(err))); - this->itr_ = resp_.contents.begin(); + resp_ = std::make_shared(); + resp_->contents.push_back(Item(std::move(err))); + itr_ = resp_->contents.begin(); } ListObjectsResult::ListObjectsResult(Client* const client, const ListObjectsArgs& args) : client_(client), args_(args) { + resp_ = std::make_shared(); StartPrefetch(); } ListObjectsResult::ListObjectsResult(Client* const client, ListObjectsArgs&& args) : client_(client), args_(std::move(args)) { + resp_ = std::make_shared(); StartPrefetch(); } void ListObjectsResult::UpdatePaginationArgs() { if (args_.include_versions) { - args_.key_marker = resp_.next_key_marker; - args_.version_id_marker = resp_.next_version_id_marker; + args_.key_marker = resp_->next_key_marker; + args_.version_id_marker = resp_->next_version_id_marker; } else if (args_.use_api_v1) { - args_.marker = resp_.next_marker; + args_.marker = resp_->next_marker; } else { - args_.start_after = resp_.start_after; - args_.continuation_token = resp_.next_continuation_token; + args_.start_after = resp_->start_after; + args_.continuation_token = resp_->next_continuation_token; } } @@ -210,19 +213,19 @@ void ListObjectsResult::Populate() { return; } try { - resp_ = prefetch_future_->get(); + resp_ = std::make_shared(prefetch_future_->get()); } catch (const std::exception& e) { - resp_ = ListObjectsResponse( + resp_ = std::make_shared( error::Error(std::string("prefetch result failed: ") + e.what())); } prefetch_future_.reset(); - if (!resp_) { + if (!*resp_) { failed_ = true; - resp_.contents.push_back(Item(resp_)); + resp_->contents.push_back(Item(*resp_)); } - itr_ = resp_.contents.begin(); + itr_ = resp_->contents.begin(); - if (resp_ && resp_.is_truncated) { + if (*resp_ && resp_->is_truncated) { UpdatePaginationArgs(); StartPrefetch(); } From 226ac52f727f1d19d61ecbb0584da7ddcee8f662 Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 29 Jun 2026 11:21:15 +0800 Subject: [PATCH 5/7] rename --- include/miniocpp/client.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/miniocpp/client.h b/include/miniocpp/client.h index 9263e785..22fd6f10 100644 --- a/include/miniocpp/client.h +++ b/include/miniocpp/client.h @@ -72,9 +72,9 @@ class ListObjectsResult { } ListObjectsResult operator++(int) { - ListObjectsResult old = *this; + ListObjectsResult curr = *this; ++(*this); - return old; + return curr; } }; // class ListObjectsResult From 353a2aaa2ece12b8b3f80f4b745db4e214494bf5 Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 29 Jun 2026 11:36:00 +0800 Subject: [PATCH 6/7] apply suggestion --- include/miniocpp/client.h | 3 +- src/client.cc | 79 +++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 37 deletions(-) diff --git a/include/miniocpp/client.h b/include/miniocpp/client.h index 22fd6f10..e493903c 100644 --- a/include/miniocpp/client.h +++ b/include/miniocpp/client.h @@ -40,7 +40,8 @@ class ListObjectsResult { bool failed_ = false; std::shared_ptr resp_; std::list::iterator itr_; - std::shared_ptr> prefetch_future_; + std::shared_ptr>> + prefetch_future_; void Populate(); void StartPrefetch(); diff --git a/src/client.cc b/src/client.cc index 8993f2e4..cf0d4346 100644 --- a/src/client.cc +++ b/src/client.cc @@ -143,6 +143,7 @@ ListObjectsResult::ListObjectsResult(Client* const client, const ListObjectsArgs& args) : client_(client), args_(args) { resp_ = std::make_shared(); + itr_ = resp_->contents.end(); StartPrefetch(); } @@ -150,11 +151,12 @@ ListObjectsResult::ListObjectsResult(Client* const client, ListObjectsArgs&& args) : client_(client), args_(std::move(args)) { resp_ = std::make_shared(); + itr_ = resp_->contents.end(); StartPrefetch(); } void ListObjectsResult::UpdatePaginationArgs() { - if (args_.include_versions) { + if (args_.include_versions || !args_.version_id_marker.empty()) { args_.key_marker = resp_->next_key_marker; args_.version_id_marker = resp_->next_version_id_marker; } else if (args_.use_api_v1) { @@ -168,43 +170,48 @@ void ListObjectsResult::UpdatePaginationArgs() { void ListObjectsResult::StartPrefetch() { ListObjectsArgs next_args = args_; try { - prefetch_future_ = - std::make_shared>(std::async( - std::launch::async, - [client = client_, next_args = std::move(next_args)]() mutable { - try { - GetRegionResponse resp = - client->GetRegion(next_args.bucket, next_args.region); - if (resp) { - next_args.region = resp.region; - if (next_args.recursive) { - next_args.delimiter = ""; - } else if (next_args.delimiter.empty()) { - next_args.delimiter = "/"; - } - - if (next_args.include_versions || - !next_args.version_id_marker.empty()) { - return client->ListObjectVersions( - ListObjectVersionsArgs(next_args)); - } else if (next_args.use_api_v1) { - return client->ListObjectsV1(ListObjectsV1Args(next_args)); - } else { - return client->ListObjectsV2(ListObjectsV2Args(next_args)); - } - } - return ListObjectsResponse(resp); - } catch (const std::exception& e) { - return ListObjectsResponse( - error::Error(std::string("prefetch failed: ") + e.what())); + prefetch_future_ = std::make_shared< + std::shared_future>>(std::async( + std::launch::async, + [client = client_, next_args = std::move(next_args)]() mutable + -> std::shared_ptr { + try { + GetRegionResponse resp = + client->GetRegion(next_args.bucket, next_args.region); + if (resp) { + next_args.region = resp.region; + if (next_args.recursive) { + next_args.delimiter = ""; + } else if (next_args.delimiter.empty()) { + next_args.delimiter = "/"; } - })); + + if (next_args.include_versions || + !next_args.version_id_marker.empty()) { + return std::make_shared( + client->ListObjectVersions( + ListObjectVersionsArgs(next_args))); + } else if (next_args.use_api_v1) { + return std::make_shared( + client->ListObjectsV1(ListObjectsV1Args(next_args))); + } else { + return std::make_shared( + client->ListObjectsV2(ListObjectsV2Args(next_args))); + } + } + return std::make_shared(resp); + } catch (const std::exception& e) { + return std::make_shared( + error::Error(std::string("prefetch failed: ") + e.what())); + } + })); } catch (const std::exception& e) { - std::promise p; - p.set_value(ListObjectsResponse( + std::promise> p; + p.set_value(std::make_shared( error::Error(std::string("failed to launch prefetch: ") + e.what()))); - prefetch_future_ = - std::make_shared>(p.get_future()); + prefetch_future_ = std::make_shared< + std::shared_future>>( + p.get_future()); } } @@ -213,7 +220,7 @@ void ListObjectsResult::Populate() { return; } try { - resp_ = std::make_shared(prefetch_future_->get()); + resp_ = prefetch_future_->get(); } catch (const std::exception& e) { resp_ = std::make_shared( error::Error(std::string("prefetch result failed: ") + e.what())); From 843055aa57b5bd80f61e57c2fcb489f742a2594b Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 29 Jun 2026 12:03:14 +0800 Subject: [PATCH 7/7] apply suggestion apply suggestion --- src/client.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client.cc b/src/client.cc index cf0d4346..0f0e1053 100644 --- a/src/client.cc +++ b/src/client.cc @@ -145,6 +145,7 @@ ListObjectsResult::ListObjectsResult(Client* const client, resp_ = std::make_shared(); itr_ = resp_->contents.end(); StartPrefetch(); + Populate(); } ListObjectsResult::ListObjectsResult(Client* const client, @@ -153,6 +154,7 @@ ListObjectsResult::ListObjectsResult(Client* const client, resp_ = std::make_shared(); itr_ = resp_->contents.end(); StartPrefetch(); + Populate(); } void ListObjectsResult::UpdatePaginationArgs() {