From 63592b85a42f01df417868ee5a36a4e64c433a1c Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Wed, 26 Nov 2025 20:17:11 +0300 Subject: [PATCH 01/13] add await-set, when-all, when-any --- runtime-light/coroutine/async-stack-methods.h | 36 +++++++++++++++++++ runtime-light/coroutine/async-stack.h | 15 +------- runtime-light/coroutine/await-set.h | 4 +-- runtime-light/coroutine/coroutine-state.cpp | 8 +++++ runtime-light/coroutine/coroutine-state.h | 3 ++ runtime-light/coroutine/detail/await-set.h | 19 +++++----- .../coroutine/detail/task-self-deleting.h | 2 +- runtime-light/coroutine/detail/when-all.h | 24 ++++++------- runtime-light/coroutine/detail/when-any.h | 23 ++++++------ runtime-light/coroutine/io-scheduler.h | 11 +++--- runtime-light/coroutine/shared-task.h | 5 +-- runtime-light/state/instance-state.cpp | 4 +-- .../stdlib/diagnostics/backtrace.cpp | 30 ++++++++++------ 13 files changed, 115 insertions(+), 69 deletions(-) create mode 100644 runtime-light/coroutine/async-stack-methods.h diff --git a/runtime-light/coroutine/async-stack-methods.h b/runtime-light/coroutine/async-stack-methods.h new file mode 100644 index 0000000000..a2e044f313 --- /dev/null +++ b/runtime-light/coroutine/async-stack-methods.h @@ -0,0 +1,36 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2025 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-light/coroutine/async-stack.h" + +#include +#include + +#include "runtime-light/coroutine/coroutine-state.h" + +namespace kphp::coro { + +using ::CoroutineInstanceState; + +/** + * The `resume` function is responsible for storing the current synchronous stack frame + * in async_stack_root::stop_sync_frame before resuming the coroutine and also creates a new async_stack_root. This allows + * capturing one of the stack frames in the synchronous stack trace and jump between parts of the async_stack. + */ +[[clang::noinline]] inline void resume(std::coroutine_handle<> handle, async_stack_root* stack_root = nullptr) noexcept { + if (stack_root == nullptr) { + stack_root = CoroutineInstanceState::get_next_root(); + } + async_stack_root root{}; + root.next_async_stack_root = stack_root; + auto* previous_stack_frame{std::exchange(stack_root->stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; + + CoroutineInstanceState::set_next_root(std::addressof(root)); + handle.resume(); + stack_root->stop_sync_stack_frame = previous_stack_frame; + CoroutineInstanceState::set_next_root(stack_root); +} +} // namespace kphp::coro diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index 85ab351a5f..3ef6938753 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -4,9 +4,6 @@ #pragma once -#include -#include - /** * This header defines the data structures used to represent a coroutine asynchronous stack. * @@ -81,19 +78,9 @@ struct async_stack_frame { struct async_stack_root { async_stack_frame* top_async_stack_frame{}; stack_frame* stop_sync_stack_frame{}; + async_stack_root* next_async_stack_root{}; }; -/** - * The `resume` function is responsible for storing the current synchronous stack frame - * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows - * capturing one of the stack frames in the synchronous stack trace. - */ -inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root) noexcept { - auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; - handle.resume(); - stack_root.stop_sync_stack_frame = previous_stack_frame; -} - /** * The async_stack_element class facilitates working with asynchronous traces in templated code. * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. diff --git a/runtime-light/coroutine/await-set.h b/runtime-light/coroutine/await-set.h index 3d3bdd76f7..3948969866 100644 --- a/runtime-light/coroutine/await-set.h +++ b/runtime-light/coroutine/await-set.h @@ -19,12 +19,12 @@ namespace kphp::coro { template class await_set { std::unique_ptr> m_await_broker; - kphp::coro::async_stack_root& m_coroutine_stack_root; + kphp::coro::async_stack_root* m_coroutine_stack_root; public: await_set() noexcept : m_await_broker(std::make_unique>()), - m_coroutine_stack_root(CoroutineInstanceState::get().coroutine_stack_root) {} + m_coroutine_stack_root(CoroutineInstanceState::get_next_root()) {} await_set(await_set&& other) noexcept : m_await_broker(std::move(other.m_await_broker)), diff --git a/runtime-light/coroutine/coroutine-state.cpp b/runtime-light/coroutine/coroutine-state.cpp index b7dc08d01d..aaba41498a 100644 --- a/runtime-light/coroutine/coroutine-state.cpp +++ b/runtime-light/coroutine/coroutine-state.cpp @@ -9,3 +9,11 @@ CoroutineInstanceState& CoroutineInstanceState::get() noexcept { return InstanceState::get().coroutine_instance_state; } + +kphp::coro::async_stack_root* CoroutineInstanceState::get_next_root() noexcept { + return CoroutineInstanceState::get().next_root; +} + +void CoroutineInstanceState::set_next_root(kphp::coro::async_stack_root* new_next_root) noexcept { + CoroutineInstanceState::get().next_root = new_next_root; +} \ No newline at end of file diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index b6d260c278..94ff5b196e 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -13,6 +13,9 @@ struct CoroutineInstanceState final : private vk::not_copyable { CoroutineInstanceState() noexcept = default; static CoroutineInstanceState& get() noexcept; + static kphp::coro::async_stack_root* get_next_root() noexcept; + static void set_next_root(kphp::coro::async_stack_root* new_next_root) noexcept; kphp::coro::async_stack_root coroutine_stack_root; + kphp::coro::async_stack_root* next_root{&coroutine_stack_root}; }; diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index 2edbd086e4..66e3771662 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -12,6 +12,7 @@ #include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" @@ -65,7 +66,7 @@ class await_broker { kphp::memory::script::free(ptr); } - void start_task(await_set_task&& task, kphp::coro::async_stack_root& coroutine_stack_root, void* return_address) noexcept { + void start_task(await_set_task&& task, kphp::coro::async_stack_root* coroutine_stack_root, void* return_address) noexcept { auto task_iterator{m_tasks_storage.insert(m_tasks_storage.begin(), std::move(task))}; task_iterator->start(*this, task_iterator, coroutine_stack_root, return_address); } @@ -80,7 +81,7 @@ class await_broker { if (m_awaiters != nullptr) { m_awaiters->m_prev = nullptr; } - awaiter->m_continuation.resume(); + kphp::coro::resume(awaiter->m_continuation); } } @@ -139,8 +140,7 @@ class await_broker { if (awaiters_to_resume != nullptr) { awaiters_to_resume->m_prev = nullptr; } - - waiter->m_continuation.resume(); + kphp::coro::resume(waiter->m_continuation); } } @@ -151,8 +151,6 @@ class await_broker { ~await_broker() { abort_all(); } - -private: }; template @@ -201,7 +199,7 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { auto start(detail::await_set::await_broker& await_broker, kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, - kphp::coro::async_stack_root& async_stack_root, void* return_address) noexcept { + kphp::coro::async_stack_root* async_stack_root, void* return_address) noexcept { m_await_broker = await_broker; m_ready_task_element.m_storage_location = storage_location; @@ -212,11 +210,12 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { * */ auto& async_stack_frame{get_async_stack_frame()}; async_stack_frame.caller_async_stack_frame = nullptr; - async_stack_frame.async_stack_root = std::addressof(async_stack_root); + async_stack_frame.async_stack_root = async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); + kphp::coro::resume(handle, async_stack_root); } }; @@ -273,7 +272,7 @@ class await_set_task { auto start(detail::await_set::await_broker& await_broker, kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, - kphp::coro::async_stack_root& async_stack_root, void* return_address) noexcept { + kphp::coro::async_stack_root* async_stack_root, void* return_address) noexcept { m_coroutine.promise().start(await_broker, storage_location, async_stack_root, return_address); } diff --git a/runtime-light/coroutine/detail/task-self-deleting.h b/runtime-light/coroutine/detail/task-self-deleting.h index daa72cdd17..6f252da115 100644 --- a/runtime-light/coroutine/detail/task-self-deleting.h +++ b/runtime-light/coroutine/detail/task-self-deleting.h @@ -69,7 +69,7 @@ class task_self_deleting { // initialize task_self_deleting's frame as top async stack frame auto& async_stack_frame{m_promise.get_async_stack_frame()}; async_stack_frame.return_address = STACK_RETURN_ADDRESS; - async_stack_frame.async_stack_root = std::addressof(CoroutineInstanceState::get().coroutine_stack_root); + async_stack_frame.async_stack_root = CoroutineInstanceState::get_next_root(); } ~task_self_deleting() noexcept = default; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index cc276f7996..b174f6380f 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -14,6 +14,7 @@ #include #include +#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" @@ -58,7 +59,7 @@ class when_all_latch { auto notify_awaitable_completed() noexcept -> void { if (--m_count == 1 && m_awaiting_coroutine != nullptr) { - m_awaiting_coroutine.resume(); + kphp::coro::resume(m_awaiting_coroutine); } } }; @@ -107,9 +108,7 @@ class when_all_ready_awaitable> { void* const return_address{STACK_RETURN_ADDRESS}; m_caller_async_stack_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); - std::apply([&latch = m_awaitable.m_latch, &caller_async_stack_frame = *m_caller_async_stack_frame, - return_address](auto&... tasks) noexcept { (tasks.start(latch, caller_async_stack_frame, return_address), ...); }, - m_awaitable.m_tasks); + std::apply([&latch = m_awaitable.m_latch, return_address](auto&... tasks) noexcept { (tasks.start(latch, return_address), ...); }, m_awaitable.m_tasks); return m_awaitable.m_latch.try_await(awaiting_coroutine); } @@ -182,17 +181,18 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { kphp::log::error("internal unhandled exception"); } - auto start(when_all_latch& latch, kphp::coro::async_stack_frame& caller_async_stack_frame, void* return_address) noexcept { + auto start(when_all_latch& latch, void* return_address) noexcept { m_latch = std::addressof(latch); auto& async_stack_frame{get_async_stack_frame()}; - kphp::log::assertion(caller_async_stack_frame.async_stack_root != nullptr); // initialize when_all_task's async stack frame and make it the top frame - async_stack_frame.caller_async_stack_frame = std::addressof(caller_async_stack_frame); - async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; + async_stack_frame.caller_async_stack_frame = nullptr; + async_stack_frame.async_stack_root = CoroutineInstanceState::get_next_root(); async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + + decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); + kphp::coro::resume(handle, async_stack_frame.async_stack_root); } }; @@ -230,7 +230,7 @@ class when_all_task { } auto result() noexcept -> T { - kphp::log::assertion(m_result); + kphp::log::assertion(static_cast(m_result)); return std::move(*m_result); } @@ -247,8 +247,8 @@ class when_all_task { constexpr auto return_void() const noexcept -> void {} }; - auto start(when_all_latch& latch, kphp::coro::async_stack_frame& caller_async_stack_frame, void* return_address) noexcept -> void { - m_coroutine.promise().start(latch, caller_async_stack_frame, return_address); + auto start(when_all_latch& latch, void* return_address) noexcept -> void { + m_coroutine.promise().start(latch, return_address); } public: diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index f124f5928e..6c1d68cc65 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -12,6 +12,8 @@ #include #include +#include "runtime-light/coroutine/async-stack-methods.h" +#include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" @@ -56,7 +58,7 @@ class when_any_latch { auto notify_awaitable_completed() noexcept -> void { m_toggled = true; if (m_awaiting_coroutine != nullptr) { - m_awaiting_coroutine.resume(); + kphp::coro::resume(m_awaiting_coroutine); } } }; @@ -106,9 +108,7 @@ class when_any_ready_awaitable> { void* const return_address{STACK_RETURN_ADDRESS}; m_caller_async_stack_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); - std::apply([&latch = m_awaitable.m_latch, &caller_async_stack_frame = *m_caller_async_stack_frame, - return_address](auto&... tasks) noexcept { (tasks.start(latch, caller_async_stack_frame, return_address), ...); }, - m_awaitable.m_tasks); + std::apply([&latch = m_awaitable.m_latch, return_address](auto&... tasks) noexcept { (tasks.start(latch, return_address), ...); }, m_awaitable.m_tasks); return m_awaitable.m_latch.try_await(awaiting_coroutine); } @@ -191,17 +191,18 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { kphp::log::error("internal unhandled exception"); } - auto start(when_any_latch& latch, kphp::coro::async_stack_frame& caller_async_stack_frame, void* return_address) noexcept { + auto start(when_any_latch& latch, void* return_address) noexcept { m_latch = std::addressof(latch); auto& async_stack_frame{get_async_stack_frame()}; - kphp::log::assertion(caller_async_stack_frame.async_stack_root != nullptr); // initialize when_all_task's async stack frame and make it the top frame - async_stack_frame.caller_async_stack_frame = std::addressof(caller_async_stack_frame); - async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; + async_stack_frame.caller_async_stack_frame = nullptr; + async_stack_frame.async_stack_root = CoroutineInstanceState::get_next_root(); async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + + decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); + kphp::coro::resume(handle, async_stack_frame.async_stack_root); } }; @@ -260,9 +261,9 @@ class when_any_task { constexpr auto return_void() const noexcept -> void {} }; - auto start(when_any_latch& latch, kphp::coro::async_stack_frame& caller_async_stack_frame, void* return_address) noexcept -> void { + auto start(when_any_latch& latch, void* return_address) noexcept -> void { if (!latch.is_ready()) [[likely]] { - m_coroutine.promise().start(latch, caller_async_stack_frame, return_address); + m_coroutine.promise().start(latch, return_address); } } diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 657dae1539..8b395880aa 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -23,6 +23,7 @@ #include "common/containers/final_action.h" #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/coroutine-state.h" @@ -395,8 +396,8 @@ inline auto io_scheduler::process_events() noexcept -> k2::PollStatus { kphp::log::assertion(m_scheduled_tasks_tmp.empty()); std::swap(m_scheduled_tasks, m_scheduled_tasks_tmp); - std::ranges::for_each(m_scheduled_tasks_tmp, [&coroutine_stack_root = m_coroutine_instance_state.coroutine_stack_root](const auto& coroutine) noexcept { - kphp::coro::resume(coroutine, coroutine_stack_root); + std::ranges::for_each(m_scheduled_tasks_tmp, [&coroutine_next_stack_root = m_coroutine_instance_state.next_root](const auto& coroutine) noexcept { + kphp::coro::resume(coroutine, coroutine_next_stack_root); }); m_scheduled_tasks_tmp.clear(); } @@ -422,7 +423,7 @@ auto io_scheduler::start(coroutine_type coroutine) noexcept -> bool { if (!handle || handle.done()) [[unlikely]] { return false; } - kphp::coro::resume(handle, m_coroutine_instance_state.coroutine_stack_root); + kphp::coro::resume(handle, m_coroutine_instance_state.next_root); return true; } @@ -434,7 +435,7 @@ inline auto io_scheduler::schedule() noexcept { explicit schedule_operation(io_scheduler& scheduler) noexcept : m_scheduler(scheduler), - m_async_stack_frame(m_scheduler.m_coroutine_instance_state.coroutine_stack_root.top_async_stack_frame) {} + m_async_stack_frame(m_scheduler.m_coroutine_instance_state.next_root->next_async_stack_root->top_async_stack_frame) {} public: constexpr auto await_ready() const noexcept -> bool { @@ -446,7 +447,7 @@ inline auto io_scheduler::schedule() noexcept { } auto await_resume() const noexcept -> void { - m_scheduler.m_coroutine_instance_state.coroutine_stack_root.top_async_stack_frame = m_async_stack_frame; + m_scheduler.m_coroutine_instance_state.next_root->next_async_stack_root->top_async_stack_frame = m_async_stack_frame; } }; return schedule_operation{*this}; diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index 76957148d9..4323335f7f 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -14,6 +14,7 @@ #include #include "runtime-common/core/allocator/script-malloc-interface.h" +#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -51,7 +52,7 @@ struct promise_base : kphp::coro::async_stack_element { // read the m_next pointer before resuming the coroutine // since resuming the coroutine may destroy the shared_task_waiter value auto* next{awaiter->m_next}; - auto& async_stack_root{*promise.get_async_stack_frame().async_stack_root}; + auto* async_stack_root{promise.get_async_stack_frame().async_stack_root}; kphp::coro::resume(awaiter->m_continuation, async_stack_root); awaiter = next; } @@ -99,7 +100,7 @@ struct promise_base : kphp::coro::async_stack_element { if (m_awaiters == NOT_STARTED_VAL) { m_awaiters = STARTED_NO_WAITERS_VAL; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; - auto& async_stack_root{*get_async_stack_frame().async_stack_root}; + auto* async_stack_root{get_async_stack_frame().async_stack_root}; kphp::coro::resume(handle, async_stack_root); } // coroutine already completed, don't suspend diff --git a/runtime-light/state/instance-state.cpp b/runtime-light/state/instance-state.cpp index 64334b904d..9c3cd72e89 100644 --- a/runtime-light/state/instance-state.cpp +++ b/runtime-light/state/instance-state.cpp @@ -76,8 +76,8 @@ void InstanceState::init_script_execution() noexcept { std::move(script_task))}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; - main_task_async_stack_frame.async_stack_root = std::addressof(coroutine_instance_state.coroutine_stack_root); - coroutine_instance_state.coroutine_stack_root.top_async_stack_frame = std::addressof(main_task_async_stack_frame); + main_task_async_stack_frame.async_stack_root = coroutine_instance_state.next_root; + coroutine_instance_state.next_root->top_async_stack_frame = std::addressof(main_task_async_stack_frame); // spawn main task onto the scheduler kphp::log::assertion(io_scheduler.spawn(std::move(main_task))); } diff --git a/runtime-light/stdlib/diagnostics/backtrace.cpp b/runtime-light/stdlib/diagnostics/backtrace.cpp index 6ec787e497..a21b249dec 100644 --- a/runtime-light/stdlib/diagnostics/backtrace.cpp +++ b/runtime-light/stdlib/diagnostics/backtrace.cpp @@ -15,11 +15,11 @@ namespace { -size_t sync_frames(std::span addresses, kphp::coro::stack_frame* start_frame, kphp::coro::stack_frame* stop_frame) noexcept { +size_t sync_frames(std::span addresses, const kphp::coro::stack_frame* start_frame, const kphp::coro::stack_frame* stop_frame) noexcept { kphp::log::assertion(start_frame != nullptr && stop_frame != nullptr); auto address{addresses.begin()}; - for (auto* current_stack_frame{start_frame}; current_stack_frame != stop_frame && address != addresses.end(); + for (const auto* current_stack_frame{start_frame}; current_stack_frame != stop_frame && address != addresses.end(); current_stack_frame = current_stack_frame->caller_stack_frame) { *address = current_stack_frame->return_address; address = std::next(address); @@ -43,16 +43,26 @@ size_t async_frames(std::span addresses, kphp::coro::async_stack_frame* t namespace kphp::diagnostic::impl { -size_t async_backtrace(std::span addresses) noexcept { - if (const auto* instance_state{k2::instance_state()}; instance_state != nullptr) [[likely]] { - const auto& async_stack_root{instance_state->coroutine_instance_state.coroutine_stack_root}; +size_t async_backtrace_helper(std::span addresses, const kphp::coro::stack_frame* start_sync_frame, + const kphp::coro::async_stack_root* async_stack_root) noexcept { + if (async_stack_root == nullptr) { + return 0; + } + const auto* const stop_sync_frame{async_stack_root->stop_sync_stack_frame}; - auto* const start_sync_frame{reinterpret_cast(STACK_FRAME_ADDRESS)}; - auto* const stop_sync_frame{async_stack_root.stop_sync_stack_frame}; + const size_t num_sync_frames{sync_frames(addresses, start_sync_frame, stop_sync_frame)}; + const size_t num_async_frames{async_frames(addresses.subspan(num_sync_frames), async_stack_root->top_async_stack_frame)}; - const size_t num_sync_frames{sync_frames(addresses, start_sync_frame, stop_sync_frame)}; - const size_t num_async_frames{async_frames(addresses.subspan(num_sync_frames), async_stack_root.top_async_stack_frame)}; - return num_sync_frames + num_async_frames; + const size_t result{num_sync_frames + num_async_frames}; + if (addresses.size() < result) + return result; + return result + async_backtrace_helper(addresses.subspan(result), stop_sync_frame, async_stack_root->next_async_stack_root); +} + +size_t async_backtrace(std::span addresses) noexcept { + if (const auto* instance_state{k2::instance_state()}; instance_state != nullptr) [[likely]] { + return async_backtrace_helper(addresses, reinterpret_cast(STACK_FRAME_ADDRESS), + instance_state->coroutine_instance_state.next_root->next_async_stack_root); } return 0; } From 13ff2261e83f133fab5699bb53541c089e7c863d Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Wed, 26 Nov 2025 20:24:15 +0300 Subject: [PATCH 02/13] add new line --- runtime-light/coroutine/coroutine-state.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-light/coroutine/coroutine-state.cpp b/runtime-light/coroutine/coroutine-state.cpp index aaba41498a..9af07fea13 100644 --- a/runtime-light/coroutine/coroutine-state.cpp +++ b/runtime-light/coroutine/coroutine-state.cpp @@ -16,4 +16,4 @@ kphp::coro::async_stack_root* CoroutineInstanceState::get_next_root() noexcept { void CoroutineInstanceState::set_next_root(kphp::coro::async_stack_root* new_next_root) noexcept { CoroutineInstanceState::get().next_root = new_next_root; -} \ No newline at end of file +} From 9e9e170ae12f9afaa56d7624485f2a90444876a3 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Thu, 27 Nov 2025 18:57:32 +0300 Subject: [PATCH 03/13] fixes --- runtime-light/coroutine/async-stack-methods.h | 23 ++++++++----------- runtime-light/coroutine/await-set.h | 10 +++----- runtime-light/coroutine/coroutine-state.cpp | 8 ------- runtime-light/coroutine/coroutine-state.h | 6 ++--- runtime-light/coroutine/detail/await-set.h | 21 ++++++++--------- .../coroutine/detail/task-self-deleting.h | 2 +- runtime-light/coroutine/detail/when-all.h | 7 +++--- runtime-light/coroutine/detail/when-any.h | 8 ++++--- runtime-light/coroutine/event.h | 2 +- runtime-light/coroutine/io-scheduler.h | 10 ++++---- runtime-light/coroutine/shared-task.h | 4 ++-- runtime-light/state/instance-state.cpp | 4 ++-- .../stdlib/diagnostics/backtrace.cpp | 7 ++---- 13 files changed, 46 insertions(+), 66 deletions(-) diff --git a/runtime-light/coroutine/async-stack-methods.h b/runtime-light/coroutine/async-stack-methods.h index a2e044f313..9d1a6f7deb 100644 --- a/runtime-light/coroutine/async-stack-methods.h +++ b/runtime-light/coroutine/async-stack-methods.h @@ -13,24 +13,19 @@ namespace kphp::coro { -using ::CoroutineInstanceState; - /** - * The `resume` function is responsible for storing the current synchronous stack frame - * in async_stack_root::stop_sync_frame before resuming the coroutine and also creates a new async_stack_root. This allows + * The `resume_with_new_root` function is responsible for storing the current synchronous stack frame + * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows * capturing one of the stack frames in the synchronous stack trace and jump between parts of the async_stack. */ -[[clang::noinline]] inline void resume(std::coroutine_handle<> handle, async_stack_root* stack_root = nullptr) noexcept { - if (stack_root == nullptr) { - stack_root = CoroutineInstanceState::get_next_root(); - } - async_stack_root root{}; - root.next_async_stack_root = stack_root; - auto* previous_stack_frame{std::exchange(stack_root->stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; +[[clang::noinline]] inline void resume_with_new_root(std::coroutine_handle<> handle, async_stack_root* new_async_stack_root) noexcept { + auto& coroutine_st{CoroutineInstanceState::get()}; + auto* previous_stack_root = coroutine_st.current_async_stack_root; + new_async_stack_root->next_async_stack_root = coroutine_st.current_async_stack_root; - CoroutineInstanceState::set_next_root(std::addressof(root)); + new_async_stack_root->stop_sync_stack_frame = reinterpret_cast(STACK_FRAME_ADDRESS); + coroutine_st.current_async_stack_root = new_async_stack_root; handle.resume(); - stack_root->stop_sync_stack_frame = previous_stack_frame; - CoroutineInstanceState::set_next_root(stack_root); + coroutine_st.current_async_stack_root = previous_stack_root; } } // namespace kphp::coro diff --git a/runtime-light/coroutine/await-set.h b/runtime-light/coroutine/await-set.h index 3948969866..8f6a1bfd37 100644 --- a/runtime-light/coroutine/await-set.h +++ b/runtime-light/coroutine/await-set.h @@ -19,21 +19,17 @@ namespace kphp::coro { template class await_set { std::unique_ptr> m_await_broker; - kphp::coro::async_stack_root* m_coroutine_stack_root; public: await_set() noexcept - : m_await_broker(std::make_unique>()), - m_coroutine_stack_root(CoroutineInstanceState::get_next_root()) {} + : m_await_broker(std::make_unique>()) {} await_set(await_set&& other) noexcept - : m_await_broker(std::move(other.m_await_broker)), - m_coroutine_stack_root(other.m_coroutine_stack_root) {} + : m_await_broker(std::move(other.m_await_broker)) {} await_set& operator=(await_set&& other) noexcept { if (this != std::addressof(other)) { m_await_broker = std::move(other.m_await_broker); - m_coroutine_stack_root = other.m_coroutine_stack_root; } return *this; } @@ -44,7 +40,7 @@ class await_set { template requires kphp::coro::concepts::awaitable && std::is_same_v::awaiter_return_type, return_type> void push(awaitable_type awaitable) noexcept { - m_await_broker->start_task(detail::await_set::make_await_set_task(std::move(awaitable)), m_coroutine_stack_root, STACK_RETURN_ADDRESS); + m_await_broker->start_task(detail::await_set::make_await_set_task(std::move(awaitable)), STACK_RETURN_ADDRESS); } auto next() noexcept { diff --git a/runtime-light/coroutine/coroutine-state.cpp b/runtime-light/coroutine/coroutine-state.cpp index 9af07fea13..b7dc08d01d 100644 --- a/runtime-light/coroutine/coroutine-state.cpp +++ b/runtime-light/coroutine/coroutine-state.cpp @@ -9,11 +9,3 @@ CoroutineInstanceState& CoroutineInstanceState::get() noexcept { return InstanceState::get().coroutine_instance_state; } - -kphp::coro::async_stack_root* CoroutineInstanceState::get_next_root() noexcept { - return CoroutineInstanceState::get().next_root; -} - -void CoroutineInstanceState::set_next_root(kphp::coro::async_stack_root* new_next_root) noexcept { - CoroutineInstanceState::get().next_root = new_next_root; -} diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index 94ff5b196e..42ced3c635 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -13,9 +13,7 @@ struct CoroutineInstanceState final : private vk::not_copyable { CoroutineInstanceState() noexcept = default; static CoroutineInstanceState& get() noexcept; - static kphp::coro::async_stack_root* get_next_root() noexcept; - static void set_next_root(kphp::coro::async_stack_root* new_next_root) noexcept; - kphp::coro::async_stack_root coroutine_stack_root; - kphp::coro::async_stack_root* next_root{&coroutine_stack_root}; + kphp::coro::async_stack_root base_coroutine_stack_root; // used to reduce the number of accesses to thread-local state + kphp::coro::async_stack_root* current_async_stack_root{&base_coroutine_stack_root}; }; diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index 66e3771662..d130a16569 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -66,9 +66,9 @@ class await_broker { kphp::memory::script::free(ptr); } - void start_task(await_set_task&& task, kphp::coro::async_stack_root* coroutine_stack_root, void* return_address) noexcept { + void start_task(await_set_task&& task, void* return_address) noexcept { auto task_iterator{m_tasks_storage.insert(m_tasks_storage.begin(), std::move(task))}; - task_iterator->start(*this, task_iterator, coroutine_stack_root, return_address); + task_iterator->start(*this, task_iterator, return_address); } void push_ready_task(await_set_ready_task_element& ready_task) noexcept { @@ -81,7 +81,7 @@ class await_broker { if (m_awaiters != nullptr) { m_awaiters->m_prev = nullptr; } - kphp::coro::resume(awaiter->m_continuation); + awaiter->m_continuation.resume(); } } @@ -140,7 +140,7 @@ class await_broker { if (awaiters_to_resume != nullptr) { awaiters_to_resume->m_prev = nullptr; } - kphp::coro::resume(waiter->m_continuation); + waiter->m_continuation.resume(); } } @@ -198,8 +198,7 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { } auto start(detail::await_set::await_broker& await_broker, - kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, - kphp::coro::async_stack_root* async_stack_root, void* return_address) noexcept { + kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, void* return_address) noexcept { m_await_broker = await_broker; m_ready_task_element.m_storage_location = storage_location; @@ -208,14 +207,15 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { * await_set_task is the top of the stack for calls from it. * Therefore, it doesn't store caller_frame, but it save `await_set::push()` return address * */ + kphp::coro::async_stack_root root{}; auto& async_stack_frame{get_async_stack_frame()}; async_stack_frame.caller_async_stack_frame = nullptr; - async_stack_frame.async_stack_root = async_stack_root; + async_stack_frame.async_stack_root = std::addressof(root); async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); - kphp::coro::resume(handle, async_stack_root); + kphp::coro::resume_with_new_root(handle, std::addressof(root)); } }; @@ -271,9 +271,8 @@ class await_set_task { }; auto start(detail::await_set::await_broker& await_broker, - kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, - kphp::coro::async_stack_root* async_stack_root, void* return_address) noexcept { - m_coroutine.promise().start(await_broker, storage_location, async_stack_root, return_address); + kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, void* return_address) noexcept { + m_coroutine.promise().start(await_broker, storage_location, return_address); } public: diff --git a/runtime-light/coroutine/detail/task-self-deleting.h b/runtime-light/coroutine/detail/task-self-deleting.h index 6f252da115..8cd10f4241 100644 --- a/runtime-light/coroutine/detail/task-self-deleting.h +++ b/runtime-light/coroutine/detail/task-self-deleting.h @@ -69,7 +69,7 @@ class task_self_deleting { // initialize task_self_deleting's frame as top async stack frame auto& async_stack_frame{m_promise.get_async_stack_frame()}; async_stack_frame.return_address = STACK_RETURN_ADDRESS; - async_stack_frame.async_stack_root = CoroutineInstanceState::get_next_root(); + async_stack_frame.async_stack_root = CoroutineInstanceState::get().current_async_stack_root; } ~task_self_deleting() noexcept = default; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index b174f6380f..b3a60f3cb6 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -59,7 +59,7 @@ class when_all_latch { auto notify_awaitable_completed() noexcept -> void { if (--m_count == 1 && m_awaiting_coroutine != nullptr) { - kphp::coro::resume(m_awaiting_coroutine); + m_awaiting_coroutine.resume(); } } }; @@ -184,15 +184,16 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { auto start(when_all_latch& latch, void* return_address) noexcept { m_latch = std::addressof(latch); + kphp::coro::async_stack_root root{}; auto& async_stack_frame{get_async_stack_frame()}; // initialize when_all_task's async stack frame and make it the top frame async_stack_frame.caller_async_stack_frame = nullptr; - async_stack_frame.async_stack_root = CoroutineInstanceState::get_next_root(); + async_stack_frame.async_stack_root = std::addressof(root); async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); - kphp::coro::resume(handle, async_stack_frame.async_stack_root); + kphp::coro::resume_with_new_root(handle, std::addressof(root)); } }; diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 6c1d68cc65..057319dbe2 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -58,7 +58,7 @@ class when_any_latch { auto notify_awaitable_completed() noexcept -> void { m_toggled = true; if (m_awaiting_coroutine != nullptr) { - kphp::coro::resume(m_awaiting_coroutine); + m_awaiting_coroutine.resume(); } } }; @@ -194,15 +194,17 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { auto start(when_any_latch& latch, void* return_address) noexcept { m_latch = std::addressof(latch); + + kphp::coro::async_stack_root root{}; auto& async_stack_frame{get_async_stack_frame()}; // initialize when_all_task's async stack frame and make it the top frame async_stack_frame.caller_async_stack_frame = nullptr; - async_stack_frame.async_stack_root = CoroutineInstanceState::get_next_root(); + async_stack_frame.async_stack_root = std::addressof(root); async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); - kphp::coro::resume(handle, async_stack_frame.async_stack_root); + kphp::coro::resume_with_new_root(handle, std::addressof(root)); } }; diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index 8e428b72ce..ab112081e8 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -32,7 +32,7 @@ class event { explicit awaiter(event& event) noexcept : m_event(event), - m_async_stack_root(CoroutineInstanceState::get().coroutine_stack_root) {} + m_async_stack_root(CoroutineInstanceState::get().base_coroutine_stack_root) {} awaiter(const awaiter&) = delete; awaiter(awaiter&&) = delete; diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 8b395880aa..bfba57bdb7 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -396,8 +396,8 @@ inline auto io_scheduler::process_events() noexcept -> k2::PollStatus { kphp::log::assertion(m_scheduled_tasks_tmp.empty()); std::swap(m_scheduled_tasks, m_scheduled_tasks_tmp); - std::ranges::for_each(m_scheduled_tasks_tmp, [&coroutine_next_stack_root = m_coroutine_instance_state.next_root](const auto& coroutine) noexcept { - kphp::coro::resume(coroutine, coroutine_next_stack_root); + std::ranges::for_each(m_scheduled_tasks_tmp, [&coroutine_next_stack_root = m_coroutine_instance_state.current_async_stack_root](const auto& coroutine) noexcept { + kphp::coro::resume_with_new_root(coroutine, coroutine_next_stack_root); }); m_scheduled_tasks_tmp.clear(); } @@ -423,7 +423,7 @@ auto io_scheduler::start(coroutine_type coroutine) noexcept -> bool { if (!handle || handle.done()) [[unlikely]] { return false; } - kphp::coro::resume(handle, m_coroutine_instance_state.next_root); + kphp::coro::resume_with_new_root(handle, m_coroutine_instance_state.current_async_stack_root); return true; } @@ -435,7 +435,7 @@ inline auto io_scheduler::schedule() noexcept { explicit schedule_operation(io_scheduler& scheduler) noexcept : m_scheduler(scheduler), - m_async_stack_frame(m_scheduler.m_coroutine_instance_state.next_root->next_async_stack_root->top_async_stack_frame) {} + m_async_stack_frame(m_scheduler.m_coroutine_instance_state.current_async_stack_root->top_async_stack_frame) {} public: constexpr auto await_ready() const noexcept -> bool { @@ -447,7 +447,7 @@ inline auto io_scheduler::schedule() noexcept { } auto await_resume() const noexcept -> void { - m_scheduler.m_coroutine_instance_state.next_root->next_async_stack_root->top_async_stack_frame = m_async_stack_frame; + m_scheduler.m_coroutine_instance_state.current_async_stack_root->top_async_stack_frame = m_async_stack_frame; } }; return schedule_operation{*this}; diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index 4323335f7f..86646123c2 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -53,7 +53,7 @@ struct promise_base : kphp::coro::async_stack_element { // since resuming the coroutine may destroy the shared_task_waiter value auto* next{awaiter->m_next}; auto* async_stack_root{promise.get_async_stack_frame().async_stack_root}; - kphp::coro::resume(awaiter->m_continuation, async_stack_root); + kphp::coro::resume_with_new_root(awaiter->m_continuation, async_stack_root); awaiter = next; } // return last awaiter's coroutine_handle to allow it to potentially be compiled as a tail-call @@ -101,7 +101,7 @@ struct promise_base : kphp::coro::async_stack_element { m_awaiters = STARTED_NO_WAITERS_VAL; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; auto* async_stack_root{get_async_stack_frame().async_stack_root}; - kphp::coro::resume(handle, async_stack_root); + kphp::coro::resume_with_new_root(handle, async_stack_root); } // coroutine already completed, don't suspend if (done()) { diff --git a/runtime-light/state/instance-state.cpp b/runtime-light/state/instance-state.cpp index 9c3cd72e89..11ae6facf2 100644 --- a/runtime-light/state/instance-state.cpp +++ b/runtime-light/state/instance-state.cpp @@ -76,8 +76,8 @@ void InstanceState::init_script_execution() noexcept { std::move(script_task))}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; - main_task_async_stack_frame.async_stack_root = coroutine_instance_state.next_root; - coroutine_instance_state.next_root->top_async_stack_frame = std::addressof(main_task_async_stack_frame); + main_task_async_stack_frame.async_stack_root = coroutine_instance_state.current_async_stack_root; + coroutine_instance_state.current_async_stack_root->top_async_stack_frame = std::addressof(main_task_async_stack_frame); // spawn main task onto the scheduler kphp::log::assertion(io_scheduler.spawn(std::move(main_task))); } diff --git a/runtime-light/stdlib/diagnostics/backtrace.cpp b/runtime-light/stdlib/diagnostics/backtrace.cpp index a21b249dec..1fa6c950df 100644 --- a/runtime-light/stdlib/diagnostics/backtrace.cpp +++ b/runtime-light/stdlib/diagnostics/backtrace.cpp @@ -45,16 +45,13 @@ namespace kphp::diagnostic::impl { size_t async_backtrace_helper(std::span addresses, const kphp::coro::stack_frame* start_sync_frame, const kphp::coro::async_stack_root* async_stack_root) noexcept { - if (async_stack_root == nullptr) { - return 0; - } const auto* const stop_sync_frame{async_stack_root->stop_sync_stack_frame}; const size_t num_sync_frames{sync_frames(addresses, start_sync_frame, stop_sync_frame)}; const size_t num_async_frames{async_frames(addresses.subspan(num_sync_frames), async_stack_root->top_async_stack_frame)}; const size_t result{num_sync_frames + num_async_frames}; - if (addresses.size() < result) + if (addresses.size() < result || async_stack_root == async_stack_root->next_async_stack_root) return result; return result + async_backtrace_helper(addresses.subspan(result), stop_sync_frame, async_stack_root->next_async_stack_root); } @@ -62,7 +59,7 @@ size_t async_backtrace_helper(std::span addresses, const kphp::coro::stac size_t async_backtrace(std::span addresses) noexcept { if (const auto* instance_state{k2::instance_state()}; instance_state != nullptr) [[likely]] { return async_backtrace_helper(addresses, reinterpret_cast(STACK_FRAME_ADDRESS), - instance_state->coroutine_instance_state.next_root->next_async_stack_root); + instance_state->coroutine_instance_state.current_async_stack_root); } return 0; } From 5c41fd75b56f8aee6bfd6d0899220e0ffc0ec4f6 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Wed, 3 Dec 2025 13:05:52 +0300 Subject: [PATCH 04/13] add event, fix shared-task --- runtime-light/coroutine/async-stack-methods.h | 7 ++-- runtime-light/coroutine/detail/when-any.h | 1 - runtime-light/coroutine/event.h | 8 ++--- runtime-light/coroutine/shared-task.h | 32 ++++++++----------- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/runtime-light/coroutine/async-stack-methods.h b/runtime-light/coroutine/async-stack-methods.h index 9d1a6f7deb..0a3237094f 100644 --- a/runtime-light/coroutine/async-stack-methods.h +++ b/runtime-light/coroutine/async-stack-methods.h @@ -10,6 +10,7 @@ #include #include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { @@ -19,11 +20,13 @@ namespace kphp::coro { * capturing one of the stack frames in the synchronous stack trace and jump between parts of the async_stack. */ [[clang::noinline]] inline void resume_with_new_root(std::coroutine_handle<> handle, async_stack_root* new_async_stack_root) noexcept { + kphp::log::assertion(new_async_stack_root != nullptr); auto& coroutine_st{CoroutineInstanceState::get()}; - auto* previous_stack_root = coroutine_st.current_async_stack_root; - new_async_stack_root->next_async_stack_root = coroutine_st.current_async_stack_root; + auto* previous_stack_root{coroutine_st.current_async_stack_root}; + new_async_stack_root->next_async_stack_root = coroutine_st.current_async_stack_root; new_async_stack_root->stop_sync_stack_frame = reinterpret_cast(STACK_FRAME_ADDRESS); + coroutine_st.current_async_stack_root = new_async_stack_root; handle.resume(); coroutine_st.current_async_stack_root = previous_stack_root; diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 057319dbe2..03f949fdd9 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -194,7 +194,6 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { auto start(when_any_latch& latch, void* return_address) noexcept { m_latch = std::addressof(latch); - kphp::coro::async_stack_root root{}; auto& async_stack_frame{get_async_stack_frame()}; // initialize when_all_task's async stack frame and make it the top frame diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index ab112081e8..0aa645908f 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -24,7 +24,7 @@ class event { event& m_event; bool m_suspended{}; std::coroutine_handle<> m_awaiting_coroutine; - kphp::coro::async_stack_root& m_async_stack_root; + kphp::coro::async_stack_root* m_async_stack_root; kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; awaiter* m_next{}; @@ -32,7 +32,7 @@ class event { explicit awaiter(event& event) noexcept : m_event(event), - m_async_stack_root(CoroutineInstanceState::get().base_coroutine_stack_root) {} + m_async_stack_root(CoroutineInstanceState::get().current_async_stack_root) {} awaiter(const awaiter&) = delete; awaiter(awaiter&&) = delete; @@ -85,7 +85,7 @@ inline auto event::awaiter::await_ready() const noexcept -> bool { template caller_promise_type> auto event::awaiter::await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> void { // save caller's async stack frame - m_caller_async_stack_frame = m_async_stack_root.top_async_stack_frame; + m_caller_async_stack_frame = m_async_stack_root->top_async_stack_frame; m_suspended = true; m_awaiting_coroutine = awaiting_coroutine; @@ -105,7 +105,7 @@ inline auto event::awaiter::await_resume() noexcept -> void { if (std::exchange(m_suspended, false)) { // restore caller's async stack frame if it was suspended kphp::log::assertion(m_caller_async_stack_frame != nullptr); - m_async_stack_root.top_async_stack_frame = std::exchange(m_caller_async_stack_frame, nullptr); + m_async_stack_root->top_async_stack_frame = std::exchange(m_caller_async_stack_frame, nullptr); } } diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index 86646123c2..54fc57ea5f 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -52,8 +52,12 @@ struct promise_base : kphp::coro::async_stack_element { // read the m_next pointer before resuming the coroutine // since resuming the coroutine may destroy the shared_task_waiter value auto* next{awaiter->m_next}; - auto* async_stack_root{promise.get_async_stack_frame().async_stack_root}; - kphp::coro::resume_with_new_root(awaiter->m_continuation, async_stack_root); + auto& async_stack_frame{promise.get_async_stack_frame()}; + + kphp::coro::async_stack_root root{}; + root.top_async_stack_frame = std::addressof(async_stack_frame); + async_stack_frame.async_stack_root = std::addressof(root); + kphp::coro::resume_with_new_root(awaiter->m_continuation, std::addressof(root)); awaiter = next; } // return last awaiter's coroutine_handle to allow it to potentially be compiled as a tail-call @@ -100,8 +104,12 @@ struct promise_base : kphp::coro::async_stack_element { if (m_awaiters == NOT_STARTED_VAL) { m_awaiters = STARTED_NO_WAITERS_VAL; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; - auto* async_stack_root{get_async_stack_frame().async_stack_root}; - kphp::coro::resume_with_new_root(handle, async_stack_root); + auto& async_stack_frame{get_async_stack_frame()}; + + kphp::coro::async_stack_root root{}; + root.top_async_stack_frame = std::addressof(async_stack_frame); + async_stack_frame.async_stack_root = std::addressof(root); + kphp::coro::resume_with_new_root(handle, std::addressof(root)); } // coroutine already completed, don't suspend if (done()) { @@ -169,24 +177,13 @@ template class awaiter_base { bool m_suspended{}; - void set_async_top_frame(async_stack_frame& caller_frame, void* return_address) noexcept { + void set_async_top_frame(void* return_address) noexcept { /** * shared_task is the top of the stack for calls from it. * Therefore, it's awaiter doesn't store caller_frame, but it save `await_suspend()` return address * */ async_stack_frame& callee_frame{m_coro.promise().get_async_stack_frame()}; - callee_frame.return_address = return_address; - auto* async_stack_root{caller_frame.async_stack_root}; - kphp::log::assertion(async_stack_root != nullptr); - callee_frame.async_stack_root = async_stack_root; - async_stack_root->top_async_stack_frame = std::addressof(callee_frame); - } - - void reset_async_top_frame(async_stack_frame& caller_frame) noexcept { - auto* async_stack_root{caller_frame.async_stack_root}; - kphp::log::assertion(async_stack_root != nullptr); - async_stack_root->top_async_stack_frame = std::addressof(caller_frame); } protected: @@ -218,10 +215,9 @@ class awaiter_base { template caller_promise_type> [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> bool { - set_async_top_frame(awaiting_coroutine.promise().get_async_stack_frame(), STACK_RETURN_ADDRESS); + set_async_top_frame(STACK_RETURN_ADDRESS); m_awaiter.m_continuation = awaiting_coroutine; m_suspended = m_coro.promise().suspend_awaiter(m_awaiter); - reset_async_top_frame(awaiting_coroutine.promise().get_async_stack_frame()); return m_suspended; } From 8f05af748645309e70bf2095eaff3a549a3ac723 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Thu, 4 Dec 2025 19:10:16 +0300 Subject: [PATCH 05/13] major fix --- runtime-light/coroutine/async-stack-methods.h | 34 ------ runtime-light/coroutine/async-stack-structs.h | 104 ++++++++++++++++ runtime-light/coroutine/async-stack.h | 113 +++++------------- runtime-light/coroutine/await-set.h | 2 +- runtime-light/coroutine/coroutine-state.h | 2 +- runtime-light/coroutine/detail/await-set.h | 21 ++-- runtime-light/coroutine/detail/when-all.h | 19 ++- runtime-light/coroutine/detail/when-any.h | 19 ++- runtime-light/coroutine/event.h | 14 +-- runtime-light/coroutine/io-scheduler.h | 11 +- runtime-light/coroutine/shared-task.h | 30 +++-- runtime-light/coroutine/task.h | 2 + 12 files changed, 198 insertions(+), 173 deletions(-) delete mode 100644 runtime-light/coroutine/async-stack-methods.h create mode 100644 runtime-light/coroutine/async-stack-structs.h diff --git a/runtime-light/coroutine/async-stack-methods.h b/runtime-light/coroutine/async-stack-methods.h deleted file mode 100644 index 0a3237094f..0000000000 --- a/runtime-light/coroutine/async-stack-methods.h +++ /dev/null @@ -1,34 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2025 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include "runtime-light/coroutine/async-stack.h" - -#include -#include - -#include "runtime-light/coroutine/coroutine-state.h" -#include "runtime-light/stdlib/diagnostics/logs.h" - -namespace kphp::coro { - -/** - * The `resume_with_new_root` function is responsible for storing the current synchronous stack frame - * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows - * capturing one of the stack frames in the synchronous stack trace and jump between parts of the async_stack. - */ -[[clang::noinline]] inline void resume_with_new_root(std::coroutine_handle<> handle, async_stack_root* new_async_stack_root) noexcept { - kphp::log::assertion(new_async_stack_root != nullptr); - auto& coroutine_st{CoroutineInstanceState::get()}; - auto* previous_stack_root{coroutine_st.current_async_stack_root}; - - new_async_stack_root->next_async_stack_root = coroutine_st.current_async_stack_root; - new_async_stack_root->stop_sync_stack_frame = reinterpret_cast(STACK_FRAME_ADDRESS); - - coroutine_st.current_async_stack_root = new_async_stack_root; - handle.resume(); - coroutine_st.current_async_stack_root = previous_stack_root; -} -} // namespace kphp::coro diff --git a/runtime-light/coroutine/async-stack-structs.h b/runtime-light/coroutine/async-stack-structs.h new file mode 100644 index 0000000000..5fb3248abd --- /dev/null +++ b/runtime-light/coroutine/async-stack-structs.h @@ -0,0 +1,104 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2025 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +/** + * This header defines the data structures used to represent a coroutine asynchronous stack. + * + * Overview: + * The asynchronous stack is used to manage the execution state of coroutines, allowing for + * efficient context switching and stack management. + * + * Diagram: Normal and Asynchronous Stacks + * + * Base Pointer (%rbp) async_stack_root + * | | + * V V + * stack_frame async_stack_frame + * | | + * V V + * stack_frame async_stack_frame + * ... ... + * | | + * V V + * stack_frame async_stack_frame + * | | + * V V + * + * In the diagram above, the left side represents a typical call stack with stack frames linked by + * the base pointer (%rbp). The right side illustrates an asynchronous stack where `async_stack_frame` + * structures are linked by `async_stack_root`. + * + * Diagram: Backtrace Mechanism + * + * Base Pointer (%rbp) + * | + * V + * stack_frame + * | + * V + * stack_frame (stop_sync_frame) <- async_stack_root + * | + * V + * async_stack_frame (top_async_frame) + * | + * V + * async_stack_frame + * ... + * | + * V + * async_stack_frame + * + * The backtrace mechanism involves traversing the stack frames to capture the call stack. + * The `stop_sync_frame` serves as a marker where the transition to the asynchronous stack occurs, + * allowing the backtrace to continue through the `async_stack_frame` structures. + */ + +#define STACK_RETURN_ADDRESS __builtin_return_address(0) + +#define STACK_FRAME_ADDRESS __builtin_frame_address(0) + +namespace kphp::coro { + + + +struct stack_frame { + stack_frame* caller_stack_frame{}; + void* return_address{}; +}; + +struct async_stack_root; + +struct async_stack_frame { + async_stack_frame* caller_async_stack_frame{}; + async_stack_root* async_stack_root{}; + void* return_address{}; +}; + +struct async_stack_root { + async_stack_frame* top_async_stack_frame{}; + stack_frame* stop_sync_stack_frame{}; + async_stack_root* next_async_stack_root{}; +}; + +/** + * The async_stack_element class facilitates working with asynchronous traces in templated code. + * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. + */ +struct async_stack_element { + async_stack_frame& get_async_stack_frame() noexcept { + return m_async_stack_frame; + } + +private: + async_stack_frame m_async_stack_frame; +}; + +/** + * The async_stack_element class facilitates working with asynchronous traces in templated code. + * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. + */ + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index 3ef6938753..94faf2bc06 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -4,94 +4,47 @@ #pragma once -/** - * This header defines the data structures used to represent a coroutine asynchronous stack. - * - * Overview: - * The asynchronous stack is used to manage the execution state of coroutines, allowing for - * efficient context switching and stack management. - * - * Diagram: Normal and Asynchronous Stacks - * - * Base Pointer (%rbp) async_stack_root - * | | - * V V - * stack_frame async_stack_frame - * | | - * V V - * stack_frame async_stack_frame - * ... ... - * | | - * V V - * stack_frame async_stack_frame - * | | - * V V - * - * In the diagram above, the left side represents a typical call stack with stack frames linked by - * the base pointer (%rbp). The right side illustrates an asynchronous stack where `async_stack_frame` - * structures are linked by `async_stack_root`. - * - * Diagram: Backtrace Mechanism - * - * Base Pointer (%rbp) - * | - * V - * stack_frame - * | - * V - * stack_frame (stop_sync_frame) <- async_stack_root - * | - * V - * async_stack_frame (top_async_frame) - * | - * V - * async_stack_frame - * ... - * | - * V - * async_stack_frame - * - * The backtrace mechanism involves traversing the stack frames to capture the call stack. - * The `stop_sync_frame` serves as a marker where the transition to the asynchronous stack occurs, - * allowing the backtrace to continue through the `async_stack_frame` structures. - */ +#include "runtime-light/coroutine/async-stack-structs.h" -#define STACK_RETURN_ADDRESS __builtin_return_address(0) +#include +#include -#define STACK_FRAME_ADDRESS __builtin_frame_address(0) +#include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { -struct stack_frame { - stack_frame* caller_stack_frame{}; - void* return_address{}; +struct async_stack_root_wrapper { + async_stack_root root; + async_stack_root_wrapper() = default; + async_stack_root_wrapper(const async_stack_root_wrapper& other) = default; + async_stack_root_wrapper& operator=(const async_stack_root_wrapper& other) = default; + ~async_stack_root_wrapper() { + CoroutineInstanceState::get().current_async_stack_root = root.next_async_stack_root; + } }; -struct async_stack_root; - -struct async_stack_frame { - async_stack_frame* caller_async_stack_frame{}; - async_stack_root* async_stack_root{}; - void* return_address{}; -}; - -struct async_stack_root { - async_stack_frame* top_async_stack_frame{}; - stack_frame* stop_sync_stack_frame{}; - async_stack_root* next_async_stack_root{}; -}; +inline void preparation_for_resume(kphp::coro::async_stack_root* root, void* stack_frame_addr, + CoroutineInstanceState& coroutine_state = CoroutineInstanceState::get()) { + root->stop_sync_stack_frame = reinterpret_cast(stack_frame_addr); + coroutine_state.current_async_stack_root = root; +} /** - * The async_stack_element class facilitates working with asynchronous traces in templated code. - * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. + * The `resume_with_new_root` function is responsible for storing the current synchronous stack frame + * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows + * capturing one of the stack frames in the synchronous stack trace and jump between parts of the async_stack. */ -struct async_stack_element { - async_stack_frame& get_async_stack_frame() noexcept { - return m_async_stack_frame; - } - -private: - async_stack_frame m_async_stack_frame; -}; - +[[clang::noinline]] inline void resume_with_new_root(std::coroutine_handle<> handle, async_stack_root* new_async_stack_root) noexcept { + kphp::log::assertion(new_async_stack_root != nullptr); + auto& coroutine_st{CoroutineInstanceState::get()}; + // auto* previous_stack_root{coroutine_st.current_async_stack_root}; + + new_async_stack_root->next_async_stack_root = coroutine_st.current_async_stack_root; + new_async_stack_root->stop_sync_stack_frame = reinterpret_cast(STACK_FRAME_ADDRESS); + + coroutine_st.current_async_stack_root = new_async_stack_root; + handle.resume(); + new_async_stack_root->stop_sync_stack_frame = nullptr; +} } // namespace kphp::coro diff --git a/runtime-light/coroutine/await-set.h b/runtime-light/coroutine/await-set.h index 8f6a1bfd37..324091d48b 100644 --- a/runtime-light/coroutine/await-set.h +++ b/runtime-light/coroutine/await-set.h @@ -40,7 +40,7 @@ class await_set { template requires kphp::coro::concepts::awaitable && std::is_same_v::awaiter_return_type, return_type> void push(awaitable_type awaitable) noexcept { - m_await_broker->start_task(detail::await_set::make_await_set_task(std::move(awaitable)), STACK_RETURN_ADDRESS); + m_await_broker->start_task(detail::await_set::make_await_set_task(std::move(awaitable))); } auto next() noexcept { diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index 42ced3c635..2a77c70977 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -6,7 +6,7 @@ #include "common/mixin/not_copyable.h" -#include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/async-stack-structs.h" struct CoroutineInstanceState final : private vk::not_copyable { diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index d130a16569..adf691faff 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -12,7 +12,6 @@ #include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-common/core/std/containers.h" -#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" @@ -66,9 +65,9 @@ class await_broker { kphp::memory::script::free(ptr); } - void start_task(await_set_task&& task, void* return_address) noexcept { + void start_task(await_set_task&& task) noexcept { auto task_iterator{m_tasks_storage.insert(m_tasks_storage.begin(), std::move(task))}; - task_iterator->start(*this, task_iterator, return_address); + task_iterator->start(*this, task_iterator); } void push_ready_task(await_set_ready_task_element& ready_task) noexcept { @@ -158,6 +157,7 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { std::optional>> m_await_broker{}; await_set_ready_task_element m_ready_task_element{}; + kphp::coro::async_stack_root_wrapper root_wrapper_{}; public: await_set_task_promise_base() noexcept = default; @@ -197,8 +197,8 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { kphp::log::error("internal unhandled exception"); } - auto start(detail::await_set::await_broker& await_broker, - kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, void* return_address) noexcept { + [[clang::noinline]] auto start(detail::await_set::await_broker& await_broker, + kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location) noexcept { m_await_broker = await_broker; m_ready_task_element.m_storage_location = storage_location; @@ -207,15 +207,14 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { * await_set_task is the top of the stack for calls from it. * Therefore, it doesn't store caller_frame, but it save `await_set::push()` return address * */ - kphp::coro::async_stack_root root{}; auto& async_stack_frame{get_async_stack_frame()}; async_stack_frame.caller_async_stack_frame = nullptr; - async_stack_frame.async_stack_root = std::addressof(root); - async_stack_frame.return_address = return_address; + async_stack_frame.async_stack_root = std::addressof(root_wrapper_.root); + async_stack_frame.return_address = STACK_RETURN_ADDRESS; // this is necessary to prevent double printing of a coroutine async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); - kphp::coro::resume_with_new_root(handle, std::addressof(root)); + kphp::coro::resume_with_new_root(handle, std::addressof(root_wrapper_.root)); } }; @@ -271,8 +270,8 @@ class await_set_task { }; auto start(detail::await_set::await_broker& await_broker, - kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, void* return_address) noexcept { - m_coroutine.promise().start(await_broker, storage_location, return_address); + kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location) noexcept { + m_coroutine.promise().start(await_broker, storage_location); } public: diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index b3a60f3cb6..cf829ac13c 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -14,7 +14,6 @@ #include #include -#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" @@ -105,10 +104,9 @@ class when_all_ready_awaitable> { template caller_promise_type> [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> bool { // async stack frame handling - void* const return_address{STACK_RETURN_ADDRESS}; m_caller_async_stack_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); - std::apply([&latch = m_awaitable.m_latch, return_address](auto&... tasks) noexcept { (tasks.start(latch, return_address), ...); }, m_awaitable.m_tasks); + std::apply([&latch = m_awaitable.m_latch](auto&... tasks) noexcept { (tasks.start(latch), ...); }, m_awaitable.m_tasks); return m_awaitable.m_latch.try_await(awaiting_coroutine); } @@ -145,7 +143,7 @@ class when_all_ready_awaitable> { template class when_all_task_promise_base : public kphp::coro::async_stack_element { when_all_latch* m_latch{}; - + kphp::coro::async_stack_root_wrapper root_wrapper_{}; public: when_all_task_promise_base() noexcept = default; @@ -181,19 +179,18 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { kphp::log::error("internal unhandled exception"); } - auto start(when_all_latch& latch, void* return_address) noexcept { + [[clang::noinline]] auto start(when_all_latch& latch) noexcept { m_latch = std::addressof(latch); - kphp::coro::async_stack_root root{}; auto& async_stack_frame{get_async_stack_frame()}; // initialize when_all_task's async stack frame and make it the top frame async_stack_frame.caller_async_stack_frame = nullptr; - async_stack_frame.async_stack_root = std::addressof(root); - async_stack_frame.return_address = return_address; + async_stack_frame.async_stack_root = std::addressof(root_wrapper_.root); + async_stack_frame.return_address = STACK_RETURN_ADDRESS; // this is necessary to prevent double printing of a coroutine async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); - kphp::coro::resume_with_new_root(handle, std::addressof(root)); + kphp::coro::resume_with_new_root(handle, std::addressof(root_wrapper_.root)); } }; @@ -248,8 +245,8 @@ class when_all_task { constexpr auto return_void() const noexcept -> void {} }; - auto start(when_all_latch& latch, void* return_address) noexcept -> void { - m_coroutine.promise().start(latch, return_address); + auto start(when_all_latch& latch) noexcept -> void { + m_coroutine.promise().start(latch); } public: diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 03f949fdd9..0481c0d2dc 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -12,7 +12,6 @@ #include #include -#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" @@ -105,14 +104,14 @@ class when_any_ready_awaitable> { template caller_promise_type> [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> bool { // async stack frame handling - void* const return_address{STACK_RETURN_ADDRESS}; m_caller_async_stack_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); - std::apply([&latch = m_awaitable.m_latch, return_address](auto&... tasks) noexcept { (tasks.start(latch, return_address), ...); }, m_awaitable.m_tasks); + std::apply([&latch = m_awaitable.m_latch](auto&... tasks) noexcept { (tasks.start(latch), ...); }, m_awaitable.m_tasks); return m_awaitable.m_latch.try_await(awaiting_coroutine); } auto await_resume() noexcept { + kphp::log::debug("await_resume in when_any"); // restore caller's async_stack_frame unless it's not set which could happen in case no suspension occured if (m_caller_async_stack_frame != nullptr) { kphp::log::assertion(m_caller_async_stack_frame->async_stack_root != nullptr); @@ -155,6 +154,7 @@ class when_any_ready_awaitable> { template class when_any_task_promise_base : public kphp::coro::async_stack_element { when_any_latch* m_latch{}; + kphp::coro::async_stack_root_wrapper root_wrapper_{}; public: when_any_task_promise_base() noexcept = default; @@ -191,19 +191,18 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { kphp::log::error("internal unhandled exception"); } - auto start(when_any_latch& latch, void* return_address) noexcept { + [[clang::noinline]] auto start(when_any_latch& latch) noexcept { m_latch = std::addressof(latch); - kphp::coro::async_stack_root root{}; auto& async_stack_frame{get_async_stack_frame()}; // initialize when_all_task's async stack frame and make it the top frame async_stack_frame.caller_async_stack_frame = nullptr; - async_stack_frame.async_stack_root = std::addressof(root); - async_stack_frame.return_address = return_address; + async_stack_frame.async_stack_root = std::addressof(root_wrapper_.root); + async_stack_frame.return_address = STACK_RETURN_ADDRESS; // this is necessary to prevent double printing of a coroutine async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); - kphp::coro::resume_with_new_root(handle, std::addressof(root)); + kphp::coro::resume_with_new_root(handle, std::addressof(root_wrapper_.root)); } }; @@ -262,9 +261,9 @@ class when_any_task { constexpr auto return_void() const noexcept -> void {} }; - auto start(when_any_latch& latch, void* return_address) noexcept -> void { + auto start(when_any_latch& latch) noexcept -> void { if (!latch.is_ready()) [[likely]] { - m_coroutine.promise().start(latch, return_address); + m_coroutine.promise().start(latch); } } diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index 0aa645908f..3cf9e3fe96 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -19,20 +19,20 @@ class event { // 2) awaiter* => linked list of awaiters waiting for the event to trigger // 3) this => The event is triggered and all awaiters are resumed void* m_state{}; + kphp::coro::async_stack_root* m_async_stack_root{}; struct awaiter { event& m_event; bool m_suspended{}; std::coroutine_handle<> m_awaiting_coroutine; - kphp::coro::async_stack_root* m_async_stack_root; - kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; + CoroutineInstanceState& m_coroutine_state; awaiter* m_next{}; awaiter* m_prev{}; explicit awaiter(event& event) noexcept : m_event(event), - m_async_stack_root(CoroutineInstanceState::get().current_async_stack_root) {} + m_coroutine_state(CoroutineInstanceState::get()) {} awaiter(const awaiter&) = delete; awaiter(awaiter&&) = delete; @@ -84,8 +84,7 @@ inline auto event::awaiter::await_ready() const noexcept -> bool { template caller_promise_type> auto event::awaiter::await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> void { - // save caller's async stack frame - m_caller_async_stack_frame = m_async_stack_root->top_async_stack_frame; + m_event.m_async_stack_root = m_coroutine_state.current_async_stack_root; m_suspended = true; m_awaiting_coroutine = awaiting_coroutine; @@ -103,9 +102,7 @@ auto event::awaiter::await_suspend(std::coroutine_handle aw inline auto event::awaiter::await_resume() noexcept -> void { if (std::exchange(m_suspended, false)) { - // restore caller's async stack frame if it was suspended - kphp::log::assertion(m_caller_async_stack_frame != nullptr); - m_async_stack_root->top_async_stack_frame = std::exchange(m_caller_async_stack_frame, nullptr); + kphp::coro::preparation_for_resume(m_event.m_async_stack_root, STACK_FRAME_ADDRESS, m_coroutine_state); } } @@ -118,6 +115,7 @@ inline auto event::set() noexcept -> void { while (awaiter != nullptr) { auto* next{awaiter->m_next}; awaiter->m_awaiting_coroutine.resume(); + m_async_stack_root->stop_sync_stack_frame = nullptr; awaiter = next; } } diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index bfba57bdb7..9e93b236f9 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -23,7 +23,6 @@ #include "common/containers/final_action.h" #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" -#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/coroutine-state.h" @@ -396,9 +395,7 @@ inline auto io_scheduler::process_events() noexcept -> k2::PollStatus { kphp::log::assertion(m_scheduled_tasks_tmp.empty()); std::swap(m_scheduled_tasks, m_scheduled_tasks_tmp); - std::ranges::for_each(m_scheduled_tasks_tmp, [&coroutine_next_stack_root = m_coroutine_instance_state.current_async_stack_root](const auto& coroutine) noexcept { - kphp::coro::resume_with_new_root(coroutine, coroutine_next_stack_root); - }); + std::ranges::for_each(m_scheduled_tasks_tmp, [](const auto& coroutine) noexcept { coroutine.resume(); }); m_scheduled_tasks_tmp.clear(); } @@ -431,11 +428,11 @@ inline auto io_scheduler::schedule() noexcept { class schedule_operation { friend class io_scheduler; io_scheduler& m_scheduler; - kphp::coro::async_stack_frame* const m_async_stack_frame{}; + kphp::coro::async_stack_root* const m_async_stack_root{}; explicit schedule_operation(io_scheduler& scheduler) noexcept : m_scheduler(scheduler), - m_async_stack_frame(m_scheduler.m_coroutine_instance_state.current_async_stack_root->top_async_stack_frame) {} + m_async_stack_root(m_scheduler.m_coroutine_instance_state.current_async_stack_root) {} public: constexpr auto await_ready() const noexcept -> bool { @@ -447,7 +444,7 @@ inline auto io_scheduler::schedule() noexcept { } auto await_resume() const noexcept -> void { - m_scheduler.m_coroutine_instance_state.current_async_stack_root->top_async_stack_frame = m_async_stack_frame; + kphp::coro::preparation_for_resume(m_async_stack_root, STACK_FRAME_ADDRESS, m_scheduler.m_coroutine_instance_state); } }; return schedule_operation{*this}; diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index 54fc57ea5f..5188ebf35d 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -14,7 +14,6 @@ #include #include "runtime-common/core/allocator/script-malloc-interface.h" -#include "runtime-light/coroutine/async-stack-methods.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -30,12 +29,23 @@ struct shared_task_awaiter final { template struct promise_base : kphp::coro::async_stack_element { +private: + mutable kphp::coro::async_stack_root_wrapper root_wrapper_{}; + +public: constexpr auto initial_suspend() const noexcept -> std::suspend_always { return {}; } constexpr auto final_suspend() const noexcept { struct awaiter { + private: + mutable kphp::coro::async_stack_root* root; + + public: + awaiter(kphp::coro::async_stack_root* prt) + : root{prt} {} + constexpr auto await_ready() const noexcept -> bool { return false; } @@ -54,10 +64,9 @@ struct promise_base : kphp::coro::async_stack_element { auto* next{awaiter->m_next}; auto& async_stack_frame{promise.get_async_stack_frame()}; - kphp::coro::async_stack_root root{}; - root.top_async_stack_frame = std::addressof(async_stack_frame); - async_stack_frame.async_stack_root = std::addressof(root); - kphp::coro::resume_with_new_root(awaiter->m_continuation, std::addressof(root)); + root->top_async_stack_frame = std::addressof(async_stack_frame); + async_stack_frame.async_stack_root = root; + kphp::coro::resume_with_new_root(awaiter->m_continuation, root); awaiter = next; } // return last awaiter's coroutine_handle to allow it to potentially be compiled as a tail-call @@ -66,7 +75,7 @@ struct promise_base : kphp::coro::async_stack_element { constexpr auto await_resume() const noexcept -> void {} }; - return awaiter{}; + return awaiter{std::addressof(root_wrapper_.root)}; } auto unhandled_exception() const noexcept -> void { @@ -106,10 +115,9 @@ struct promise_base : kphp::coro::async_stack_element { const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; auto& async_stack_frame{get_async_stack_frame()}; - kphp::coro::async_stack_root root{}; - root.top_async_stack_frame = std::addressof(async_stack_frame); - async_stack_frame.async_stack_root = std::addressof(root); - kphp::coro::resume_with_new_root(handle, std::addressof(root)); + async_stack_frame.async_stack_root = std::addressof(root_wrapper_.root); + async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); + kphp::coro::resume_with_new_root(handle, std::addressof(root_wrapper_.root)); } // coroutine already completed, don't suspend if (done()) { @@ -222,6 +230,8 @@ class awaiter_base { } auto await_resume() noexcept -> void { + auto& frame = m_coro.promise().get_async_stack_frame(); + kphp::coro::preparation_for_resume(frame.async_stack_root, STACK_FRAME_ADDRESS); m_suspended = false; } }; diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index 7ba3fb2041..93a70e61ca 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -133,6 +133,8 @@ class awaiter_base { } auto await_resume() noexcept -> void { + auto& frame = m_coro.promise().get_async_stack_frame(); + kphp::coro::preparation_for_resume(frame.async_stack_root, STACK_FRAME_ADDRESS); m_suspended = false; } }; From 0baca2bee6dae912f20ebb08f4c2102e280fa3e3 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Fri, 5 Dec 2025 11:39:56 +0300 Subject: [PATCH 06/13] fix async_backtrace_helper --- runtime-light/stdlib/diagnostics/backtrace.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/runtime-light/stdlib/diagnostics/backtrace.cpp b/runtime-light/stdlib/diagnostics/backtrace.cpp index 1fa6c950df..490c585420 100644 --- a/runtime-light/stdlib/diagnostics/backtrace.cpp +++ b/runtime-light/stdlib/diagnostics/backtrace.cpp @@ -47,12 +47,16 @@ size_t async_backtrace_helper(std::span addresses, const kphp::coro::stac const kphp::coro::async_stack_root* async_stack_root) noexcept { const auto* const stop_sync_frame{async_stack_root->stop_sync_stack_frame}; - const size_t num_sync_frames{sync_frames(addresses, start_sync_frame, stop_sync_frame)}; + size_t num_sync_frames{0}; + if (start_sync_frame && stop_sync_frame) { + num_sync_frames = sync_frames(addresses, start_sync_frame, stop_sync_frame); + } const size_t num_async_frames{async_frames(addresses.subspan(num_sync_frames), async_stack_root->top_async_stack_frame)}; const size_t result{num_sync_frames + num_async_frames}; - if (addresses.size() < result || async_stack_root == async_stack_root->next_async_stack_root) + if (async_stack_root == async_stack_root->next_async_stack_root) { return result; + } return result + async_backtrace_helper(addresses.subspan(result), stop_sync_frame, async_stack_root->next_async_stack_root); } From a1ee47bc0abda62bdcd8e6123b8319bbb8c30098 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Fri, 5 Dec 2025 11:47:40 +0300 Subject: [PATCH 07/13] del comm --- runtime-light/coroutine/async-stack.h | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index 94faf2bc06..f4a160b743 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -38,7 +38,6 @@ inline void preparation_for_resume(kphp::coro::async_stack_root* root, void* sta [[clang::noinline]] inline void resume_with_new_root(std::coroutine_handle<> handle, async_stack_root* new_async_stack_root) noexcept { kphp::log::assertion(new_async_stack_root != nullptr); auto& coroutine_st{CoroutineInstanceState::get()}; - // auto* previous_stack_root{coroutine_st.current_async_stack_root}; new_async_stack_root->next_async_stack_root = coroutine_st.current_async_stack_root; new_async_stack_root->stop_sync_stack_frame = reinterpret_cast(STACK_FRAME_ADDRESS); From 9c7ad36811fa0a56388267852c4131ee4b478b84 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Fri, 5 Dec 2025 12:00:00 +0300 Subject: [PATCH 08/13] fix --- runtime-light/coroutine/async-stack.h | 4 ++-- runtime-light/coroutine/event.h | 2 +- runtime-light/coroutine/io-scheduler.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index f4a160b743..ec93d44b90 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -24,8 +24,8 @@ struct async_stack_root_wrapper { } }; -inline void preparation_for_resume(kphp::coro::async_stack_root* root, void* stack_frame_addr, - CoroutineInstanceState& coroutine_state = CoroutineInstanceState::get()) { +inline void preparation_for_resume(kphp::coro::async_stack_root* root, void* stack_frame_addr) { + static CoroutineInstanceState& coroutine_state = CoroutineInstanceState::get(); root->stop_sync_stack_frame = reinterpret_cast(stack_frame_addr); coroutine_state.current_async_stack_root = root; } diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index 3cf9e3fe96..ae4753eac1 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -102,7 +102,7 @@ auto event::awaiter::await_suspend(std::coroutine_handle aw inline auto event::awaiter::await_resume() noexcept -> void { if (std::exchange(m_suspended, false)) { - kphp::coro::preparation_for_resume(m_event.m_async_stack_root, STACK_FRAME_ADDRESS, m_coroutine_state); + kphp::coro::preparation_for_resume(m_event.m_async_stack_root, STACK_FRAME_ADDRESS); } } diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 9e93b236f9..4541befe90 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -444,7 +444,7 @@ inline auto io_scheduler::schedule() noexcept { } auto await_resume() const noexcept -> void { - kphp::coro::preparation_for_resume(m_async_stack_root, STACK_FRAME_ADDRESS, m_scheduler.m_coroutine_instance_state); + kphp::coro::preparation_for_resume(m_async_stack_root, STACK_FRAME_ADDRESS); } }; return schedule_operation{*this}; From 340d6d4edf68e3d04438b54f0295f9757b3628c4 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Fri, 5 Dec 2025 12:25:05 +0300 Subject: [PATCH 09/13] fix --- runtime-light/coroutine/detail/await-set.h | 2 ++ runtime-light/coroutine/detail/when-all.h | 7 +++++-- runtime-light/coroutine/detail/when-any.h | 8 +++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index adf691faff..932e447fd0 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -343,6 +343,8 @@ class await_set_awaitable { kphp::log::assertion(async_stack_root != nullptr); async_stack_root->top_async_stack_frame = caller_frame; + kphp::coro::preparation_for_resume(async_stack_root, STACK_FRAME_ADDRESS); + m_suspended = false; return m_await_broker.try_get_result(); } diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index cf829ac13c..fead099de6 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -113,8 +113,11 @@ class when_all_ready_awaitable> { auto await_resume() noexcept { // restore caller's async_stack_frame unless it's not set which could happen in case no suspension occured if (m_caller_async_stack_frame != nullptr) { - kphp::log::assertion(m_caller_async_stack_frame->async_stack_root != nullptr); - m_caller_async_stack_frame->async_stack_root->top_async_stack_frame = m_caller_async_stack_frame; + auto* root = m_caller_async_stack_frame->async_stack_root; + kphp::log::assertion(root != nullptr); + root->top_async_stack_frame = m_caller_async_stack_frame; + + kphp::coro::preparation_for_resume(root, STACK_FRAME_ADDRESS); } return std::apply([](task_types&&... tasks) noexcept { return std::make_tuple(std::move(tasks).result()...); }, std::move(m_awaitable.m_tasks)); } diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 0481c0d2dc..9721978974 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -111,11 +111,13 @@ class when_any_ready_awaitable> { } auto await_resume() noexcept { - kphp::log::debug("await_resume in when_any"); // restore caller's async_stack_frame unless it's not set which could happen in case no suspension occured if (m_caller_async_stack_frame != nullptr) { - kphp::log::assertion(m_caller_async_stack_frame->async_stack_root != nullptr); - m_caller_async_stack_frame->async_stack_root->top_async_stack_frame = m_caller_async_stack_frame; + auto* root = m_caller_async_stack_frame->async_stack_root; + kphp::log::assertion(root != nullptr); + root->top_async_stack_frame = m_caller_async_stack_frame; + + kphp::coro::preparation_for_resume(root, STACK_FRAME_ADDRESS); } const auto task_result_processor{[&result = m_awaitable.m_result](auto&& task) noexcept { From 402c7d482111402549e62e0d379b9924ad4504e3 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Fri, 5 Dec 2025 12:49:28 +0300 Subject: [PATCH 10/13] fix --- runtime-light/coroutine/detail/await-set.h | 7 ++++++- runtime-light/coroutine/detail/when-all.h | 8 ++++++-- runtime-light/coroutine/detail/when-any.h | 7 +++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index 932e447fd0..682447a809 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -30,6 +30,7 @@ struct await_set_awaiter { await_set_awaiter* m_next{}; await_set_awaiter* m_prev{}; std::coroutine_handle<> m_continuation; + kphp::coro::async_stack_root* m_async_stack_root{}; }; template @@ -81,6 +82,7 @@ class await_broker { m_awaiters->m_prev = nullptr; } awaiter->m_continuation.resume(); + awaiter->m_async_stack_root->stop_sync_stack_frame = nullptr; } } @@ -140,6 +142,7 @@ class await_broker { awaiters_to_resume->m_prev = nullptr; } waiter->m_continuation.resume(); + waiter->m_async_stack_root->stop_sync_stack_frame = nullptr; } } @@ -158,6 +161,7 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { await_set_ready_task_element m_ready_task_element{}; kphp::coro::async_stack_root_wrapper root_wrapper_{}; + public: await_set_task_promise_base() noexcept = default; @@ -198,7 +202,7 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { } [[clang::noinline]] auto start(detail::await_set::await_broker& await_broker, - kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location) noexcept { + kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location) noexcept { m_await_broker = await_broker; m_ready_task_element.m_storage_location = storage_location; @@ -332,6 +336,7 @@ class await_set_awaitable { caller_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); m_awaiter.m_continuation = awaiting_coroutine; + m_awaiter.m_async_stack_root = caller_frame->async_stack_root; m_suspended = m_await_broker.suspend_awaiter(m_awaiter); return m_suspended; } diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index fead099de6..fa751029b9 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -25,6 +25,7 @@ namespace kphp::coro::detail::when_all { class when_all_latch { size_t m_count{1}; std::coroutine_handle<> m_awaiting_coroutine; + kphp::coro::async_stack_root* m_async_stack_root{}; public: explicit when_all_latch(size_t count) noexcept @@ -51,14 +52,16 @@ class when_all_latch { return m_awaiting_coroutine != nullptr && m_count == 1; } - auto try_await(std::coroutine_handle<> awaiting_coroutine) noexcept -> bool { + auto try_await(std::coroutine_handle<> awaiting_coroutine, kphp::coro::async_stack_root* root) noexcept -> bool { m_awaiting_coroutine = awaiting_coroutine; + m_async_stack_root = root; return m_count != 1; } auto notify_awaitable_completed() noexcept -> void { if (--m_count == 1 && m_awaiting_coroutine != nullptr) { m_awaiting_coroutine.resume(); + m_async_stack_root->stop_sync_stack_frame = nullptr; } } }; @@ -107,7 +110,7 @@ class when_all_ready_awaitable> { m_caller_async_stack_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); std::apply([&latch = m_awaitable.m_latch](auto&... tasks) noexcept { (tasks.start(latch), ...); }, m_awaitable.m_tasks); - return m_awaitable.m_latch.try_await(awaiting_coroutine); + return m_awaitable.m_latch.try_await(awaiting_coroutine, m_caller_async_stack_frame->async_stack_root); } auto await_resume() noexcept { @@ -147,6 +150,7 @@ template class when_all_task_promise_base : public kphp::coro::async_stack_element { when_all_latch* m_latch{}; kphp::coro::async_stack_root_wrapper root_wrapper_{}; + public: when_all_task_promise_base() noexcept = default; diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 9721978974..b4632b8468 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -24,6 +24,7 @@ namespace kphp::coro::detail::when_any { class when_any_latch { bool m_toggled{}; std::coroutine_handle<> m_awaiting_coroutine; + kphp::coro::async_stack_root* m_async_stack_root{}; public: when_any_latch() noexcept = default; @@ -49,8 +50,9 @@ class when_any_latch { return m_toggled; } - auto try_await(std::coroutine_handle<> awaiting_coroutine) noexcept -> bool { + auto try_await(std::coroutine_handle<> awaiting_coroutine, kphp::coro::async_stack_root* root) noexcept -> bool { m_awaiting_coroutine = awaiting_coroutine; + m_async_stack_root = root; return !m_toggled; } @@ -58,6 +60,7 @@ class when_any_latch { m_toggled = true; if (m_awaiting_coroutine != nullptr) { m_awaiting_coroutine.resume(); + m_async_stack_root->stop_sync_stack_frame = nullptr; } } }; @@ -107,7 +110,7 @@ class when_any_ready_awaitable> { m_caller_async_stack_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); std::apply([&latch = m_awaitable.m_latch](auto&... tasks) noexcept { (tasks.start(latch), ...); }, m_awaitable.m_tasks); - return m_awaitable.m_latch.try_await(awaiting_coroutine); + return m_awaitable.m_latch.try_await(awaiting_coroutine, m_caller_async_stack_frame->async_stack_root); } auto await_resume() noexcept { From 8cda6dbd3a543e3e7451226d8fe818dc11061b44 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Fri, 5 Dec 2025 12:53:15 +0300 Subject: [PATCH 11/13] minifix --- runtime-light/coroutine/async-stack-structs.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/runtime-light/coroutine/async-stack-structs.h b/runtime-light/coroutine/async-stack-structs.h index 5fb3248abd..578060ed8f 100644 --- a/runtime-light/coroutine/async-stack-structs.h +++ b/runtime-light/coroutine/async-stack-structs.h @@ -62,8 +62,6 @@ namespace kphp::coro { - - struct stack_frame { stack_frame* caller_stack_frame{}; void* return_address{}; From ce8ebb7af014c4808aa99ca66e01a3f5205dcabd Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Fri, 5 Dec 2025 13:17:56 +0300 Subject: [PATCH 12/13] minifix --- runtime-light/coroutine/async-stack.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index ec93d44b90..0dd2de04cb 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -16,9 +16,12 @@ namespace kphp::coro { struct async_stack_root_wrapper { async_stack_root root; + async_stack_root_wrapper() = default; - async_stack_root_wrapper(const async_stack_root_wrapper& other) = default; - async_stack_root_wrapper& operator=(const async_stack_root_wrapper& other) = default; + async_stack_root_wrapper(const async_stack_root_wrapper& other) = delete; + async_stack_root_wrapper& operator=(const async_stack_root_wrapper& other) = delete; + async_stack_root_wrapper(async_stack_root_wrapper&& other) = default; + async_stack_root_wrapper& operator=(async_stack_root_wrapper&& other) = default; ~async_stack_root_wrapper() { CoroutineInstanceState::get().current_async_stack_root = root.next_async_stack_root; } From 4ca71fe45443175115aa70bab4962d60614dcd52 Mon Sep 17 00:00:00 2001 From: Denis Zubarev Date: Mon, 8 Dec 2025 10:17:44 +0300 Subject: [PATCH 13/13] fix --- runtime-light/coroutine/async-stack.h | 4 ++-- runtime-light/coroutine/event.h | 2 +- runtime-light/coroutine/io-scheduler.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index 0dd2de04cb..f3e248e2df 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -27,8 +27,8 @@ struct async_stack_root_wrapper { } }; -inline void preparation_for_resume(kphp::coro::async_stack_root* root, void* stack_frame_addr) { - static CoroutineInstanceState& coroutine_state = CoroutineInstanceState::get(); +inline void preparation_for_resume(kphp::coro::async_stack_root* root, void* stack_frame_addr, + CoroutineInstanceState& coroutine_state = CoroutineInstanceState::get()) { root->stop_sync_stack_frame = reinterpret_cast(stack_frame_addr); coroutine_state.current_async_stack_root = root; } diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index ae4753eac1..3cf9e3fe96 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -102,7 +102,7 @@ auto event::awaiter::await_suspend(std::coroutine_handle aw inline auto event::awaiter::await_resume() noexcept -> void { if (std::exchange(m_suspended, false)) { - kphp::coro::preparation_for_resume(m_event.m_async_stack_root, STACK_FRAME_ADDRESS); + kphp::coro::preparation_for_resume(m_event.m_async_stack_root, STACK_FRAME_ADDRESS, m_coroutine_state); } } diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 4541befe90..9e93b236f9 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -444,7 +444,7 @@ inline auto io_scheduler::schedule() noexcept { } auto await_resume() const noexcept -> void { - kphp::coro::preparation_for_resume(m_async_stack_root, STACK_FRAME_ADDRESS); + kphp::coro::preparation_for_resume(m_async_stack_root, STACK_FRAME_ADDRESS, m_scheduler.m_coroutine_instance_state); } }; return schedule_operation{*this};