use symmetric transfer when co_await'ing a task from another task

.github/workflows/ci.cpu.yml

@@ -170,7 +170,7 @@ jobs:
       - run: echo "CI (CPU) (Windows) success"
 
   build-cpu-macos:
-    runs-on: macos-15-large
+    runs-on: macos-26-large
     name: macos-${{ matrix.name }}
     strategy:
       fail-fast: false

include/exec/any_sender_of.hpp

@@ -977,7 +977,7 @@ namespace experimental::execution
       STDEXEC_ATTRIBUTE(no_unique_address) _Receiver __rcvr_;
       STDEXEC::inplace_stop_source __stop_source_{};
       using __stop_callback = typename STDEXEC::stop_token_of_t<
-        STDEXEC::env_of_t<_Receiver>>::template callback_type<STDEXEC::__forward_stop_request>;
+        STDEXEC::env_of_t<_Receiver>>::template callback_type<STDEXEC::__forward_stop_request<>>;
       std::optional<__stop_callback> __on_stop_{};
     };
 

include/exec/task.hpp

@@ -186,7 +186,7 @@ namespace experimental::execution
     struct __default_awaiter_context<_ParentPromise>
     {
       using __stop_token_t    = stop_token_of_t<env_of_t<_ParentPromise>>;
-      using __stop_callback_t = __stop_token_t::template callback_type<__forward_stop_request>;
+      using __stop_callback_t = __stop_token_t::template callback_type<__forward_stop_request<>>;
 
       template <__scheduler_affinity _Affinity>
       constexpr explicit __default_awaiter_context(__default_task_context_impl<_Affinity>& __self,
@@ -199,7 +199,7 @@ namespace experimental::execution
       {
         static_assert(std::is_nothrow_constructible_v<__stop_callback_t,
                                                       __stop_token_t,
-                                                      __forward_stop_request>);
+                                                      __forward_stop_request<>>);
         __self.__stop_token_ = __stop_source_.get_token();
       }
 
@@ -251,7 +251,7 @@ namespace experimental::execution
         // stop_source when stop is requested on the parent coroutine's stop
         // token.
         using __stop_token_t    = stop_token_of_t<env_of_t<_ParentPromise>>;
-        using __stop_callback_t = stop_callback_for_t<__stop_token_t, __forward_stop_request>;
+        using __stop_callback_t = stop_callback_for_t<__stop_token_t, __forward_stop_request<>>;
 
         if constexpr (std::same_as<__stop_token_t, inplace_stop_token>)
         {

include/exec/when_any.hpp

@@ -95,7 +95,7 @@ namespace experimental::execution
       {}
 
       using __on_stop =
-        stop_callback_for_t<stop_token_of_t<env_of_t<_Receiver>&>, __forward_stop_request>;
+        stop_callback_for_t<stop_token_of_t<env_of_t<_Receiver>&>, __forward_stop_request<>>;
 
       inplace_stop_source      __stop_source_{};
       std::optional<__on_stop> __on_stop_{};
@@ -260,7 +260,7 @@ namespace experimental::execution
                                               __copy_cvref_t<_Self, _Senders>...>)
         -> __opstate_t<_Self, _Receiver>
       {
-        return STDEXEC::__apply(STDEXEC::__construct<__opstate_t<_Self, _Receiver>>{},
+        return STDEXEC::__apply(STDEXEC::__construct_from<__opstate_t<_Self, _Receiver>>{},
                                 static_cast<_Self&&>(__self).__sndrs_,
                                 static_cast<_Receiver&&>(__rcvr));
       }

include/nvexec/stream/ensure_started.cuh

@@ -247,7 +247,7 @@ namespace nv::execution::_strm
       , public _strm::opstate_base<Receiver>
     {
       using on_stop_t = std::optional<
-        stop_callback_for_t<stop_token_of_t<env_of_t<Receiver>>, __forward_stop_request>>;
+        stop_callback_for_t<stop_token_of_t<env_of_t<Receiver>>, __forward_stop_request<>>>;
 
       on_stop_t                         on_stop_{};
       __intrusive_ptr<sh_state<Sender>> shared_state_;

include/nvexec/stream/split.cuh

@@ -246,7 +246,7 @@ namespace nv::execution::_strm
       , public _strm::opstate_base<Receiver>
     {
       using on_stop_t = std::optional<
-        stop_callback_for_t<stop_token_of_t<env_of_t<Receiver>>, __forward_stop_request>>;
+        stop_callback_for_t<stop_token_of_t<env_of_t<Receiver>>, __forward_stop_request<>>>;
 
       on_stop_t                         on_stop_{};
       std::shared_ptr<sh_state<Sender>> sh_state_;

include/nvexec/stream/when_all.cuh

@@ -301,7 +301,7 @@ namespace nv::execution::_strm
       using _indices_t          = __indices_for<Senders...>;
       using _stream_providers_t = std::array<stream_provider, sizeof...(Senders)>;
       using _stop_callback_t =
-        stop_callback_for_t<stop_token_of_t<env_of_t<_receiver_t>>, __forward_stop_request>;
+        stop_callback_for_t<stop_token_of_t<env_of_t<_receiver_t>>, __forward_stop_request<>>;
 
       template <class Sender, std::size_t Index>
       using _child_opstate_t =

include/stdexec/__detail/__as_awaitable.hpp

@@ -17,6 +17,7 @@
 
 #include "__execution_fwd.hpp"
 
+#include "../functional.hpp"
 #include "__atomic.hpp"
 #include "__awaitable.hpp"
 #include "__completion_signatures_of.hpp"
@@ -40,7 +41,10 @@ STDEXEC_PRAGMA_IGNORE_MSVC(4714)  // marked as __forceinline not inlined
 namespace STDEXEC
 {
 #if !STDEXEC_NO_STDCPP_COROUTINES()
-  namespace __detail
+  /////////////////////////////////////////////////////////////////////////////
+  // STDEXEC::as_awaitable [exec.as.awaitable]
+
+  namespace __as_awaitable
   {
     template <std::size_t _Count>
     extern __q<__decayed_std_tuple> const __as_single;
@@ -74,12 +78,7 @@ namespace STDEXEC
     template <class _Sender>
     using __adapted_sender_t =
       __remove_rvalue_reference_t<__call_result_t<__adapt_completion_t<_Sender>, _Sender>>;
-  }  // namespace __detail
 
-  /////////////////////////////////////////////////////////////////////////////
-  // STDEXEC::as_awaitable [exec.as.awaitable]
-  namespace __as_awaitable
-  {
     struct __void
     {};
 
@@ -90,6 +89,8 @@ namespace STDEXEC
     using __expected_t =
       std::variant<std::monostate, __value_or_void_t<_Value>, std::exception_ptr>;
 
+    using __connect_await::__has_as_awaitable_member;
+
     template <class _Tag, class _Sender, class... _Env>
     concept __completes_inline_for = __never_sends<_Tag, _Sender, _Env...>
                                   || STDEXEC::__completes_inline<_Tag, env_of_t<_Sender>, _Env...>;
@@ -278,19 +279,18 @@ namespace STDEXEC
     };
 
     template <class _Sender, class _Promise>
-    using __sync_receiver_t = __sync_receiver<_Promise, __detail::__value_t<_Sender, _Promise>>;
+    using __sync_receiver_t = __sync_receiver<_Promise, __value_t<_Sender, _Promise>>;
 
     template <class _Sender, class _Promise>
-    using __async_receiver_t = __async_receiver<_Promise, __detail::__value_t<_Sender, _Promise>>;
+    using __async_receiver_t = __async_receiver<_Promise, __value_t<_Sender, _Promise>>;
 
     //////////////////////////////////////////////////////////////////////////////////////
     // __sender_awaitable: awaitable type returned by as_awaitable when given a sender
     // that does not have an as_awaitable member function
-    template <class _Promise, class _Sender>
-    struct __sender_awaitable
-      : __sender_awaitable_base<__detail::__value_t<_Sender, _Promise>, false>
+    template <class _Promise, sender_in<env_of_t<_Promise&>> _Sender>
+    struct __sender_awaitable : __sender_awaitable_base<__value_t<_Sender, _Promise>, false>
     {
-      using __value_t = __detail::__value_t<_Sender, _Promise>;
+      using __value_t = __as_awaitable::__value_t<_Sender, _Promise>;
 
       constexpr explicit __sender_awaitable(_Sender&&                         __sndr,
                                             __std::coroutine_handle<_Promise> __hcoro)
@@ -350,12 +350,12 @@ namespace STDEXEC
 
     // When the sender is known to complete inline, we can connect and start the operation
     // in await_suspend.
-    template <class _Promise, class _Sender>
+    template <class _Promise, sender_in<env_of_t<_Promise&>> _Sender>
       requires __completes_inline<_Sender, env_of_t<_Promise&>>
     struct __sender_awaitable<_Promise, _Sender>
-      : __sender_awaitable_base<__detail::__value_t<_Sender, _Promise>, true>
+      : __sender_awaitable_base<__value_t<_Sender, _Promise>, true>
     {
-      using __value_t = __detail::__value_t<_Sender, _Promise>;
+      using __value_t = __as_awaitable::__value_t<_Sender, _Promise>;
 
       constexpr explicit __sender_awaitable(_Sender&&                         sndr,
                                             __std::coroutine_handle<_Promise> __hcoro)
@@ -404,16 +404,15 @@ namespace STDEXEC
 
     template <class _Sender, class _Promise>
     concept __awaitable_adapted_sender = sender_in<_Sender, env_of_t<_Promise&>>
-                                      && __minvocable_q<__detail::__value_t, _Sender, _Promise>
+                                      && __minvocable_q<__value_t, _Sender, _Promise>
                                       && requires(_Promise& __promise) {
                                            {
                                              __promise.unhandled_stopped()
                                            } -> __std::convertible_to<__std::coroutine_handle<>>;
                                          };
 
     template <class _Sender, class _Promise>
-    concept __awaitable_sender =
-      __awaitable_adapted_sender<__detail::__adapted_sender_t<_Sender>, _Promise>;
+    concept __awaitable_sender = __awaitable_adapted_sender<__adapted_sender_t<_Sender>, _Promise>;
 
     struct __unspecified
     {
@@ -426,92 +425,68 @@ namespace STDEXEC
     };
 
     template <class _Sender, class _Promise>
-    concept __incompatible_sender = sender<_Sender>
-                                 && __merror<__detail::__value_t<_Sender, _Promise>>;
-  }  // namespace __as_awaitable
+    concept __incompatible_sender = sender<_Sender> && __merror<__value_t<_Sender, _Promise>>;
 
-  struct as_awaitable_t
-  {
-    template <class _Tp, class _Promise>
-    static consteval auto __get_declfn() noexcept
-    {
-      using namespace __as_awaitable;
-      if constexpr (__connect_await::__has_as_awaitable_member<_Tp, _Promise>)
-      {
-        using __result_t = decltype(__declval<_Tp>().as_awaitable(__declval<_Promise&>()));
-        constexpr bool __is_nothrow = noexcept(
-          __declval<_Tp>().as_awaitable(__declval<_Promise&>()));
-        return __declfn<__result_t, __is_nothrow>();
-      }
-      else if constexpr (__awaitable<_Tp, __unspecified>)  // NOT __awaitable<_Tp, _Promise> !!
-      {                                                    // NOLINT(bugprone-branch-clone)
-        return __declfn<_Tp&&>();
-      }
-      else if constexpr (__awaitable_sender<_Tp, _Promise>)
-      {
-        using __result_t            = decltype(  //
-          __sender_awaitable{__detail::__adapt_sender_for_await(__declval<_Tp>()),
-                             __std::coroutine_handle<_Promise>()});
-        constexpr bool __is_nothrow = noexcept(
-          __sender_awaitable{__detail::__adapt_sender_for_await(__declval<_Tp>()),
-                             __std::coroutine_handle<_Promise>()});
-        return __declfn<__result_t, __is_nothrow>();
-      }
-      else if constexpr (__incompatible_sender<_Tp, _Promise>)
-      {
-        // NOT TO SPEC: It's a sender, but it isn't a sender in the current promise's
-        // environment, so we can return the error type that results from trying to
-        // compute the sender's value type:
-        return __declfn<__detail::__value_t<_Tp, _Promise>>();
-      }
-      else
-      {
-        return __declfn<_Tp&&>();
-      }
-    }
+    template <class _Sender, class _Promise>
+    concept __has_transform_as_awaitable_member =
+      sender_in<_Sender, env_of_t<_Promise>>
+      && __has_as_awaitable_member<transform_sender_result_t<_Sender, env_of_t<_Promise>>,
+                                   _Promise>;
 
-    template <class _Tp, class _Promise, auto _DeclFn = __get_declfn<_Tp, _Promise>()>
-      requires __callable<__mtypeof<_DeclFn>>
-    auto operator()(_Tp&& __t, _Promise& __promise) const noexcept(noexcept(_DeclFn()))
-      -> decltype(_DeclFn())
+    template <class _Sender, class _Promise>
+    concept __awaitable_transform_sender =  //
+      sender_in<_Sender, env_of_t<_Promise>>
+      && __awaitable_sender<transform_sender_result_t<_Sender, env_of_t<_Promise>>, _Promise>;
+
+    inline constexpr auto __with_member =  //
+      []<class _Promise, __has_as_awaitable_member<_Promise> _Tp>(_Tp&& __t, auto& __promise)
+        STDEXEC_AUTO_RETURN(static_cast<_Tp&&>(__t).as_awaitable(__promise));
+
+    inline constexpr auto __with_transform_member =  //
+      []<class _Promise, __has_transform_as_awaitable_member<_Promise> _Tp>(_Tp&&     __t,
+                                                                            _Promise& __promise)
+        STDEXEC_AUTO_RETURN(
+          STDEXEC::transform_sender(static_cast<_Tp&&>(__t), STDEXEC::get_env(__promise))
+            .as_awaitable(__promise));
+
+    inline constexpr auto __with_await =  //
+      []<__awaitable<__unspecified> _Tp>(_Tp&& __t, __ignore)
+        STDEXEC_AUTO_RETURN(static_cast<_Tp&&>(__t));
+
+    inline constexpr auto __with_sender =  //
+      []<class _Promise, __awaitable_transform_sender<_Promise> _Tp>(_Tp&& __t, _Promise& __promise)
+        STDEXEC_AUTO_RETURN(__sender_awaitable{
+          __as_awaitable::__adapt_sender_for_await(
+            STDEXEC::transform_sender(static_cast<_Tp&&>(__t), STDEXEC::get_env(__promise))),
+          __std::coroutine_handle<_Promise>::from_promise(__promise)});
+
+    // NOT TO SPEC: It's a sender, but it isn't a sender in the current promise's
+    // environment, so we can return the error type that results from trying to
+    // compute the sender's value type:
+    inline constexpr auto __with_incompatible_sender =  //
+      []<class _Promise, __incompatible_sender<_Promise> _Tp>(_Tp&&, _Promise&)
     {
-      using namespace __as_awaitable;
-      if constexpr (__connect_await::__has_as_awaitable_member<_Tp, _Promise>)
-      {
-        return static_cast<_Tp&&>(__t).as_awaitable(__promise);
-      }
-      else if constexpr (__awaitable<_Tp, __unspecified>)  // NOT __awaitable<_Tp, _Promise> !!
-      {                                                    // NOLINT(bugprone-branch-clone)
-        return static_cast<_Tp&&>(__t);
-      }
-      else if constexpr (__awaitable_sender<_Tp, _Promise>)
-      {
-        auto __hcoro = __std::coroutine_handle<_Promise>::from_promise(__promise);
-        return __sender_awaitable{__detail::__adapt_sender_for_await(static_cast<_Tp&&>(__t)),
-                                  __hcoro};
-      }
-      else if constexpr (__incompatible_sender<_Tp, _Promise>)
-      {
-        return __detail::__value_t<_Tp, _Promise>();
-      }
-      else
-      {
-        return static_cast<_Tp&&>(__t);
-      }
-    }
+      return __value_t<_Tp, _Promise>{};
+    };
 
-    template <class _Tp, class _Promise, auto _DeclFn = __get_declfn<_Tp, _Promise>()>
-      requires __callable<__mtypeof<_DeclFn>> || __tag_invocable<as_awaitable_t, _Tp, _Promise&>
-    [[deprecated("the use of tag_invoke for as_awaitable is deprecated")]]
-    auto operator()(_Tp&& __t, _Promise& __promise) const
-      noexcept(__nothrow_tag_invocable<as_awaitable_t, _Tp, _Promise&>)
-        -> __tag_invoke_result_t<as_awaitable_t, _Tp, _Promise&>
+    inline constexpr auto __identity =  //
+      []<class _Tp>(_Tp&& __t, __ignore) noexcept -> decltype(auto)
     {
-      using __result_t = __tag_invoke_result_t<as_awaitable_t, _Tp, _Promise&>;
-      static_assert(__awaitable<__result_t, _Promise>);
-      return __tag_invoke(*this, static_cast<_Tp&&>(__t), __promise);
-    }
-  };
+      return static_cast<_Tp&&>(__t);
+    };
+
+    inline constexpr auto __as_awaitable_impl =  //
+      __first_callable{__with_member,
+                       __with_transform_member,
+                       __with_await,
+                       __with_sender,
+                       __with_incompatible_sender,
+                       __identity};
+
+  }  // namespace __as_awaitable
+
+  struct as_awaitable_t : decltype(__as_awaitable::__as_awaitable_impl)
+  {};
 
   inline constexpr as_awaitable_t as_awaitable{};
 #endif

include/stdexec/__detail/__parallel_scheduler_backend.hpp

@@ -116,7 +116,7 @@ namespace STDEXEC
     template <class _Token>
     struct __stop_callback_for
     {
-      using __callback_t = stop_callback_for_t<_Token, __forward_stop_request>;
+      using __callback_t = stop_callback_for_t<_Token, __forward_stop_request<>>;
 
       bool __register_stop_callback(_Token __token)
       {