Rollup merge of #150436 - va-list-copy, r=workingjubilee,RalfJung

`c_variadic`: impl `va_copy` and `va_end` as Rust intrinsics

tracking issue: #44930

Implement `va_copy` as (the rust equivalent of) `memcpy`, which is the behavior of all current LLVM targets. By providing our own implementation, we can guarantee its behavior. These guarantees are important for implementing c-variadics in e.g. const-eval.

Discussed in [#t-compiler/const-eval > c-variadics in const-eval](https://rust-lang.zulipchat.com/#narrow/channel/146212-t-compiler.2Fconst-eval/topic/c-variadics.20in.20const-eval/with/565509704).

I've also updated the comment for `Drop` a bit. The background here is that the C standard requires that `va_end` is used in the same function (and really, in the same scope) as the corresponding `va_start` or `va_copy`. That is because historically `va_start` would start a scope, which `va_end` would then close. e.g.

https://softwarepreservation.computerhistory.org/c_plus_plus/cfront/release_3.0.3/source/incl-master/proto-headers/stdarg.sol

```c
#define         va_start(ap, parmN)     {\
        va_buf  _va;\
        _vastart(ap = (va_list)_va, (char *)&parmN + sizeof parmN)
#define         va_end(ap)      }
#define         va_arg(ap, mode)        *((mode *)_vaarg(ap, sizeof (mode)))
```

The C standard still has to consider such implementations, but for Rust they are irrelevant. Hence we can use `Clone` for `va_copy` and `Drop` for `va_end`.

Author: jhpratt

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -1506,7 +1506,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
         }
 
         // FIXME implement variadics in cranelift
-        sym::va_copy | sym::va_arg | sym::va_end => {
+        sym::va_arg | sym::va_end => {
             fx.tcx.dcx().span_fatal(
                 source_info.span,
                 "Defining variadic functions is not yet supported by Cranelift",

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -391,9 +391,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             sym::breakpoint => {
                 unimplemented!();
             }
-            sym::va_copy => {
-                unimplemented!();
-            }
             sym::va_arg => {
                 unimplemented!();
             }

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -269,14 +269,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 return Ok(());
             }
             sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]),
-            sym::va_copy => {
-                let dest = args[0].immediate();
-                self.call_intrinsic(
-                    "llvm.va_copy",
-                    &[self.val_ty(dest)],
-                    &[dest, args[1].immediate()],
-                )
-            }
             sym::va_arg => {
                 match result.layout.backend_repr {
                     BackendRepr::Scalar(scalar) => {

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -216,6 +216,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi
         | sym::type_name
         | sym::type_of
         | sym::ub_checks
+        | sym::va_copy
         | sym::variant_count
         | sym::vtable_for
         | sym::wrapping_add
@@ -629,14 +630,13 @@ pub(crate) fn check_intrinsic_type(
             )
         }
 
-        sym::va_start | sym::va_end => {
-            (0, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], tcx.types.unit)
-        }
-
         sym::va_copy => {
             let (va_list_ref_ty, va_list_ty) = mk_va_list_ty(hir::Mutability::Not);
-            let va_list_ptr_ty = Ty::new_mut_ptr(tcx, va_list_ty);
-            (0, 0, vec![va_list_ptr_ty, va_list_ref_ty], tcx.types.unit)
+            (0, 0, vec![va_list_ref_ty], va_list_ty)
+        }
+
+        sym::va_start | sym::va_end => {
+            (0, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], tcx.types.unit)
         }
 
         sym::va_arg => (1, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], param(0)),

library/core/src/ffi/va_list.rs

@@ -5,7 +5,7 @@
 #[cfg(not(target_arch = "xtensa"))]
 use crate::ffi::c_void;
 use crate::fmt;
-use crate::intrinsics::{va_arg, va_copy};
+use crate::intrinsics::{va_arg, va_copy, va_end};
 use crate::marker::PhantomCovariantLifetime;
 
 // There are currently three flavors of how a C `va_list` is implemented for
@@ -34,6 +34,10 @@ use crate::marker::PhantomCovariantLifetime;
 //
 // The Clang `BuiltinVaListKind` enumerates the `va_list` variations that Clang supports,
 // and we mirror these here.
+//
+// For all current LLVM targets, `va_copy` lowers to `memcpy`. Hence the inner structs below all
+// derive `Copy`. However, in the future we might want to support a target where `va_copy`
+// allocates, or otherwise violates the requirements of `Copy`. Therefore `VaList` is only `Clone`.
 crate::cfg_select! {
     all(
         target_arch = "aarch64",
@@ -45,10 +49,12 @@ crate::cfg_select! {
         ///
         /// See the [AArch64 Procedure Call Standard] for more details.
         ///
+        /// `va_copy` is `memcpy`: <https://github.com/llvm/llvm-project/blob/5aee01a3df011e660f26660bc30a8c94a1651d8e/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp#L12682-L12700>
+        ///
         /// [AArch64 Procedure Call Standard]:
         /// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf
         #[repr(C)]
-        #[derive(Debug)]
+        #[derive(Debug, Clone, Copy)]
         struct VaListInner {
             stack: *const c_void,
             gr_top: *const c_void,
@@ -62,11 +68,13 @@ crate::cfg_select! {
         ///
         /// See the [LLVM source] and [GCC header] for more details.
         ///
+        /// `va_copy` is `memcpy`: <https://github.com/llvm/llvm-project/blob/5aee01a3df011e660f26660bc30a8c94a1651d8e/llvm/lib/Target/PowerPC/PPCISelLowering.cpp#L3755-L3764>
+        ///
         /// [LLVM source]:
         /// https://github.com/llvm/llvm-project/blob/af9a4263a1a209953a1d339ef781a954e31268ff/llvm/lib/Target/PowerPC/PPCISelLowering.cpp#L4089-L4111
         /// [GCC header]: https://web.mit.edu/darwin/src/modules/gcc/gcc/ginclude/va-ppc.h
         #[repr(C)]
-        #[derive(Debug)]
+        #[derive(Debug, Clone, Copy)]
         #[rustc_pass_indirectly_in_non_rustic_abis]
         struct VaListInner {
             gpr: u8,
@@ -81,10 +89,12 @@ crate::cfg_select! {
         ///
         /// See the [S/390x ELF Application Binary Interface Supplement] for more details.
         ///
+        /// `va_copy` is `memcpy`: <https://github.com/llvm/llvm-project/blob/5aee01a3df011e660f26660bc30a8c94a1651d8e/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp#L4457-L4472>
+        ///
         /// [S/390x ELF Application Binary Interface Supplement]:
         /// https://docs.google.com/gview?embedded=true&url=https://github.com/IBM/s390x-abi/releases/download/v1.7/lzsabi_s390x.pdf
         #[repr(C)]
-        #[derive(Debug)]
+        #[derive(Debug, Clone, Copy)]
         #[rustc_pass_indirectly_in_non_rustic_abis]
         struct VaListInner {
             gpr: i64,
@@ -98,10 +108,13 @@ crate::cfg_select! {
         ///
         /// See the [System V AMD64 ABI] for more details.
         ///
+        /// `va_copy` is `memcpy`: <https://github.com/llvm/llvm-project/blob/5aee01a3df011e660f26660bc30a8c94a1651d8e/llvm/lib/Target/X86/X86ISelLowering.cpp#26319>
+        /// (github won't render that file, look for `SDValue LowerVACOPY`)
+        ///
         /// [System V AMD64 ABI]:
         /// https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf
         #[repr(C)]
-        #[derive(Debug)]
+        #[derive(Debug, Clone, Copy)]
         #[rustc_pass_indirectly_in_non_rustic_abis]
         struct VaListInner {
             gp_offset: i32,
@@ -115,10 +128,12 @@ crate::cfg_select! {
         ///
         /// See the [LLVM source] for more details.
         ///
+        /// `va_copy` is `memcpy`: <https://github.com/llvm/llvm-project/blob/5aee01a3df011e660f26660bc30a8c94a1651d8e/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp#L1260>
+        ///
         /// [LLVM source]:
         /// https://github.com/llvm/llvm-project/blob/af9a4263a1a209953a1d339ef781a954e31268ff/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp#L1211-L1215
         #[repr(C)]
-        #[derive(Debug)]
+        #[derive(Debug, Clone, Copy)]
         #[rustc_pass_indirectly_in_non_rustic_abis]
         struct VaListInner {
             stk: *const i32,
@@ -132,10 +147,12 @@ crate::cfg_select! {
         ///
         /// See the [LLVM source] for more details. On bare metal Hexagon uses an opaque pointer.
         ///
+        /// `va_copy` is `memcpy`: <https://github.com/llvm/llvm-project/blob/5aee01a3df011e660f26660bc30a8c94a1651d8e/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp#L1087-L1102>
+        ///
         /// [LLVM source]:
         /// https://github.com/llvm/llvm-project/blob/0cdc1b6dd4a870fc41d4b15ad97e0001882aba58/clang/lib/CodeGen/Targets/Hexagon.cpp#L407-L417
         #[repr(C)]
-        #[derive(Debug)]
+        #[derive(Debug, Clone, Copy)]
         #[rustc_pass_indirectly_in_non_rustic_abis]
         struct VaListInner {
             __current_saved_reg_area_pointer: *const c_void,
@@ -156,8 +173,10 @@ crate::cfg_select! {
     // That pointer is probably just the next variadic argument on the caller's stack.
     _ => {
         /// Basic implementation of a `va_list`.
+        ///
+        /// `va_copy` is `memcpy`: <https://github.com/llvm/llvm-project/blob/87e8e7d8f0db53060ef2f6ef4ab612fc0f2b4490/llvm/lib/Transforms/IPO/ExpandVariadics.cpp#L127-L129>
         #[repr(transparent)]
-        #[derive(Debug)]
+        #[derive(Debug, Clone, Copy)]
         struct VaListInner {
             ptr: *const c_void,
         }
@@ -179,6 +198,31 @@ impl fmt::Debug for VaList<'_> {
     }
 }
 
+impl VaList<'_> {
+    // Helper used in the implementation of the `va_copy` intrinsic.
+    pub(crate) fn duplicate(&self) -> Self {
+        Self { inner: self.inner.clone(), _marker: self._marker }
+    }
+}
+
+impl Clone for VaList<'_> {
+    #[inline]
+    fn clone(&self) -> Self {
+        // We only implement Clone and not Copy because some future target might not be able to
+        // implement Copy (e.g. because it allocates). For the same reason we use an intrinsic
+        // to do the copying: the fact that on all current targets, this is just `memcpy`, is an implementation
+        // detail. The intrinsic lets Miri catch UB from code incorrectly relying on that implementation detail.
+        va_copy(self)
+    }
+}
+
+impl<'f> Drop for VaList<'f> {
+    fn drop(&mut self) {
+        // SAFETY: this variable argument list is being dropped, so won't be read from again.
+        unsafe { va_end(self) }
+    }
+}
+
 mod sealed {
     pub trait Sealed {}
 
@@ -253,26 +297,6 @@ impl<'f> VaList<'f> {
     }
 }
 
-impl<'f> Clone for VaList<'f> {
-    #[inline]
-    fn clone(&self) -> Self {
-        let mut dest = crate::mem::MaybeUninit::uninit();
-        // SAFETY: we write to the `MaybeUninit`, thus it is initialized and `assume_init` is legal.
-        unsafe {
-            va_copy(dest.as_mut_ptr(), self);
-            dest.assume_init()
-        }
-    }
-}
-
-impl<'f> Drop for VaList<'f> {
-    fn drop(&mut self) {
-        // Rust requires that not calling `va_end` on a `va_list` does not cause undefined behaviour
-        // (as it is safe to leak values). As `va_end` is a no-op on all current LLVM targets, this
-        // destructor is empty.
-    }
-}
-
 // Checks (via an assert in `compiler/rustc_ty_utils/src/abi.rs`) that the C ABI for the current
 // target correctly implements `rustc_pass_indirectly_in_non_rustic_abis`.
 const _: () = {

library/core/src/intrinsics/mod.rs

@@ -3451,19 +3451,6 @@ pub(crate) const fn miri_promise_symbolic_alignment(ptr: *const (), align: usize
     )
 }
 
-/// Copies the current location of arglist `src` to the arglist `dst`.
-///
-/// # Safety
-///
-/// You must check the following invariants before you call this function:
-///
-/// - `dest` must be non-null and point to valid, writable memory.
-/// - `dest` must not alias `src`.
-///
-#[rustc_intrinsic]
-#[rustc_nounwind]
-pub unsafe fn va_copy<'f>(dest: *mut VaList<'f>, src: &VaList<'f>);
-
 /// Loads an argument of type `T` from the `va_list` `ap` and increment the
 /// argument `ap` points to.
 ///
@@ -3482,12 +3469,35 @@ pub unsafe fn va_copy<'f>(dest: *mut VaList<'f>, src: &VaList<'f>);
 #[rustc_nounwind]
 pub unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
 
-/// Destroy the arglist `ap` after initialization with `va_start` or `va_copy`.
+/// Duplicates a variable argument list. The returned list is initially at the same position as
+/// the one in `src`, but can be advanced independently.
+///
+/// Codegen backends should not have custom behavior for this intrinsic, they should always use
+/// this fallback implementation. This intrinsic *does not* map to the LLVM `va_copy` intrinsic.
+///
+/// This intrinsic exists only as a hook for Miri and constant evaluation, and is used to detect UB
+/// when a variable argument list is used incorrectly.
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub fn va_copy<'f>(src: &VaList<'f>) -> VaList<'f> {
+    src.duplicate()
+}
+
+/// Destroy the variable argument list `ap` after initialization with `va_start` (part of the
+/// desugaring of `...`) or `va_copy`.
+///
+/// Code generation backends should not provide a custom implementation for this intrinsic. This
+/// intrinsic *does not* map to the LLVM `va_end` intrinsic.
+///
+/// This function is a no-op on all current targets, but used as a hook for const evaluation to
+/// detect UB when a variable argument list is used incorrectly.
 ///
 /// # Safety
 ///
 /// `ap` must not be used to access variable arguments after this call.
 ///
 #[rustc_intrinsic]
 #[rustc_nounwind]
-pub unsafe fn va_end(ap: &mut VaList<'_>);
+pub unsafe fn va_end(ap: &mut VaList<'_>) {
+    /* deliberately does nothing */
+}

tests/codegen-llvm/cffi/c-variadic-copy.rs

@@ -17,14 +17,3 @@ pub unsafe extern "C" fn c_variadic_no_use(fmt: *const i8, mut ap: ...) -> i32 {
     vprintf(fmt, ap)
     // CHECK: call void @llvm.va_end
 }
-
-// Check that `VaList::clone` gets inlined into a direct call to `llvm.va_copy`
-#[no_mangle]
-pub unsafe extern "C" fn c_variadic_clone(fmt: *const i8, mut ap: ...) -> i32 {
-    // CHECK: call void @llvm.va_start
-    let mut ap2 = ap.clone();
-    // CHECK: call void @llvm.va_copy
-    let res = vprintf(fmt, ap2);
-    res
-    // CHECK: call void @llvm.va_end
-}

tests/codegen-llvm/cffi/c-variadic-opt.rs

@@ -0,0 +1,24 @@
+//@ run-pass
+//@ ignore-backends: gcc
+#![feature(c_variadic)]
+
+// Test the behavior of `VaList::clone`. In C a `va_list` is duplicated using `va_copy`, but the
+// rust api just uses `Clone`. This should create a completely independent cursor into the
+// variable argument list: advancing the original has no effect on the copy and vice versa.
+
+fn main() {
+    unsafe { variadic(1, 2, 3) }
+}
+
+unsafe extern "C" fn variadic(mut ap1: ...) {
+    let mut ap2 = ap1.clone();
+
+    assert_eq!(ap1.arg::<i32>(), 1);
+    assert_eq!(ap2.arg::<i32>(), 1);
+
+    assert_eq!(ap2.arg::<i32>(), 2);
+    assert_eq!(ap1.arg::<i32>(), 2);
+
+    drop(ap1);
+    assert_eq!(ap2.arg::<i32>(), 3);
+}