diff options
Diffstat (limited to 'vendor/rustix/src/thread/futex.rs')
| -rw-r--r-- | vendor/rustix/src/thread/futex.rs | 600 |
1 files changed, 600 insertions, 0 deletions
diff --git a/vendor/rustix/src/thread/futex.rs b/vendor/rustix/src/thread/futex.rs new file mode 100644 index 00000000..7ac49e69 --- /dev/null +++ b/vendor/rustix/src/thread/futex.rs @@ -0,0 +1,600 @@ +//! Linux `futex`. +//! +//! Futex is a very low-level mechanism for implementing concurrency primitives +//! such as mutexes, rwlocks, and condvars. For a higher-level API that +//! provides those abstractions, see [rustix-futex-syntax]. +//! +//! # Examples +//! +//! ``` +//! use rustix::thread::futex; +//! use std::sync::atomic::AtomicU32; +//! +//! # fn test(futex: &AtomicU32) -> rustix::io::Result<()> { +//! // Wake up one waiter. +//! futex::wake(futex, futex::Flags::PRIVATE, 1)?; +//! # Ok(()) +//! # } +//! ``` +//! +//! # References +//! - [Linux `futex` system call] +//! - [Linux `futex` feature] +//! +//! [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +//! [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +//! [rustix-futex-sync]: https://crates.io/crates/rustix-futex-sync +#![allow(unsafe_code)] + +use core::ffi::c_void; +use core::num::NonZeroU32; +use core::ptr; +use core::sync::atomic::AtomicU32; + +use crate::backend::thread::futex::Operation; +use crate::backend::thread::syscalls::{futex_timeout, futex_val2}; +use crate::fd::{FromRawFd as _, OwnedFd, RawFd}; +use crate::{backend, io}; + +pub use crate::clockid::ClockId; +pub use crate::timespec::{Nsecs, Secs, Timespec}; + +pub use backend::thread::futex::{Flags, WaitFlags, OWNER_DIED, WAITERS}; + +/// `syscall(SYS_futex, uaddr, FUTEX_WAIT, val, timeout, NULL, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wait( + uaddr: &AtomicU32, + flags: Flags, + val: u32, + timeout: Option<&Timespec>, +) -> io::Result<()> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_timeout(uaddr, Operation::Wait, flags, val, timeout, ptr::null(), 0).map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_WAKE, val, NULL, NULL, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wake(uaddr: &AtomicU32, flags: Flags, val: u32) -> io::Result<usize> { + // SAFETY: The raw pointers come from references or null. + unsafe { futex_val2(uaddr, Operation::Wake, flags, val, 0, ptr::null(), 0) } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_FD, val, NULL, NULL, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn fd(uaddr: &AtomicU32, flags: Flags, val: u32) -> io::Result<OwnedFd> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_val2(uaddr, Operation::Fd, flags, val, 0, ptr::null(), 0).map(|val| { + let fd = val as RawFd; + debug_assert_eq!(fd as usize, val, "return value should be a valid fd"); + OwnedFd::from_raw_fd(fd) + }) + } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_REQUEUE, val, val2, uaddr2, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn requeue( + uaddr: &AtomicU32, + flags: Flags, + val: u32, + val2: u32, + uaddr2: &AtomicU32, +) -> io::Result<usize> { + // SAFETY: The raw pointers come from references or null. + unsafe { futex_val2(uaddr, Operation::Requeue, flags, val, val2, uaddr2, 0) } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_CMP_REQUEUE, val, val2, uaddr2, val3)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn cmp_requeue( + uaddr: &AtomicU32, + flags: Flags, + val: u32, + val2: u32, + uaddr2: &AtomicU32, + val3: u32, +) -> io::Result<usize> { + // SAFETY: The raw pointers come from references or null. + unsafe { futex_val2(uaddr, Operation::CmpRequeue, flags, val, val2, uaddr2, val3) } +} + +/// `FUTEX_OP_*` operations for use with [`wake_op`]. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[repr(u32)] +#[allow(clippy::identity_op)] +pub enum WakeOp { + /// `FUTEX_OP_SET`: `uaddr2 = oparg;` + Set = 0, + /// `FUTEX_OP_ADD`: `uaddr2 += oparg;` + Add = 1, + /// `FUTEX_OP_OR`: `uaddr2 |= oparg;` + Or = 2, + /// `FUTEX_OP_ANDN`: `uaddr2 &= ~oparg;` + AndN = 3, + /// `FUTEX_OP_XOR`: `uaddr2 ^= oparg;` + XOr = 4, + /// `FUTEX_OP_SET | FUTEX_OP_ARG_SHIFT`: `uaddr2 = (oparg << 1);` + SetShift = 0 | 8, + /// `FUTEX_OP_ADD | FUTEX_OP_ARG_SHIFT`: `uaddr2 += (oparg << 1);` + AddShift = 1 | 8, + /// `FUTEX_OP_OR | FUTEX_OP_ARG_SHIFT`: `uaddr2 |= (oparg << 1);` + OrShift = 2 | 8, + /// `FUTEX_OP_ANDN | FUTEX_OP_ARG_SHIFT`: `uaddr2 &= !(oparg << 1);` + AndNShift = 3 | 8, + /// `FUTEX_OP_XOR | FUTEX_OP_ARG_SHIFT`: `uaddr2 ^= (oparg << 1);` + XOrShift = 4 | 8, +} + +/// `FUTEX_OP_CMP_*` operations for use with [`wake_op`]. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[repr(u32)] +pub enum WakeOpCmp { + /// `FUTEX_OP_CMP_EQ`: `if oldval == cmparg { wake(); }` + Eq = 0, + /// `FUTEX_OP_CMP_EQ`: `if oldval != cmparg { wake(); }` + Ne = 1, + /// `FUTEX_OP_CMP_EQ`: `if oldval < cmparg { wake(); }` + Lt = 2, + /// `FUTEX_OP_CMP_EQ`: `if oldval <= cmparg { wake(); }` + Le = 3, + /// `FUTEX_OP_CMP_EQ`: `if oldval > cmparg { wake(); }` + Gt = 4, + /// `FUTEX_OP_CMP_EQ`: `if oldval >= cmparg { wake(); }` + Ge = 5, +} + +/// `syscall(SYS_futex, uaddr, FUTEX_WAKE_OP, val, val2, uaddr2, val3)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +#[allow(clippy::too_many_arguments)] +pub fn wake_op( + uaddr: &AtomicU32, + flags: Flags, + val: u32, + val2: u32, + uaddr2: &AtomicU32, + op: WakeOp, + cmp: WakeOpCmp, + oparg: u16, + cmparg: u16, +) -> io::Result<usize> { + if oparg >= 1 << 12 || cmparg >= 1 << 12 { + return Err(io::Errno::INVAL); + } + + let val3 = + ((op as u32) << 28) | ((cmp as u32) << 24) | ((oparg as u32) << 12) | (cmparg as u32); + + // SAFETY: The raw pointers come from references or null. + unsafe { futex_val2(uaddr, Operation::WakeOp, flags, val, val2, uaddr2, val3) } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_LOCK_PI, 0, timeout, NULL, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn lock_pi(uaddr: &AtomicU32, flags: Flags, timeout: Option<&Timespec>) -> io::Result<()> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_timeout(uaddr, Operation::LockPi, flags, 0, timeout, ptr::null(), 0).map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_UNLOCK_PI, 0, NULL, NULL, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn unlock_pi(uaddr: &AtomicU32, flags: Flags) -> io::Result<()> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_val2(uaddr, Operation::UnlockPi, flags, 0, 0, ptr::null(), 0).map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_TRYLOCK_PI, 0, NULL, NULL, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn trylock_pi(uaddr: &AtomicU32, flags: Flags) -> io::Result<bool> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_val2(uaddr, Operation::TrylockPi, flags, 0, 0, ptr::null(), 0).map(|ret| ret == 0) + } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_WAIT_BITSET, val, timeout, NULL, val3)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wait_bitset( + uaddr: &AtomicU32, + flags: Flags, + val: u32, + timeout: Option<&Timespec>, + val3: NonZeroU32, +) -> io::Result<()> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_timeout( + uaddr, + Operation::WaitBitset, + flags, + val, + timeout, + ptr::null(), + val3.get(), + ) + .map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_WAKE_BITSET, val, NULL, NULL, val3)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wake_bitset( + uaddr: &AtomicU32, + flags: Flags, + val: u32, + val3: NonZeroU32, +) -> io::Result<usize> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_val2( + uaddr, + Operation::WakeBitset, + flags, + val, + 0, + ptr::null(), + val3.get(), + ) + } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_WAIT_REQUEUE_PI, val, timeout, uaddr2, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn wait_requeue_pi( + uaddr: &AtomicU32, + flags: Flags, + val: u32, + timeout: Option<&Timespec>, + uaddr2: &AtomicU32, +) -> io::Result<()> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_timeout( + uaddr, + Operation::WaitRequeuePi, + flags, + val, + timeout, + uaddr2, + 0, + ) + .map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_CMP_REQUEUE_PI, 1, val2, uaddr2, val3)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn cmp_requeue_pi( + uaddr: &AtomicU32, + flags: Flags, + val2: u32, + uaddr2: &AtomicU32, + val3: u32, +) -> io::Result<usize> { + // SAFETY: The raw pointers come from references or null. + unsafe { futex_val2(uaddr, Operation::CmpRequeuePi, flags, 1, val2, uaddr2, val3) } +} + +/// `syscall(SYS_futex, uaddr, FUTEX_LOCK_PI2, 0, timeout, NULL, 0)` +/// +/// This is a very low-level feature for implementing synchronization +/// primitives. See the references links. +/// +/// # References +/// - [Linux `futex` system call] +/// - [Linux `futex` feature] +/// +/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html +/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html +#[inline] +pub fn lock_pi2(uaddr: &AtomicU32, flags: Flags, timeout: Option<&Timespec>) -> io::Result<()> { + // SAFETY: The raw pointers come from references or null. + unsafe { + futex_timeout(uaddr, Operation::LockPi2, flags, 0, timeout, ptr::null(), 0).map(|val| { + debug_assert_eq!( + val, 0, + "The return value should always equal zero, if the call is successful" + ); + }) + } +} + +/// A pointer in the [`Wait`] struct. +#[repr(C)] +#[derive(Copy, Clone)] +#[non_exhaustive] +pub struct WaitPtr { + #[cfg(all(target_pointer_width = "32", target_endian = "big"))] + #[doc(hidden)] + pub __pad32: u32, + #[cfg(all(target_pointer_width = "16", target_endian = "big"))] + #[doc(hidden)] + pub __pad16: u16, + + /// The pointer value. + pub ptr: *mut c_void, + + #[cfg(all(target_pointer_width = "16", target_endian = "little"))] + #[doc(hidden)] + pub __pad16: u16, + #[cfg(all(target_pointer_width = "32", target_endian = "little"))] + #[doc(hidden)] + pub __pad32: u32, +} + +impl WaitPtr { + /// Construct a new `WaitPtr` holding the given raw pointer value. + #[inline] + pub const fn new(ptr: *mut c_void) -> Self { + Self { + ptr, + + #[cfg(target_pointer_width = "16")] + __pad16: 0, + #[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))] + __pad32: 0, + } + } +} + +impl Default for WaitPtr { + #[inline] + fn default() -> Self { + Self::new(ptr::null_mut()) + } +} + +impl From<*mut c_void> for WaitPtr { + #[inline] + fn from(ptr: *mut c_void) -> Self { + Self::new(ptr) + } +} + +impl core::fmt::Debug for WaitPtr { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.ptr.fmt(f) + } +} + +/// For use with [`waitv`]. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +#[non_exhaustive] +pub struct Wait { + /// The expected value. + pub val: u64, + /// The address to wait for. + pub uaddr: WaitPtr, + /// The type and size of futex to perform. + pub flags: WaitFlags, + + /// Reserved for future use. + pub(crate) __reserved: u32, +} + +impl Wait { + /// Construct a zero-initialized `Wait`. + #[inline] + pub const fn new() -> Self { + Self { + val: 0, + uaddr: WaitPtr::new(ptr::null_mut()), + flags: WaitFlags::empty(), + __reserved: 0, + } + } +} + +impl Default for Wait { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// `futex_waitv(waiters.as_ptr(), waiters.len(), flags, timeout, clockd)`— +/// Wait on an array of futexes, wake on any. +/// +/// This requires Linux ≥ 5.16. +/// +/// # References +/// - [Linux] +/// +/// [Linux]: https://www.kernel.org/doc/html/latest/userspace-api/futex2.html +#[inline] +pub fn waitv( + waiters: &[Wait], + flags: WaitvFlags, + timeout: Option<&Timespec>, + clockid: ClockId, +) -> io::Result<usize> { + backend::thread::syscalls::futex_waitv(waiters, flags, timeout, clockid) +} + +bitflags::bitflags! { + /// Flags for use with the flags argument in [`waitv`]. + /// + /// At this time, no flags are defined. + #[repr(transparent)] + #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] + pub struct WaitvFlags: u32 { + /// <https://docs.rs/bitflags/*/bitflags/#externally-defined-flags> + const _ = !0; + } +} + +#[cfg(linux_raw)] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_layouts() { + use crate::backend::c; + + check_renamed_struct!(Wait, futex_waitv, val, uaddr, flags, __reserved); + } +} |