diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b770c5ba6..9f1d275ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -987,6 +987,8 @@ jobs: # if: matrix.rust == 'stable' - run: tools/no-std.sh +esp xtensa-esp32s2-none-elf if: matrix.rust == 'stable' + - run: tools/no-std.sh +esp xtensa-esp32s3-none-elf + if: matrix.rust == 'stable' miri: needs: tidy diff --git a/build.rs b/build.rs index 03bd291fe..fbc0707e5 100644 --- a/build.rs +++ b/build.rs @@ -56,13 +56,17 @@ fn main() { // Custom cfgs set by build script. Not public API. // grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\)?".*$//' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' println!( - "cargo:rustc-check-cfg=cfg(portable_atomic_atomic_intrinsics,portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_strict_provenance,portable_atomic_no_strict_provenance_atomic_ptr,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_15,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_privileged,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)" + "cargo:rustc-check-cfg=cfg(portable_atomic_atomic_intrinsics,portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_outline_atomics,portable_atomic_no_strict_provenance,portable_atomic_no_strict_provenance_atomic_ptr,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_15,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_cpu,portable_atomic_target_feature,portable_atomic_unsafe_assume_privileged,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)" ); // TODO: handle multi-line target_feature_fallback // grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' println!( r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","lsfe","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","rmw","v6","v7","zaamo","zabha","zacas"))"# ); + // grep -F 'target_cpu_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_cpu_fallback\(//; s/"\).*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' + println!( + r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_cpu,values("esp32","esp32s3"))"# + ); } // https://github.com/rust-lang/rust/pull/123745 (includes https://github.com/rust-lang/cargo/pull/13560) merged in Rust 1.79 (nightly-2024-04-11). @@ -193,6 +197,16 @@ fn main() { println!("cargo:rustc-cfg=portable_atomic_unstable_asm_experimental_arch"); } } + "xtensa" => { + // https://github.com/rust-lang/rust/pull/93868 merged in Rust 1.60 (nightly-2022-02-13). + if is_allowed_feature("asm_experimental_arch") { + println!("cargo:rustc-cfg=portable_atomic_unstable_asm_experimental_arch"); + } + // Between 1.90 and 1.94 some register clobbers were not defined correctly, so asm implementation cannot be used. + if version.probe(91, 2025, 8, 30) && !version.probe(94, 2026, 3, 6) { + println!("cargo:rustc-cfg=portable_atomic_no_asm"); + } + } _ => {} } } @@ -502,6 +516,27 @@ fn main() { } target_feature_fallback("rmw", xmegau); } + "xtensa" => { + // Some Xtensa CPUs have CAS for internal memory, but also have an external address space, + // which does not correctly provide atomic access. The affected CPUs have their own build targets, + // but may also be specified with `-C target-cpu`. + // Xtensa targets have been introduced in Rust 1.81.0. + if let Some(cpu) = target_cpu() { + if cpu == "esp32" || cpu == "esp32s3" { + target_cpu_fallback(&cpu); + } + } else { + match target { + "xtensa-esp32-none-elf" | "xtensa-esp32-espidf" => target_cpu_fallback("esp32"), + "xtensa-esp32s3-none-elf" | "xtensa-esp32s3-espidf" => { + target_cpu_fallback("esp32s3") + } + // ESP32-S2 does not have atomic CAS, so it is not affected by the issue the same way. + // For other Xtensa CPUs, assume they are not affected. + _ => {} + } + } + } _ => {} } } @@ -560,6 +595,11 @@ fn target_cpu() -> Option { cpu.map(str::to_owned) } +// `target_cpu` is not a valid cfg option. Where there is absolutely no other option, inject a cfg fallback. +fn target_cpu_fallback(cpu: &str) { + println!("cargo:rustc-cfg=portable_atomic_target_cpu=\"{}\"", cpu); +} + fn is_allowed_feature(name: &str) -> bool { // https://github.com/dtolnay/thiserror/pull/248 if env::var_os("RUSTC_STAGE").is_some() { diff --git a/src/imp/interrupt/mod.rs b/src/imp/interrupt/mod.rs index 16631c250..78f21d48a 100644 --- a/src/imp/interrupt/mod.rs +++ b/src/imp/interrupt/mod.rs @@ -9,7 +9,11 @@ Fallback implementation based on disabling interrupts or critical-section See README.md of this directory for details. */ -#[cfg(not(feature = "critical-section"))] +#[cfg(any( + not(feature = "critical-section"), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", +))] #[cfg_attr( all( target_arch = "arm", @@ -30,13 +34,26 @@ See README.md of this directory for details. #[cfg_attr(target_arch = "xtensa", path = "xtensa.rs")] pub(super) mod arch; +// ESP32 and ESP32-S3 require a critical-section based implementation for some of their address spaces. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, - cfg(any(test, portable_atomic_no_atomic_cas, portable_atomic_unsafe_assume_single_core)) + cfg(any( + test, + portable_atomic_no_atomic_cas, + portable_atomic_unsafe_assume_single_core, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", + )) )] #[cfg_attr( not(portable_atomic_no_cfg_target_has_atomic), - cfg(any(test, not(target_has_atomic = "ptr"), portable_atomic_unsafe_assume_single_core)) + cfg(any( + test, + not(target_has_atomic = "ptr"), + portable_atomic_unsafe_assume_single_core, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", + )) )] items!({ use core::{cell::UnsafeCell, sync::atomic::Ordering}; @@ -243,6 +260,8 @@ items!({ test, target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", not(target_has_atomic = "ptr") )) )] @@ -616,6 +635,46 @@ items!({ } }); + // Current target data unsoundly enables CAS for these devices. Generate the necessary + // critical-section based implementations that we'll wrap in `imp/xtensa.rs`. + #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(all()))] + #[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(all( + target_has_atomic, + any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3") + )) + )] + items!({ + use self::arch::atomic; + + atomic_int!(load_store_atomic, AtomicIsize, isize, 4); + atomic_int!(load_store_atomic, AtomicUsize, usize, 4); + + atomic_int!(load_store_atomic[sub_word], AtomicI8, i8, 1); + atomic_int!(load_store_atomic[sub_word], AtomicU8, u8, 1); + + atomic_int!(load_store_atomic[sub_word], AtomicI16, i16, 2); + atomic_int!(load_store_atomic[sub_word], AtomicU16, u16, 2); + + atomic_int!(load_store_atomic, AtomicI32, i32, 4); + atomic_int!(load_store_atomic, AtomicU32, u32, 4); + + atomic_base!(native_load_store, [T] AtomicPtr, *mut T); + impl AtomicPtr { + #[cfg(test)] + #[inline] + fn fetch_ptr_add(&self, val: usize, order: Ordering) -> *mut T { + self.fetch_byte_add(val.wrapping_mul(core::mem::size_of::()), order) + } + #[cfg(test)] + #[inline] + fn fetch_ptr_sub(&self, val: usize, order: Ordering) -> *mut T { + self.fetch_byte_sub(val.wrapping_mul(core::mem::size_of::()), order) + } + } + }); + // Double or more width atomics (require fallback feature for consistency with other situations). #[cfg(target_pointer_width = "16")] #[cfg(any(test, feature = "fallback"))] diff --git a/src/imp/interrupt/xtensa.rs b/src/imp/interrupt/xtensa.rs index da400cdd1..507109238 100644 --- a/src/imp/interrupt/xtensa.rs +++ b/src/imp/interrupt/xtensa.rs @@ -15,10 +15,20 @@ use core::arch::asm; )] #[cfg_attr( not(portable_atomic_no_cfg_target_has_atomic), - cfg(any(test, not(target_has_atomic = "ptr"))) + cfg(any( + test, + not(any( + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", + target_has_atomic = "ptr" + )) + )) )] pub(super) use core::sync::atomic; +#[cfg(any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3"))] +pub(super) use crate::imp::xtensa as atomic; + pub(crate) type State = u32; /// Disables interrupts and returns the previous interrupt state. diff --git a/src/imp/mod.rs b/src/imp/mod.rs index 54fbd1555..c9e6d6b39 100644 --- a/src/imp/mod.rs +++ b/src/imp/mod.rs @@ -35,6 +35,10 @@ not(target_has_atomic = "ptr"), ))) )] +#[cfg(all( + target_arch = "xtensa", + any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3",) +))] mod core_atomic; // AVR @@ -47,6 +51,10 @@ mod avr; #[cfg(target_arch = "msp430")] pub(crate) mod msp430; +// Xtensas with address ranges that do not support atomic operations +#[cfg(any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3"))] +pub(crate) mod xtensa; + // RISC-V without A-extension #[cfg(any(test, not(feature = "critical-section")))] #[cfg_attr( @@ -97,6 +105,8 @@ mod atomic128; #[cfg(not(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", portable_atomic_unsafe_assume_single_core, )))] #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] @@ -161,10 +171,13 @@ mod fallback; // On AVR, we always use critical section based fallback implementation. // AVR can be safely assumed to be single-core, so this is sound. // MSP430 as well. +// On ESP32, we always use critical section based fallback implementation. // See the module-level comments of interrupt module for more. #[cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", feature = "critical-section", portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged, @@ -176,6 +189,8 @@ mod fallback; portable_atomic_no_atomic_cas, portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", )) )] #[cfg_attr( @@ -185,6 +200,8 @@ mod fallback; not(target_has_atomic = "ptr"), portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", )) )] #[cfg(any( @@ -208,7 +225,7 @@ pub(crate) mod float; // {8,16,32}-bit & ptr-sized atomics cfg_sel!({ - // has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic + // has CAS | (has core atomic & !(avr | msp430 | critical section | esp32)) => core atomic #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(not(any( @@ -224,6 +241,8 @@ cfg_sel!({ ), portable_atomic_no_atomic_cas, ), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", ))) )] #[cfg_attr( @@ -241,6 +260,8 @@ cfg_sel!({ ), not(target_has_atomic = "ptr"), ), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", ))) )] { @@ -253,7 +274,13 @@ cfg_sel!({ #[cfg(any( target_arch = "avr", target_arch = "msp430", - feature = "critical-section", + all( + not(any( + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3" + )), + feature = "critical-section", + ), portable_atomic_unsafe_assume_single_core, ))] { @@ -271,6 +298,14 @@ cfg_sel!({ AtomicUsize, }; } + // Xtensa with quirky external data bus address range + #[cfg(any(portable_atomic_target_cpu = "esp32", portable_atomic_target_cpu = "esp32s3"))] + { + pub(crate) use self::xtensa::{ + AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32, + AtomicUsize, + }; + } // bpf & !(critical section) => core atomic #[cfg(target_arch = "bpf")] { @@ -284,7 +319,7 @@ cfg_sel!({ // 64-bit atomics cfg_sel!({ - // has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic + // (has CAS & !esp32) | (has core atomic & !(avr | msp430 | critical section)) => core atomic #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(all( @@ -300,7 +335,9 @@ cfg_sel!({ portable_atomic_unsafe_assume_single_core, ), portable_atomic_no_atomic_cas, - ) + ), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", )), any( not(portable_atomic_no_atomic_64), @@ -323,7 +360,9 @@ cfg_sel!({ portable_atomic_unsafe_assume_single_core, ), not(target_has_atomic = "ptr"), - ) + ), + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", )), any( target_has_atomic = "64", @@ -365,12 +404,14 @@ cfg_sel!({ { pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64}; } - // no native atomic CAS & (assume single core | critical section) => critical section based fallback + // (no native atomic CAS | esp32) & (assume single core | critical section) => critical section based fallback #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", all(feature = "critical-section", portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core, )) @@ -380,6 +421,8 @@ cfg_sel!({ cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", all(feature = "critical-section", not(target_has_atomic = "ptr")), portable_atomic_unsafe_assume_single_core, )) @@ -548,12 +591,14 @@ cfg_sel!({ { pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128}; } - // no native atomic CAS & (assume single core | critical section) => critical section based fallback + // (no native atomic CAS | esp32) & (assume single core | critical section) => critical section based fallback #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", all(feature = "critical-section", portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core, )) @@ -563,6 +608,8 @@ cfg_sel!({ cfg(any( target_arch = "avr", target_arch = "msp430", + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", all(feature = "critical-section", not(target_has_atomic = "ptr")), portable_atomic_unsafe_assume_single_core, )) diff --git a/src/imp/xtensa.rs b/src/imp/xtensa.rs new file mode 100644 index 000000000..1f3fb50ab --- /dev/null +++ b/src/imp/xtensa.rs @@ -0,0 +1,267 @@ +//! Implementations for Xtensa CPUs that have atomic CAS, but also have an address range where it is not working properly. +//! +//! For these CPUs rustc may, but should not provide atomic access unconditionally. portable-atomic generates code which selects +//! the appropriate implementation based on the address of the atomic variable. +use core::{ops::Range, sync::atomic::Ordering}; + +// https://documentation.espressif.com/esp32_technical_reference_manual_en.pdf, Table 3.3-4. External Memory Address Mapping +#[cfg(portable_atomic_target_cpu = "esp32")] +const EXTERNAL_DATA_BUS_ADDRESS_RANGE: Range = 0x3F80_0000..0x3FC0_0000; + +// https://documentation.espressif.com/esp32-s3_technical_reference_manual_en.pdf, Table 4.3-2. External Memory Address Mapping +#[cfg(portable_atomic_target_cpu = "esp32s3")] +const EXTERNAL_DATA_BUS_ADDRESS_RANGE: Range = 0x3C00_0000..0x3E00_0000; + +macro_rules! dispatch_impl { + ($self:expr, $([$($generics:tt)*])? $atomic_type:ident, self.v.$($fallback:tt)*) => { + { + let addr = $self.as_ptr() as usize; + if addr >= EXTERNAL_DATA_BUS_ADDRESS_RANGE.start && addr < EXTERNAL_DATA_BUS_ADDRESS_RANGE.end { + return $self.v.$($fallback)*; + } else { + let atomic = unsafe { ::core::mem::transmute::<&Self, &crate::imp::core_atomic::$atomic_type $(<$($generics)*>)?>($self) }; + return atomic.$($fallback)*; + } + } + }; +} + +macro_rules! atomic { + (base, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + #[repr(transparent)] + pub(crate) struct $atomic_type $(<$($generics)*>)? { + v: crate::imp::interrupt::$atomic_type $(<$($generics)*>)?, + } + + impl_default_bit_opts!($atomic_type, $int_type); + + // Send is implicitly implemented for atomic integers, but not for atomic pointers. + // SAFETY: any data races are prevented by atomic operations. + unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {} + // SAFETY: any data races are prevented by atomic operations. + unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {} + + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) const fn new(v: $value_type) -> Self { + Self { v: crate::imp::interrupt::$atomic_type::new(v) } + } + + #[inline] + pub(crate) const fn as_ptr(&self) -> *mut $value_type { + self.v.as_ptr() + } + + #[inline] + pub(crate) fn is_lock_free() -> bool { + )?>::is_lock_free() + } + pub(crate) const IS_ALWAYS_LOCK_FREE: bool = )?>::IS_ALWAYS_LOCK_FREE; + + #[inline] + pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.swap(val, order) + ) + } + + #[inline] + pub(crate) fn compare_exchange( + &self, + current: $value_type, + new: $value_type, + success: Ordering, + failure: Ordering, + ) -> Result<$value_type, $value_type> { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.compare_exchange(current, new, success, failure) + ) + } + + #[inline] + pub(crate) fn compare_exchange_weak( + &self, + current: $value_type, + new: $value_type, + success: Ordering, + failure: Ordering, + ) -> Result<$value_type, $value_type> { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.compare_exchange_weak(current, new, success, failure) + ) + } + } + }; + (load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(base, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + #[cfg_attr( + all(debug_assertions, not(portable_atomic_no_track_caller)), + track_caller + )] + pub(crate) fn load(&self, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.load(order) + ) + } + + #[inline] + #[cfg_attr( + all(debug_assertions, not(portable_atomic_no_track_caller)), + track_caller + )] + pub(crate) fn store(&self, val: $value_type, order: Ordering) { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.store(val, order) + ) + } + } + }; + (boolean_fetch, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(load_store, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_and(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_or(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_xor(val, order) + ) + } + } + }; + (boolean, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(boolean_fetch, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) fn and(&self, val: $int_type, order: Ordering) { + self.fetch_and(val, order); + } + + #[inline] + pub(crate) fn or(&self, val: $int_type, order: Ordering) { + self.fetch_or(val, order); + } + + #[inline] + pub(crate) fn xor(&self, val: $int_type, order: Ordering) { + self.fetch_xor(val, order); + } + + #[inline] + pub(crate) fn not(&self, order: Ordering) { + self.fetch_not(order); + } + + #[inline] + pub(crate) fn neg(&self, order: Ordering) { + self.fetch_neg(order); + } + + #[inline] + pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_not(order) + ) + } + + #[inline] + pub(crate) fn fetch_neg(&self, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_neg(order) + ) + } + + #[inline] + pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_nand(val, order) + ) + } + } + }; + ($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt) => { + atomic!(boolean, $([$($generics)*])? $atomic_type, $value_type, $int_type, $size); + impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? { + #[inline] + pub(crate) fn add(&self, val: $value_type, order: Ordering) { + self.fetch_add(val, order); + } + + #[inline] + pub(crate) fn sub(&self, val: $value_type, order: Ordering) { + self.fetch_sub(val, order); + } + + #[inline] + pub(crate) fn fetch_sub(&self, val: $value_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_sub(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_add(&self, val: $value_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_add(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_min(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $value_type { + dispatch_impl!(self, $([$($generics)*])? $atomic_type, + self.v.fetch_max(val, order) + ) + } + } + }; +} + +atomic!(AtomicI8, i8, i8, "b"); +atomic!(AtomicU8, u8, u8, "b"); +atomic!(AtomicI16, i16, i16, "h"); +atomic!(AtomicU16, u16, u16, "h"); +atomic!(AtomicI32, i32, i32, "w"); +atomic!(AtomicU32, u32, u32, "w"); +atomic!(AtomicIsize, isize, isize, "w"); +atomic!(AtomicUsize, usize, usize, "w"); +atomic!(boolean_fetch, [T] AtomicPtr, *mut T, usize, "w"); + +impl AtomicPtr { + #[inline] + pub(crate) fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T { + dispatch_impl!(self, [T] AtomicPtr, + self.v.fetch_byte_sub(val, order) + ) + } + + #[inline] + pub(crate) fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T { + dispatch_impl!(self, [T] AtomicPtr, + self.v.fetch_byte_add(val, order) + ) + } +} + +#[cfg(feature = "fallback")] +pub(crate) use crate::imp::interrupt::{AtomicI64, AtomicU64}; diff --git a/src/lib.rs b/src/lib.rs index 3dfdaa8e3..dbf9a30dc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -310,6 +310,8 @@ See also the [`interrupt` module's readme](https://github.com/taiki-e/portable-a any( portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged, + portable_atomic_target_cpu = "esp32", + portable_atomic_target_cpu = "esp32s3", ), ), all(target_arch = "powerpc64", portable_atomic_unstable_asm_experimental_arch), @@ -348,7 +350,7 @@ See also the [`interrupt` module's readme](https://github.com/taiki-e/portable-a #![cfg_attr( all( portable_atomic_unstable_asm_experimental_arch, - any(target_arch = "arm64ec", target_arch = "s390x"), + any(target_arch = "arm64ec", target_arch = "s390x", target_arch = "xtensa"), ), feature(asm_experimental_arch) )] diff --git a/src/utils.rs b/src/utils.rs index f53afe584..d7c513a60 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -356,7 +356,7 @@ pub(crate) fn unlikely(b: bool) -> bool { } // Equivalent to core::hint::assert_unchecked, but compatible with pre-1.81 rustc. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "xtensa"))] #[allow(dead_code)] #[inline(always)] #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] @@ -507,16 +507,16 @@ pub(crate) struct Pair { pub(crate) lo: T, } -#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "xtensa"))] type MinWord = u32; -#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "xtensa"))] type RetInt = u32; // Adapted from https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.6/src/utils.rs#L255. // Helper for implementing sub-word atomic operations using word-sized LL/SC loop or CAS loop. // // Refs: https://github.com/llvm/llvm-project/blob/llvmorg-21.1.0/llvm/lib/CodeGen/AtomicExpandPass.cpp#L812 // (aligned_ptr, shift, mask) -#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "xtensa"))] #[allow(dead_code)] #[inline] pub(crate) fn create_sub_word_mask_values(ptr: *mut T) -> (*mut MinWord, RetInt, RetInt) { diff --git a/tests/xtensa/Cargo.toml b/tests/xtensa/Cargo.toml index 4e18d7745..1ab0e18ff 100644 --- a/tests/xtensa/Cargo.toml +++ b/tests/xtensa/Cargo.toml @@ -14,6 +14,11 @@ paste = "1" esp-println = { default-features = false, features = ["uart", "esp32s2"], git = "https://github.com/taiki-e/esp-hal.git", rev = "293a19f" } esp-hal = { features = ["esp32s2"], git = "https://github.com/taiki-e/esp-hal.git", rev = "293a19f" } +[target.xtensa-esp32s3-none-elf.dependencies] +esp-println = { default-features = false, features = ["uart", "esp32s3"], git = "https://github.com/taiki-e/esp-hal.git", rev = "293a19f" } +esp-hal = { features = ["esp32s3"], git = "https://github.com/taiki-e/esp-hal.git", rev = "293a19f" } +portable-atomic = { path = "../..", features = ["critical-section"] } + [lints] workspace = true diff --git a/tools/no-std.sh b/tools/no-std.sh index 1f9ec4650..ee9a8f261 100755 --- a/tools/no-std.sh +++ b/tools/no-std.sh @@ -209,6 +209,9 @@ run() { armv[4-5]t* | thumbv[4-5]t* | thumbv6m* | xtensa-esp32s2-*) target_rustflags+=" --cfg portable_atomic_unsafe_assume_single_core" ;; + xtensa-esp32*) + # At this time, these chips require critical-section to be enabled, which is incompatible with the single-core and privileged assumptions. + ;; arm* | thumb* | xtensa*) assume_single_core_target_rustflags+=" --cfg portable_atomic_unsafe_assume_single_core" assume_privileged_target_rustflags+=" --cfg portable_atomic_unsafe_assume_privileged"