Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -779,6 +779,38 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
}
}

sym::cold_path => {
// Emit an empty inline asm with a `cold` call-site attribute.
// This causes LLVM's BranchProbabilityInfo to mark this block
// as unlikely. Unlike `llvm.expect`-based `!prof` metadata
// (which can be lost when the InlinerPass folds branches),
// this marker survives the full optimization pipeline.
let void_ty = self.cx.type_void();
let fty = self.cx.type_func(&[], void_ty);
let asm_val = unsafe {
llvm::LLVMGetInlineAsm(
fty,
c"".as_ptr().cast(),
0,
c"".as_ptr().cast(),
0,
llvm::TRUE, // HasSideEffects
llvm::FALSE, // IsAlignStack
llvm::AsmDialect::Att,
llvm::FALSE, // CanThrow
)
};
let call = self.call(fty, None, None, asm_val, &[], None, None);
let cold_attr =
llvm::AttributeKind::Cold.create_attr(self.llcx);
crate::attributes::apply_to_callsite(
call,
llvm::AttributePlace::Function,
&[cold_attr],
);
return Ok(());
}

_ => {
debug!("unknown intrinsic '{}' -- falling back to default body", name);
// Call the fallback body instead of generating the intrinsic code
Expand Down
5 changes: 0 additions & 5 deletions compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -579,11 +579,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}

sym::cold_path => {
// This is a no-op. The intrinsic is just a hint to the optimizer.
return Ok(());
}
Comment on lines -582 to -585
Copy link
Copy Markdown
Member

@jieyouxu jieyouxu Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


_ => {
// Need to use backend-specific things in the implementation.
return bx.codegen_intrinsic_call(instance, args, result, span);
Expand Down
38 changes: 38 additions & 0 deletions tests/codegen-llvm/hint/cold_path_propagation.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//@ compile-flags: -Copt-level=3
// Regression test for #155263: cold_path must propagate through
// FnOnce::call_once boundaries after LLVM inlining.
#![crate_type = "lib"]

use std::hint::cold_path;

fn dispatch<F: FnOnce(&mut u64) -> Option<u8>>(x: &mut u64, f: F) -> Option<u8> {
if *x == 0 {
cold_path();
return None;
}
*x -= 1;

let result = f(x);
if result.is_none() {
cold_path();
return None;
}
result
}

fn dec(x: &mut u64) -> Option<u8> {
if *x == 0 {
None
} else {
*x -= 1;
Some(1)
}
}

// CHECK-LABEL: @test_cold_path_through_fnonce(
// CHECK: asm sideeffect
// CHECK: asm sideeffect
#[no_mangle]
pub fn test_cold_path_through_fnonce(x: &mut u64, y: &mut u64) -> Option<u8> {
dispatch(x, |x| dec(y))
}
Loading