WIP: callsite labels and callsite return address machinery.

cranelift/codegen/meta/src/shared/entities.rs

@@ -43,6 +43,10 @@ pub(crate) struct EntityRefs {
 
     /// A variable-sized list of value operands. Use for Block and function call arguments.
     pub(crate) varargs: OperandKind,
+
+    /// A callsite. Used for primitives that return info about the
+    /// callsite, like its return address.
+    pub(crate) callsite: OperandKind,
 }
 
 impl EntityRefs {
@@ -85,6 +89,8 @@ impl EntityRefs {
                         returned from an instruction.
                     "#,
             ),
+
+            callsite: new("callsite", "ir::CallSite", "A callsite."),
         }
     }
 }

cranelift/codegen/meta/src/shared/formats.rs

@@ -15,6 +15,8 @@ pub(crate) struct Formats {
     pub(crate) branch_table: Rc<InstructionFormat>,
     pub(crate) call: Rc<InstructionFormat>,
     pub(crate) call_indirect: Rc<InstructionFormat>,
+    pub(crate) labeled_call: Rc<InstructionFormat>,
+    pub(crate) labeled_call_indirect: Rc<InstructionFormat>,
     pub(crate) cond_trap: Rc<InstructionFormat>,
     pub(crate) float_compare: Rc<InstructionFormat>,
     pub(crate) float_cond: Rc<InstructionFormat>,
@@ -49,6 +51,7 @@ pub(crate) struct Formats {
     pub(crate) unary_ieee32: Rc<InstructionFormat>,
     pub(crate) unary_ieee64: Rc<InstructionFormat>,
     pub(crate) unary_imm: Rc<InstructionFormat>,
+    pub(crate) callsite: Rc<InstructionFormat>,
 }
 
 impl Formats {
@@ -180,6 +183,21 @@ impl Formats {
                 .varargs()
                 .build(),
 
+            labeled_call: Builder::new("LabeledCall")
+                .imm(&entities.callsite)
+                .imm(&entities.func_ref)
+                .varargs()
+                .build(),
+
+            labeled_call_indirect: Builder::new("LabeledCallIndirect")
+                .imm(&entities.callsite)
+                .imm(&entities.sig_ref)
+                .value()
+                .varargs()
+                .build(),
+
+            callsite: Builder::new("CallSite").imm(&entities.callsite).build(),
+
             func_addr: Builder::new("FuncAddr").imm(&entities.func_ref).build(),
 
             atomic_rmw: Builder::new("AtomicRmw")

cranelift/codegen/meta/src/shared/instructions.rs

@@ -354,6 +354,7 @@ fn define_control_flow(
         .with_doc("function to call, declared by `function`");
     let args = &Operand::new("args", &entities.varargs).with_doc("call arguments");
     let rvals = &Operand::new("rvals", &entities.varargs).with_doc("return values");
+    let callsite = &Operand::new("callsite", &entities.callsite).with_doc("callsite label");
     ig.push(
         Inst::new(
             "call",
@@ -370,10 +371,30 @@ fn define_control_flow(
         .is_call(true),
     );
 
+    ig.push(
+        Inst::new(
+            "labeled_call",
+            r#"
+        Direct function call.
+
+        Call a function which has been declared in the preamble. The argument
+        types must match the function's signature.
+
+        This variant takes an explicit callsite label that can be used to query
+        metadata about the callsite.
+        "#,
+            &formats.labeled_call,
+        )
+        .operands_in(vec![callsite, FN, args])
+        .operands_out(vec![rvals])
+        .is_call(true),
+    );
+
     let SIG = &Operand::new("SIG", &entities.sig_ref).with_doc("function signature");
     let callee = &Operand::new("callee", iAddr).with_doc("address of function to call");
     let args = &Operand::new("args", &entities.varargs).with_doc("call arguments");
     let rvals = &Operand::new("rvals", &entities.varargs).with_doc("return values");
+    let callsite = &Operand::new("callsite", &entities.callsite).with_doc("callsite label");
     ig.push(
         Inst::new(
             "call_indirect",
@@ -395,6 +416,30 @@ fn define_control_flow(
         .is_call(true),
     );
 
+    ig.push(
+        Inst::new(
+            "labeled_call_indirect",
+            r#"
+        Indirect function call.
+
+        Call the function pointed to by `callee` with the given arguments. The
+        called function must match the specified signature.
+
+        Note that this is different from WebAssembly's ``call_indirect``; the
+        callee is a native address, rather than a table index. For WebAssembly,
+        `table_addr` and `load` are used to obtain a native address
+        from a table.
+
+        This variant takes an explicit callsite label that can be used to query
+        metadata about the callsite.
+        "#,
+            &formats.labeled_call_indirect,
+        )
+        .operands_in(vec![callsite, SIG, callee, args])
+        .operands_out(vec![rvals])
+        .is_call(true),
+    );
+
     let FN = &Operand::new("FN", &entities.func_ref)
         .with_doc("function to call, declared by `function`");
     let addr = &Operand::new("addr", iAddr);
@@ -415,6 +460,30 @@ fn define_control_flow(
         .operands_in(vec![FN])
         .operands_out(vec![addr]),
     );
+
+    let callsite = &Operand::new("callsite", &entities.callsite).with_doc("callsite label");
+    let addr = &Operand::new("addr", iAddr);
+    ig.push(
+        Inst::new(
+            "callsite_return_addr",
+            r#"
+        Get the address of the return point of a callsite.
+
+        This is the native-code PC value that will be returned by a
+        "return address" primitive in the called function, and that
+        will receive control when it returns.
+
+        This operator should only be used to implement low-level
+        behavior such as unwind-metadata maintenance in
+        trampolines. The address should *not* be jumped to directly by
+        CLIF: the results will be undefined, as the register and stack
+        state expected by the ABI may not be set up properly.
+        "#,
+            &formats.callsite,
+        )
+        .operands_in(vec![callsite])
+        .operands_out(vec![addr]),
+    );
 }
 
 #[inline(never)]

cranelift/codegen/src/ir/callsite.rs

@@ -0,0 +1,21 @@
+//! Data for callsite labels.
+
+use crate::ir::Inst;
+use cranelift_entity::packed_option::PackedOption;
+use std::fmt;
+
+/// Data associated with a callsite label.
+#[derive(Clone, Default)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub struct CallSiteData {
+    /// The call instruction that this callsite label refers to.
+    pub call_inst: PackedOption<Inst>,
+}
+
+impl fmt::Display for CallSiteData {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // We don't write out the instruction index in the textual
+        // form of CLIF; we fill it in automatically when parsing.
+        write!(f, "callsite")
+    }
+}

cranelift/codegen/src/ir/entities.rs

@@ -413,6 +413,44 @@ impl Table {
     }
 }
 
+/// An IR entity for naming a callsite. Used to get information about
+/// the callsite with various primitives.
+///
+/// Typically used like:
+///
+/// ```ignore
+///     callsite0 = callsite
+///
+/// block0(...):
+///     v1 := callsite_return_addr callsite0
+///     v2 := labeled_call callsite0, fn0(vN, vM, ...)
+/// ```
+///
+/// A `call` or `call_indirect` instruction *may* reference a
+/// callsite, but does not have to. (We may strengthen this in the
+/// future.) At most one call instruction may refer to a given
+/// callsite label; that call instruction *defines* the callsite.
+///
+/// A callsite label may then be used by primitives like
+/// `callsite_return_address`.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub struct CallSite(u32);
+entity_impl!(CallSite, "callsite");
+
+impl CallSite {
+    /// Create a new callsite label from its number.
+    ///
+    /// This method is for use by the parser.
+    pub fn with_number(n: u32) -> Option<Self> {
+        if n < u32::MAX {
+            Some(Self(n))
+        } else {
+            None
+        }
+    }
+}
+
 /// An opaque reference to any of the entities defined in this module that can appear in CLIF IR.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
@@ -447,6 +485,8 @@ pub enum AnyEntity {
     Table(Table),
     /// A function's stack limit
     StackLimit,
+    /// A callsite label.
+    CallSite(CallSite),
 }
 
 impl fmt::Display for AnyEntity {
@@ -466,6 +506,7 @@ impl fmt::Display for AnyEntity {
             Self::SigRef(r) => r.fmt(f),
             Self::Heap(r) => r.fmt(f),
             Self::Table(r) => r.fmt(f),
+            Self::CallSite(r) => r.fmt(f),
             Self::StackLimit => write!(f, "stack_limit"),
         }
     }
@@ -555,6 +596,12 @@ impl From<Table> for AnyEntity {
     }
 }
 
+impl From<CallSite> for AnyEntity {
+    fn from(r: CallSite) -> Self {
+        Self::CallSite(r)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

cranelift/codegen/src/ir/function.rs

@@ -7,9 +7,10 @@ use crate::entity::{PrimaryMap, SecondaryMap};
 use crate::ir;
 use crate::ir::JumpTables;
 use crate::ir::{
-    instructions::BranchInfo, Block, DynamicStackSlot, DynamicStackSlotData, DynamicType,
-    ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, Inst, InstructionData,
-    JumpTable, JumpTableData, Opcode, SigRef, StackSlot, StackSlotData, Table, TableData, Type,
+    instructions::BranchInfo, Block, CallSite, CallSiteData, DynamicStackSlot,
+    DynamicStackSlotData, DynamicType, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap,
+    HeapData, Inst, InstructionData, JumpTable, JumpTableData, Opcode, SigRef, StackSlot,
+    StackSlotData, Table, TableData, Type,
 };
 use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature};
 use crate::ir::{DynamicStackSlots, SourceLocs, StackSlots};
@@ -96,6 +97,9 @@ pub struct Function {
     /// Jump tables used in this function.
     pub jump_tables: JumpTables,
 
+    /// Callsite labels.
+    pub callsite_labels: PrimaryMap<ir::CallSite, ir::CallSiteData>,
+
     /// Data flow graph containing the primary definition of all instructions, blocks and values.
     pub dfg: DataFlowGraph,
 
@@ -129,6 +133,7 @@ impl Function {
             heaps: PrimaryMap::new(),
             tables: PrimaryMap::new(),
             jump_tables: PrimaryMap::new(),
+            callsite_labels: PrimaryMap::new(),
             dfg: DataFlowGraph::new(),
             layout: Layout::new(),
             srclocs: SecondaryMap::new(),
@@ -148,6 +153,7 @@ impl Function {
         self.dfg.clear();
         self.layout.clear();
         self.srclocs.clear();
+        self.callsite_labels.clear();
         self.stack_limit = None;
     }
 
@@ -218,6 +224,11 @@ impl Function {
         self.tables.push(data)
     }
 
+    /// Creates a new callsite label.
+    pub fn create_callsite(&mut self, data: CallSiteData) -> CallSite {
+        self.callsite_labels.push(data)
+    }
+
     /// Return an object that can display this function with correct ISA-specific annotations.
     pub fn display(&self) -> DisplayFunction<'_> {
         DisplayFunction(self, Default::default())

cranelift/codegen/src/ir/mod.rs

@@ -2,6 +2,7 @@
 
 mod atomic_rmw_op;
 mod builder;
+mod callsite;
 pub mod condcodes;
 pub mod constant;
 pub mod dfg;
@@ -32,12 +33,13 @@ pub use crate::ir::atomic_rmw_op::AtomicRmwOp;
 pub use crate::ir::builder::{
     InsertBuilder, InstBuilder, InstBuilderBase, InstInserterBase, ReplaceBuilder,
 };
+pub use crate::ir::callsite::CallSiteData;
 pub use crate::ir::constant::{ConstantData, ConstantPool};
 pub use crate::ir::dfg::{DataFlowGraph, ValueDef};
 pub use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes};
 pub use crate::ir::entities::{
-    Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap, Immediate, Inst,
-    JumpTable, SigRef, StackSlot, Table, Value,
+    Block, CallSite, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap,
+    Immediate, Inst, JumpTable, SigRef, StackSlot, Table, Value,
 };
 pub use crate::ir::extfunc::{
     AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature,

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -589,10 +589,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             });
         }
 
-        Opcode::Call | Opcode::CallIndirect => {
+        Opcode::Call | Opcode::CallIndirect | Opcode::LabeledCall | Opcode::LabeledCallIndirect => {
             let caller_conv = ctx.abi().call_conv();
             let (mut abi, inputs) = match op {
-                Opcode::Call => {
+                Opcode::Call | Opcode::LabeledCall => {
                     let (extname, dist) = ctx.call_target(insn).unwrap();
                     let extname = extname.clone();
                     let sig = ctx.call_sig(insn).unwrap();
@@ -603,7 +603,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                         &inputs[..],
                     )
                 }
-                Opcode::CallIndirect => {
+                Opcode::CallIndirect | Opcode::LabeledCallIndirect => {
                     let ptr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64);
                     let sig = ctx.call_sig(insn).unwrap();
                     assert!(inputs.len() - 1 == sig.params.len());
@@ -631,6 +631,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             abi.emit_stack_post_adjust(ctx);
         }
 
+        Opcode::CallsiteReturnAddr => {
+            unimplemented!("Not implemented yet");
+        }
+
         Opcode::GetPinnedReg => {
             let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             ctx.emit(Inst::gen_move(rd, xreg(PINNED_REG), I64));

cranelift/codegen/src/isa/s390x/inst.isle

@@ -1581,7 +1581,7 @@
       (memarg_reg_plus_reg x y 0 flags))
 
 (rule (lower_address flags
-                     (symbol_value (symbol_value_data name (reloc_distance_near) sym_offset))
+                     (symbol_value (symbol_value_data name (RelocDistance.Near) sym_offset))
                      (i64_from_offset offset))
       (if-let final_offset (memarg_symbol_offset_sum offset sym_offset))
       (memarg_symbol name final_offset flags))
@@ -1602,15 +1602,15 @@
 
 (decl pure load_sym (Inst) Inst)
 (rule (load_sym inst)
-      (if-let (load _ (symbol_value (symbol_value_data _ (reloc_distance_near) sym_offset))
+      (if-let (load _ (symbol_value (symbol_value_data _ (RelocDistance.Near) sym_offset))
                     (i64_from_offset load_offset))
           inst)
       (if (memarg_symbol_offset_sum sym_offset load_offset))
       inst)
 
 (decl pure uload16_sym (Inst) Inst)
 (rule (uload16_sym inst)
-      (if-let (uload16 _ (symbol_value (symbol_value_data _ (reloc_distance_near) sym_offset))
+      (if-let (uload16 _ (symbol_value (symbol_value_data _ (RelocDistance.Near) sym_offset))
                        (i64_from_offset load_offset))
           inst)
       (if (memarg_symbol_offset_sum sym_offset load_offset))