rt: Move call target to end of instruction stream

dora-asm/src/arm64.rs

@@ -220,7 +220,7 @@ impl AssemblerArm64 {
         self.buffer.bind_label(lbl);
     }
 
-    fn offset(&self, lbl: Label) -> Option<u32> {
+    pub fn offset(&self, lbl: Label) -> Option<u32> {
         self.buffer.offset(lbl)
     }
 
@@ -230,7 +230,7 @@ impl AssemblerArm64 {
         self.buffer
     }
 
-    fn align_to(&mut self, alignment: usize) {
+    pub fn align_to(&mut self, alignment: usize) {
         while self.buffer.code.len() % alignment != 0 {
             self.brk(0);
         }

dora-asm/src/x64.rs

@@ -97,7 +97,7 @@ impl AssemblerX64 {
         self.buffer.bind_label(lbl);
     }
 
-    fn offset(&self, lbl: Label) -> Option<u32> {
+    pub fn offset(&self, lbl: Label) -> Option<u32> {
         self.buffer.offset(lbl)
     }
 
@@ -107,7 +107,7 @@ impl AssemblerX64 {
         self.buffer
     }
 
-    fn align_to(&mut self, alignment: usize) {
+    pub fn align_to(&mut self, alignment: usize) {
         while self.buffer.code.len() % alignment != 0 {
             self.int3();
         }

dora-runtime/src/masm.rs

@@ -54,12 +54,12 @@ pub struct MacroAssembler {
     asm: Assembler,
     bailouts: Vec<(Label, Trap, Location)>,
     lazy_compilation: LazyCompilationData,
-    constpool: ConstPool,
+    direct_call_sites: Vec<(u32, Label)>,
     epilog_constants: Vec<(Label, EpilogConstant)>,
     gcpoints: GcPointTable,
     comments: CommentTable,
     positions: LocationTable,
-    relocations: Vec<(i32, Label)>,
+    relocations: Vec<(u32, Label)>,
     scratch_registers: ScratchRegisters,
 }
 
@@ -69,7 +69,7 @@ impl MacroAssembler {
             asm: MacroAssembler::create_assembler(),
             bailouts: Vec::new(),
             lazy_compilation: LazyCompilationData::new(),
-            constpool: ConstPool::new(),
+            direct_call_sites: Vec::new(),
             epilog_constants: Vec::new(),
             gcpoints: GcPointTable::new(),
             comments: CommentTable::new(),
@@ -89,8 +89,30 @@ impl MacroAssembler {
         self.emit_bailouts();
         self.emit_epilog_constants();
 
-        // Align data such that code start is properly aligned.
-        let cp_size = self.constpool.align(CODE_ALIGNMENT as i32);
+        for (pos, label) in self.direct_call_sites {
+            let entry = self
+                .lazy_compilation
+                .get_mut(pos)
+                .expect("missing call site");
+            match entry.clone() {
+                LazyCompilationSite::Direct {
+                    fct_id,
+                    type_params,
+                    const_pool_offset_from_ra,
+                } => {
+                    if const_pool_offset_from_ra == 0 {
+                        let label_pos = self.asm.offset(label).expect("missing label");
+                        let const_pool_offset_from_ra = label_pos as i32 - pos as i32;
+                        *entry = LazyCompilationSite::Direct {
+                            fct_id,
+                            type_params,
+                            const_pool_offset_from_ra,
+                        };
+                    }
+                }
+                _ => unreachable!(),
+            }
+        }
 
         let asm = self.asm.finalize(CODE_ALIGNMENT);
 
@@ -99,15 +121,12 @@ impl MacroAssembler {
             .into_iter()
             .map(|(pos, label)| {
                 let offset = asm.offset(label).expect("unresolved label");
-                (
-                    (cp_size + pos) as u32,
-                    RelocationKind::JumpTableEntry(offset),
-                )
+                (pos, RelocationKind::JumpTableEntry(offset))
             })
             .collect::<Vec<_>>();
 
         CodeDescriptor {
-            constpool: self.constpool,
+            constpool: ConstPool::new(),
             code: asm.code(),
             lazy_compilation: self.lazy_compilation,
             gcpoints: self.gcpoints,
@@ -137,7 +156,17 @@ impl MacroAssembler {
 
     fn emit_epilog_constants(&mut self) {
         for (label, value) in &self.epilog_constants {
+            let align = match value {
+                EpilogConstant::Float32(..) => std::mem::size_of::<u32>(),
+                EpilogConstant::Address(..)
+                | EpilogConstant::Float64(..)
+                | EpilogConstant::Int128(..)
+                | EpilogConstant::JumpTable(..) => std::mem::size_of::<u64>(),
+            };
+
+            self.asm.align_to(align);
             self.asm.bind_label(*label);
+
             match value {
                 EpilogConstant::Address(value) => {
                     self.asm.emit_u64(value.to_usize() as u64);
@@ -167,10 +196,6 @@ impl MacroAssembler {
         }
     }
 
-    pub fn add_const_addr(&mut self, ptr: Address) -> i32 {
-        self.constpool.add_addr(ptr)
-    }
-
     pub fn emit_epilog_const(&mut self, value: EpilogConstant) -> Label {
         let label = self.create_label();
         self.epilog_constants.push((label, value));

dora-runtime/src/masm/arm64.rs

@@ -80,20 +80,21 @@ impl MacroAssembler {
         ptr: Address,
         type_params: BytecodeTypeArray,
     ) {
-        let disp = self.add_const_addr(ptr);
-        let pos = self.pos() as i32;
+        let label = self.emit_epilog_const(EpilogConstant::Address(ptr));
 
         let scratch = self.get_scratch();
 
-        self.load_constpool(*scratch, disp + pos);
+        self.asm.adr_label((*scratch).into(), label);
+        self.asm.ldur((*scratch).into(), (*scratch).into(), 0);
         self.asm.bl_r((*scratch).into());
 
-        let pos = self.pos() as i32;
+        let pos = self.pos() as u32;
         self.emit_lazy_compilation_site(LazyCompilationSite::Direct {
             fct_id,
             type_params,
-            const_pool_offset_from_ra: -(disp + pos),
+            const_pool_offset_from_ra: 0,
         });
+        self.direct_call_sites.push((pos, label));
     }
 
     pub fn raw_call(&mut self, ptr: Address) {

dora-runtime/src/masm/x64.rs

@@ -79,18 +79,18 @@ impl MacroAssembler {
         ptr: Address,
         type_params: BytecodeTypeArray,
     ) {
-        let disp = self.add_const_addr(ptr);
-        let pos = self.pos() as i32;
+        let label = self.emit_epilog_const(EpilogConstant::Address(ptr));
 
-        self.load_constpool(REG_RESULT, disp + pos);
+        self.asm.movq_rl(REG_RESULT.into(), label);
         self.call_reg(REG_RESULT);
 
-        let pos = self.pos() as i32;
+        let pos = self.pos() as u32;
         self.emit_lazy_compilation_site(LazyCompilationSite::Direct {
             fct_id,
             type_params,
-            const_pool_offset_from_ra: -(disp + pos),
+            const_pool_offset_from_ra: 0,
         });
+        self.direct_call_sites.push((pos, label));
     }
 
     pub fn raw_call(&mut self, ptr: Address) {

dora-runtime/src/vm/code.rs

@@ -475,6 +475,17 @@ impl LazyCompilationData {
         }
     }
 
+    pub fn get_mut(&mut self, offset: u32) -> Option<&mut LazyCompilationSite> {
+        let result = self
+            .entries
+            .binary_search_by_key(&offset, |&(offset, _)| offset);
+
+        match result {
+            Ok(idx) => Some(&mut self.entries[idx].1),
+            Err(_) => None,
+        }
+    }
+
     pub fn entries(&self) -> &[(u32, LazyCompilationSite)] {
         &self.entries
     }