Skip to content

Commit

Permalink
Auto merge of rust-lang#106364 - JakobDegen:top-down-inlining, r=cjgi…
Browse files Browse the repository at this point in the history 
…llot

Reenable limited top-down MIR inlining

Reverts most of rust-lang#105119 and uses an alternative strategy to prevent exponential blowup. Specifically, we allow doing top-down inlining up to depth at most five, and for at most one call site per nested body.

r? `@cjgillot`
  • Loading branch information
@bors
bors committed Jan 2, 2023
2 parents d6f99e5 + ee6503a commit 67d1617
Show file tree
Hide file tree
Showing 11 changed files with 220 additions and 97 deletions.
42 changes: 36 additions & 6 deletions compiler/rustc_mir_transform/src/inline.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! Inlining pass for MIR functions
use crate::deref_separator::deref_finder;
use rustc_attr::InlineAttr;
use rustc_hir::def_id::DefId;
use rustc_index::bit_set::BitSet;
use rustc_index::vec::Idx;
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
Expand All @@ -27,6 +28,8 @@ const RESUME_PENALTY: usize = 45;

const UNKNOWN_SIZE_COST: usize = 10;

const TOP_DOWN_DEPTH_LIMIT: usize = 5;

pub struct Inline;

#[derive(Copy, Clone, Debug)]
Expand Down Expand Up @@ -86,8 +89,13 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {

let param_env = tcx.param_env_reveal_all_normalized(def_id);

let mut this =
Inliner { tcx, param_env, codegen_fn_attrs: tcx.codegen_fn_attrs(def_id), changed: false };
let mut this = Inliner {
tcx,
param_env,
codegen_fn_attrs: tcx.codegen_fn_attrs(def_id),
history: Vec::new(),
changed: false,
};
let blocks = BasicBlock::new(0)..body.basic_blocks.next_index();
this.process_blocks(body, blocks);
this.changed
Expand All @@ -98,12 +106,26 @@ struct Inliner<'tcx> {
param_env: ParamEnv<'tcx>,
/// Caller codegen attributes.
codegen_fn_attrs: &'tcx CodegenFnAttrs,
/// Stack of inlined instances.
/// We only check the `DefId` and not the substs because we want to
/// avoid inlining cases of polymorphic recursion.
/// The number of `DefId`s is finite, so checking history is enough
/// to ensure that we do not loop endlessly while inlining.
history: Vec<DefId>,
/// Indicates that the caller body has been modified.
changed: bool,
}

impl<'tcx> Inliner<'tcx> {
fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range<BasicBlock>) {
// How many callsites in this body are we allowed to inline? We need to limit this in order
// to prevent super-linear growth in MIR size
let inline_limit = match self.history.len() {
0 => usize::MAX,
1..=TOP_DOWN_DEPTH_LIMIT => 1,
_ => return,
};
let mut inlined_count = 0;
for bb in blocks {
let bb_data = &caller_body[bb];
if bb_data.is_cleanup {
Expand All @@ -122,12 +144,16 @@ impl<'tcx> Inliner<'tcx> {
debug!("not-inlined {} [{}]", callsite.callee, reason);
continue;
}
Ok(_) => {
Ok(new_blocks) => {
debug!("inlined {}", callsite.callee);
self.changed = true;
// We could process the blocks returned by `try_inlining` here. However, that
// leads to exponential compile times due to the top-down nature of this kind
// of inlining.
inlined_count += 1;
if inlined_count == inline_limit {
return;
}
self.history.push(callsite.callee.def_id());
self.process_blocks(caller_body, new_blocks);
self.history.pop();
}
}
}
Expand Down Expand Up @@ -301,6 +327,10 @@ impl<'tcx> Inliner<'tcx> {
return None;
}

if self.history.contains(&callee.def_id()) {
return None;
}

let fn_sig = self.tcx.bound_fn_sig(def_id).subst(self.tcx, substs);
let source_info = SourceInfo { span: fn_span, ..terminator.source_info };

Expand Down
23 changes: 11 additions & 12 deletions src/test/mir-opt/inline/cycle.g.Inline.diff
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
+ let _3: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
+ let mut _4: &fn() {main}; // in scope 1 at $DIR/cycle.rs:6:5: 6:6
+ let mut _5: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
+ scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) { // at $DIR/cycle.rs:6:5: 6:8
+ }
+ }

bb0: {
Expand All @@ -27,10 +29,7 @@
+ StorageLive(_4); // scope 1 at $DIR/cycle.rs:6:5: 6:6
+ _4 = &_2; // scope 1 at $DIR/cycle.rs:6:5: 6:6
+ StorageLive(_5); // scope 1 at $DIR/cycle.rs:6:5: 6:8
+ _3 = <fn() {main} as Fn<()>>::call(move _4, move _5) -> [return: bb2, unwind: bb3]; // scope 1 at $DIR/cycle.rs:6:5: 6:8
+ // mir::Constant
+ // + span: $DIR/cycle.rs:6:5: 6:6
+ // + literal: Const { ty: for<'a> extern "rust-call" fn(&'a fn() {main}, ()) -> <fn() {main} as FnOnce<()>>::Output {<fn() {main} as Fn<()>>::call}, val: Value(<ZST>) }
+ _3 = move (*_4)() -> [return: bb4, unwind: bb2]; // scope 2 at $SRC_DIR/core/src/ops/function.rs:LL:COL
}

bb1: {
Expand All @@ -40,19 +39,19 @@
return; // scope 0 at $DIR/cycle.rs:+2:2: +2:2
+ }
+
+ bb2: {
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
+ bb2 (cleanup): {
+ drop(_2) -> bb3; // scope 1 at $DIR/cycle.rs:7:1: 7:2
+ }
+
+ bb3 (cleanup): {
+ drop(_2) -> bb4; // scope 1 at $DIR/cycle.rs:7:1: 7:2
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
+ }
+
+ bb4 (cleanup): {
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
+ bb4: {
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
}
}

23 changes: 11 additions & 12 deletions src/test/mir-opt/inline/cycle.main.Inline.diff
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
+ let _3: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
+ let mut _4: &fn() {g}; // in scope 1 at $DIR/cycle.rs:6:5: 6:6
+ let mut _5: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
+ scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) { // at $DIR/cycle.rs:6:5: 6:8
+ }
+ }

bb0: {
Expand All @@ -27,10 +29,7 @@
+ StorageLive(_4); // scope 1 at $DIR/cycle.rs:6:5: 6:6
+ _4 = &_2; // scope 1 at $DIR/cycle.rs:6:5: 6:6
+ StorageLive(_5); // scope 1 at $DIR/cycle.rs:6:5: 6:8
+ _3 = <fn() {g} as Fn<()>>::call(move _4, move _5) -> [return: bb2, unwind: bb3]; // scope 1 at $DIR/cycle.rs:6:5: 6:8
+ // mir::Constant
+ // + span: $DIR/cycle.rs:6:5: 6:6
+ // + literal: Const { ty: for<'a> extern "rust-call" fn(&'a fn() {g}, ()) -> <fn() {g} as FnOnce<()>>::Output {<fn() {g} as Fn<()>>::call}, val: Value(<ZST>) }
+ _3 = move (*_4)() -> [return: bb4, unwind: bb2]; // scope 2 at $SRC_DIR/core/src/ops/function.rs:LL:COL
}

bb1: {
Expand All @@ -40,19 +39,19 @@
return; // scope 0 at $DIR/cycle.rs:+2:2: +2:2
+ }
+
+ bb2: {
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
+ bb2 (cleanup): {
+ drop(_2) -> bb3; // scope 1 at $DIR/cycle.rs:7:1: 7:2
+ }
+
+ bb3 (cleanup): {
+ drop(_2) -> bb4; // scope 1 at $DIR/cycle.rs:7:1: 7:2
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
+ }
+
+ bb4 (cleanup): {
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
+ bb4: {
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
}
}

53 changes: 39 additions & 14 deletions src/test/mir-opt/inline/exponential_runtime.main.Inline.diff
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,43 +8,68 @@
+ let _2: (); // in scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
+ let _3: (); // in scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
+ let _4: (); // in scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
+ scope 2 (inlined <() as F>::call) { // at $DIR/exponential_runtime.rs:73:9: 73:25
+ let _5: (); // in scope 2 at $DIR/exponential_runtime.rs:61:9: 61:25
+ let _6: (); // in scope 2 at $DIR/exponential_runtime.rs:62:9: 62:25
+ let _7: (); // in scope 2 at $DIR/exponential_runtime.rs:63:9: 63:25
+ }
+ }

bb0: {
StorageLive(_1); // scope 0 at $DIR/exponential_runtime.rs:+1:5: +1:22
- _1 = <() as G>::call() -> bb1; // scope 0 at $DIR/exponential_runtime.rs:+1:5: +1:22
+ StorageLive(_2); // scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
+ _2 = <() as F>::call() -> bb1; // scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
+ StorageLive(_5); // scope 2 at $DIR/exponential_runtime.rs:61:9: 61:25
+ _5 = <() as E>::call() -> bb3; // scope 2 at $DIR/exponential_runtime.rs:61:9: 61:25
// mir::Constant
- // + span: $DIR/exponential_runtime.rs:86:5: 86:20
- // + literal: Const { ty: fn() {<() as G>::call}, val: Value(<ZST>) }
+ // + span: $DIR/exponential_runtime.rs:73:9: 73:23
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
+ // + span: $DIR/exponential_runtime.rs:61:9: 61:23
+ // + literal: Const { ty: fn() {<() as E>::call}, val: Value(<ZST>) }
}

bb1: {
+ StorageDead(_2); // scope 1 at $DIR/exponential_runtime.rs:73:25: 73:26
+ StorageLive(_3); // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
+ _3 = <() as F>::call() -> bb2; // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
+ // mir::Constant
+ // + span: $DIR/exponential_runtime.rs:74:9: 74:23
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
+ }
+
+ bb2: {
+ StorageDead(_3); // scope 1 at $DIR/exponential_runtime.rs:74:25: 74:26
+ StorageLive(_4); // scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
+ _4 = <() as F>::call() -> bb3; // scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
+ _4 = <() as F>::call() -> bb2; // scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
+ // mir::Constant
+ // + span: $DIR/exponential_runtime.rs:75:9: 75:23
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
+ }
+
+ bb3: {
+ bb2: {
+ StorageDead(_4); // scope 1 at $DIR/exponential_runtime.rs:75:25: 75:26
StorageDead(_1); // scope 0 at $DIR/exponential_runtime.rs:+1:22: +1:23
_0 = const (); // scope 0 at $DIR/exponential_runtime.rs:+0:11: +2:2
return; // scope 0 at $DIR/exponential_runtime.rs:+2:2: +2:2
+ }
+
+ bb3: {
+ StorageDead(_5); // scope 2 at $DIR/exponential_runtime.rs:61:25: 61:26
+ StorageLive(_6); // scope 2 at $DIR/exponential_runtime.rs:62:9: 62:25
+ _6 = <() as E>::call() -> bb4; // scope 2 at $DIR/exponential_runtime.rs:62:9: 62:25
+ // mir::Constant
+ // + span: $DIR/exponential_runtime.rs:62:9: 62:23
+ // + literal: Const { ty: fn() {<() as E>::call}, val: Value(<ZST>) }
+ }
+
+ bb4: {
+ StorageDead(_6); // scope 2 at $DIR/exponential_runtime.rs:62:25: 62:26
+ StorageLive(_7); // scope 2 at $DIR/exponential_runtime.rs:63:9: 63:25
+ _7 = <() as E>::call() -> bb5; // scope 2 at $DIR/exponential_runtime.rs:63:9: 63:25
+ // mir::Constant
+ // + span: $DIR/exponential_runtime.rs:63:9: 63:23
+ // + literal: Const { ty: fn() {<() as E>::call}, val: Value(<ZST>) }
+ }
+
+ bb5: {
+ StorageDead(_7); // scope 2 at $DIR/exponential_runtime.rs:63:25: 63:26
+ StorageDead(_2); // scope 1 at $DIR/exponential_runtime.rs:73:25: 73:26
+ StorageLive(_3); // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
+ _3 = <() as F>::call() -> bb1; // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
+ // mir::Constant
+ // + span: $DIR/exponential_runtime.rs:74:9: 74:23
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
}
}

11 changes: 7 additions & 4 deletions src/test/mir-opt/inline/inline_cycle.one.Inline.diff
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,20 @@
let mut _0: (); // return place in scope 0 at $DIR/inline_cycle.rs:+0:10: +0:10
let _1: (); // in scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
+ scope 1 (inlined <C as Call>::call) { // at $DIR/inline_cycle.rs:14:5: 14:24
+ scope 2 (inlined <A<C> as Call>::call) { // at $DIR/inline_cycle.rs:43:9: 43:23
+ scope 3 (inlined <B<C> as Call>::call) { // at $DIR/inline_cycle.rs:28:9: 28:31
+ }
+ }
+ }

bb0: {
StorageLive(_1); // scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
- _1 = <C as Call>::call() -> bb1; // scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
+ _1 = <A<C> as Call>::call() -> bb1; // scope 1 at $DIR/inline_cycle.rs:43:9: 43:23
+ _1 = <C as Call>::call() -> bb1; // scope 3 at $DIR/inline_cycle.rs:36:9: 36:28
// mir::Constant
- // + span: $DIR/inline_cycle.rs:14:5: 14:22
- // + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
+ // + span: $DIR/inline_cycle.rs:43:9: 43:21
+ // + literal: Const { ty: fn() {<A<C> as Call>::call}, val: Value(<ZST>) }
+ // + span: $DIR/inline_cycle.rs:36:9: 36:26
// + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
}

bb1: {
Expand Down
7 changes: 3 additions & 4 deletions src/test/mir-opt/inline/inline_cycle.two.Inline.diff
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
+ debug f => _2; // in scope 1 at $DIR/inline_cycle.rs:53:22: 53:23
+ let _3: (); // in scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
+ let mut _4: (); // in scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
+ scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) { // at $DIR/inline_cycle.rs:54:5: 54:8
+ }
+ }

bb0: {
Expand All @@ -24,10 +26,7 @@
// + literal: Const { ty: fn() {f}, val: Value(<ZST>) }
+ StorageLive(_3); // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
+ StorageLive(_4); // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
+ _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, move _4) -> bb1; // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
+ // mir::Constant
+ // + span: $DIR/inline_cycle.rs:54:5: 54:6
+ // + literal: Const { ty: extern "rust-call" fn(fn() {f}, ()) -> <fn() {f} as FnOnce<()>>::Output {<fn() {f} as FnOnce<()>>::call_once}, val: Value(<ZST>) }
+ _3 = move _2() -> bb1; // scope 2 at $SRC_DIR/core/src/ops/function.rs:LL:COL
}

bb1: {
Expand Down
9 changes: 6 additions & 3 deletions src/test/mir-opt/inline/inline_cycle_generic.main.Inline.diff
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,21 @@
let _1: (); // in scope 0 at $DIR/inline_cycle_generic.rs:+1:5: +1:24
+ scope 1 (inlined <C as Call>::call) { // at $DIR/inline_cycle_generic.rs:9:5: 9:24
+ scope 2 (inlined <B<A> as Call>::call) { // at $DIR/inline_cycle_generic.rs:38:9: 38:31
+ scope 3 (inlined <A as Call>::call) { // at $DIR/inline_cycle_generic.rs:31:9: 31:28
+ scope 4 (inlined <B<C> as Call>::call) { // at $DIR/inline_cycle_generic.rs:23:9: 23:31
+ }
+ }
+ }
+ }

bb0: {
StorageLive(_1); // scope 0 at $DIR/inline_cycle_generic.rs:+1:5: +1:24
- _1 = <C as Call>::call() -> bb1; // scope 0 at $DIR/inline_cycle_generic.rs:+1:5: +1:24
+ _1 = <A as Call>::call() -> bb1; // scope 2 at $DIR/inline_cycle_generic.rs:31:9: 31:28
+ _1 = <C as Call>::call() -> bb1; // scope 4 at $DIR/inline_cycle_generic.rs:31:9: 31:28
// mir::Constant
- // + span: $DIR/inline_cycle_generic.rs:9:5: 9:22
- // + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
+ // + span: $DIR/inline_cycle_generic.rs:31:9: 31:26
+ // + literal: Const { ty: fn() {<A as Call>::call}, val: Value(<ZST>) }
// + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
}

bb1: {
Expand Down
Loading

0 comments on commit 67d1617

Please sign in to comment.