Skip to content

Optimize DST size_of_val, align_of_val, and field projection codegen #164492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
joshlf wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Optimize DST size_of_val, align_of_val, and field projection codegen #164492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
} else {
None
};
let (size, _align) = crate::unsize::size_and_align_of(fx, layout, meta);
let size = crate::unsize::size_of_dst(fx, layout, meta);
ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
}
sym::align_of_val => {
Expand All @@ -475,7 +475,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
} else {
None
};
let (_size, align) = crate::unsize::size_and_align_of(fx, layout, meta);
let align = crate::unsize::align_of_dst(fx, layout, meta);
ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
}

Expand Down
400 changes: 288 additions & 112 deletions compiler/rustc_codegen_cranelift/src/unsize.rs
View file
Open in desktop

Large diffs are not rendered by default.

41 changes: 7 additions & 34 deletions compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,41 +18,14 @@ fn codegen_field<'tcx>(
let field_offset = layout.fields.offset(field.index());
let field_layout = layout.field(&*fx, field.index());

let simple = |fx: &mut FunctionCx<'_, '_, '_>| {
(base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), field_layout)
let ptr = if let Some(dyn_offset) =
crate::unsize::dst_field_offset(fx, layout, field_layout, field_offset, extra)
{
base.offset_value(fx, dyn_offset)
} else {
base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap())
};

if field_layout.is_sized() {
return simple(fx);
}
match field_layout.ty.kind() {
ty::Slice(..) | ty::Str => simple(fx),
_ => {
let unaligned_offset = field_offset.bytes();

// Get the alignment of the field
let (_, mut unsized_align) = crate::unsize::size_and_align_of(fx, field_layout, extra);

// For packed types, we need to cap alignment.
if let ty::Adt(def, _) = layout.ty.kind() {
if let Some(packed) = def.repr().pack {
let packed = fx.bcx.ins().iconst(fx.pointer_type, packed.bytes() as i64);
let cmp = fx.bcx.ins().icmp(IntCC::UnsignedLessThan, unsized_align, packed);
unsized_align = fx.bcx.ins().select(cmp, unsized_align, packed);
}
}

// Bump the unaligned offset up to the appropriate alignment
let one = fx.bcx.ins().iconst(fx.pointer_type, 1);
let align_sub_1 = fx.bcx.ins().isub(unsized_align, one);
let and_lhs = fx.bcx.ins().iadd_imm_u(align_sub_1, unaligned_offset as i64);
let zero = fx.bcx.ins().iconst(fx.pointer_type, 0);
let and_rhs = fx.bcx.ins().isub(zero, unsized_align);
let offset = fx.bcx.ins().band(and_lhs, and_rhs);

(base.offset_value(fx, offset), field_layout)
}
}
(ptr, field_layout)
}

/// A read-only value
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,13 +165,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
sym::size_of_val => {
let tp_ty = fn_args.type_at(0);
let (_, meta) = args[0].val.pointer_parts();
let (llsize, _) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta, span);
let llsize = size_of_val::size_of_dst(bx, tp_ty, meta, span);
OperandValue::Immediate(llsize)
}
sym::align_of_val => {
let tp_ty = fn_args.type_at(0);
let (_, meta) = args[0].val.pointer_parts();
let (_, llalign) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta, span);
let llalign = size_of_val::align_of_dst(bx, tp_ty, meta, span);
OperandValue::Immediate(llalign)
}
sym::vtable_size | sym::vtable_align => {
Expand Down
137 changes: 22 additions & 115 deletions compiler/rustc_codegen_ssa/src/mir/place.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,13 @@ use rustc_abi::{
use rustc_middle::mir;
use rustc_middle::mir::PlaceTy;
use rustc_middle::mir::interpret::Scalar;
use rustc_middle::ty::Ty;
use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
use rustc_middle::ty::{self, Ty};
use rustc_span::{DUMMY_SP, bug};
use rustc_span::bug;
use tracing::{debug, instrument};

use super::operand::OperandValue;
use super::{FunctionCx, LocalRef};
use crate::common::IntPredicate;
use crate::size_of_val;
use crate::traits::*;

Expand Down Expand Up @@ -181,79 +180,29 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
let offset = self.layout.fields.offset(ix);
let effective_field_align = self.val.align.restrict_for_offset(offset);

// `simple` is called when we don't need to adjust the offset to
// the dynamic alignment of the field.
let mut simple = || {
let llval = if offset.bytes() == 0 {
self.val.llval
} else {
bx.inbounds_ptradd(self.val.llval, bx.const_usize(offset.bytes()))
};
let val = PlaceValue {
llval,
llextra: if bx.cx().tcx().type_has_metadata(field.ty, bx.cx().typing_env()) {
self.val.llextra
} else {
None
},
align: effective_field_align,
};
val.with_type(field)
};
let tcx = bx.cx().tcx();
let typing_env = bx.cx().typing_env();

// Simple cases, which don't need DST adjustment:
// * known alignment - sized types, `[T]`, `str`
// * offset 0 -- rounding up to alignment cannot change the offset
// Note that looking at `field.align` is incorrect since that is not necessarily equal
// to the dynamic alignment of the type.
match field.ty.kind() {
_ if field.is_sized() => return simple(),
ty::Slice(..) | ty::Str => return simple(),
_ if offset.bytes() == 0 => return simple(),
_ => {}
}

// We need to get the pointer manually now.
// We do this by casting to a `*i8`, then offsetting it by the appropriate amount.
// We do this instead of, say, simply adjusting the pointer from the result of a GEP
// because the field may have an arbitrary alignment in the LLVM representation.
//
// To demonstrate:
//
// struct Foo<T: ?Sized> {
// x: u16,
// y: T
// }
//
// The type `Foo<Foo<Trait>>` is represented in LLVM as `{ u16, { u16, u8 }}`, meaning that
// the `y` field has 16-bit alignment.

let meta = self.val.llextra;

let unaligned_offset = bx.cx().const_usize(offset.bytes());

// Get the alignment of the field. No span is available here to blame a layout error on.
let (_, mut unsized_align) =
size_of_val::size_and_align_of_dst(bx, field.ty, meta, DUMMY_SP);

// For packed types, we need to cap alignment.
if let ty::Adt(def, _) = self.layout.ty.kind()
&& let Some(packed) = def.repr().pack
let llval = if let Some(dyn_offset) =
size_of_val::dst_field_offset(bx, self.layout, field, offset, self.val.llextra)
{
let packed = bx.const_usize(packed.bytes());
let cmp = bx.icmp(IntPredicate::IntULT, unsized_align, packed);
unsized_align = bx.select(cmp, unsized_align, packed)
}

// Bump the unaligned offset up to the appropriate alignment
let offset = round_up_const_value_to_alignment(bx, unaligned_offset, unsized_align);

debug!("struct_field_ptr: DST field offset: {:?}", offset);
debug!("struct_field_ptr: DST field offset: {:?}", dyn_offset);
bx.inbounds_ptradd(self.val.llval, dyn_offset)
} else if offset.bytes() == 0 {
self.val.llval
} else {
bx.inbounds_ptradd(self.val.llval, bx.const_usize(offset.bytes()))
};

// Adjust pointer.
let ptr = bx.inbounds_ptradd(self.val.llval, offset);
let val =
PlaceValue { llval: ptr, llextra: self.val.llextra, align: effective_field_align };
let val = PlaceValue {
llval,
llextra: if tcx.type_has_metadata(field.ty, typing_env) {
self.val.llextra
} else {
None
},
align: effective_field_align,
};
val.with_type(field)
}

Expand Down Expand Up @@ -428,48 +377,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}

fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
bx: &mut Bx,
value: Bx::Value,
align: Bx::Value,
) -> Bx::Value {
// In pseudo code:
//
// if value & (align - 1) == 0 {
// value
// } else {
// (value & !(align - 1)) + align
// }
//
// Usually this is written without branches as
//
// (value + align - 1) & !(align - 1)
//
// But this formula cannot take advantage of constant `value`. E.g. if `value` is known
// at compile time to be `1`, this expression should be optimized to `align`. However,
// optimization only holds if `align` is a power of two. Since the optimizer doesn't know
// that `align` is a power of two, it cannot perform this optimization.
//
// Instead we use
//
// value + (-value & (align - 1))
//
// Since `align` is used only once, the expression can be optimized. For `value = 0`
// its optimized to `0` even in debug mode.
//
// NB: The previous version of this code used
//
// (value + align - 1) & -align
//
// Even though `-align == !(align - 1)`, LLVM failed to optimize this even for
// `value = 0`. Bug report: https://bugs.llvm.org/show_bug.cgi?id=48559
let one = bx.const_usize(1);
let align_minus_1 = bx.sub(align, one);
let neg_value = bx.neg(value);
let offset = bx.and(neg_value, align_minus_1);
bx.add(value, offset)
}

/// Calculates the value that needs to be stored to mark the discriminant.
///
/// This might be `None` for a `struct` or a niched variant (like `Some(&3)`).
Expand Down
Loading
Loading