Stop using LLVM struct types for array/pointer offset GEPs (rebase)

compiler/rustc_codegen_gcc/src/builder.rs

@@ -17,8 +17,8 @@ use rustc_codegen_ssa::common::{
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::{
-    BackendTypes, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods,
-    LayoutTypeCodegenMethods, OverflowOp, ReturnSlot, StaticBuilderMethods,
+    BackendTypes, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, OverflowOp,
+    ReturnSlot, StaticBuilderMethods,
 };
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
@@ -1184,11 +1184,9 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         let align = dest.val.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
         cg_elem.val.store(self, PlaceRef::new_sized_aligned(current_val, cg_elem.layout, align));
 
-        let next = self.inbounds_gep(
-            self.backend_type(cg_elem.layout),
-            current.to_rvalue(),
-            &[self.const_usize(1)],
-        );
+        let next = self
+            .inbounds_ptradd(current.to_rvalue(), self.const_usize(cg_elem.layout.size.bytes()));
+        let next = self.context.new_cast(self.location, next, ptr_type);
         self.llbb().add_assignment(self.location, current, next);
         self.br(header_bb);
 
@@ -1267,52 +1265,29 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         );
     }
 
-    fn gep(
-        &mut self,
-        typ: Type<'gcc>,
-        ptr: RValue<'gcc>,
-        indices: &[RValue<'gcc>],
-    ) -> RValue<'gcc> {
-        // NOTE: due to opaque pointers now being used, we need to cast here.
-        let ptr = self.context.new_cast(self.location, ptr, typ.make_pointer());
+    fn gep(&mut self, stride: Size, ptr: RValue<'gcc>, index: RValue<'gcc>) -> RValue<'gcc> {
         let ptr_type = ptr.get_type();
-        let mut pointee_type = ptr.get_type();
         // NOTE: we cannot use array indexing here like in inbounds_gep because array indexing is
         // always considered in bounds in GCC (FIXME(antoyo): to be verified).
         // So, we have to cast to a number.
-        let mut result = self.context.new_bitcast(self.location, ptr, self.sizet_type);
-        // FIXME(antoyo): if there were more than 1 index, this code is probably wrong and would
-        // require dereferencing the pointer.
-        for index in indices {
-            pointee_type = pointee_type.get_pointee().expect("pointee type");
-            #[cfg(feature = "master")]
-            let pointee_size = {
-                let size = self.cx.context.new_sizeof(pointee_type);
-                self.context.new_cast(self.location, size, index.get_type())
-            };
-            #[cfg(not(feature = "master"))]
-            let pointee_size =
-                self.context.new_rvalue_from_int(index.get_type(), pointee_type.get_size() as i32);
-            result = result + self.gcc_int_cast(*index * pointee_size, self.sizet_type);
-        }
+        let ptr = self.context.new_bitcast(self.location, ptr, self.sizet_type);
+        let pointee_size =
+            self.context.new_rvalue_from_int(index.get_type(), stride.bytes() as i32);
+        let result = ptr + self.gcc_int_cast(index * pointee_size, self.sizet_type);
         self.context.new_bitcast(self.location, result, ptr_type)
     }
 
     fn inbounds_gep(
         &mut self,
-        typ: Type<'gcc>,
+        stride: Size,
         ptr: RValue<'gcc>,
-        indices: &[RValue<'gcc>],
+        index: RValue<'gcc>,
     ) -> RValue<'gcc> {
+        let typ = self.cx.type_array(self.cx.type_i8(), stride.bytes());
         // NOTE: due to opaque pointers now being used, we need to cast here.
         let ptr = self.context.new_cast(self.location, ptr, typ.make_pointer());
         // NOTE: array indexing is always considered in bounds in GCC (FIXME(antoyo): to be verified).
-        let mut indices = indices.iter();
-        let index = indices.next().expect("first index in inbounds_gep");
-        let mut result = self.context.new_array_access(self.location, ptr, *index);
-        for index in indices {
-            result = self.context.new_array_access(self.location, result, *index);
-        }
+        let result = self.context.new_array_access(self.location, ptr, index);
         result.get_address(self.location)
     }
 

compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

@@ -11,7 +11,7 @@ use rustc_codegen_ssa::diagnostics::ExpectedPointerMutability;
 use rustc_codegen_ssa::diagnostics::InvalidMonomorphization;
 use rustc_codegen_ssa::mir::operand::OperandRef;
 use rustc_codegen_ssa::mir::place::PlaceRef;
-use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, LayoutTypeCodegenMethods};
+use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods};
 #[cfg(feature = "master")]
 use rustc_hir as hir;
 use rustc_middle::mir::BinOp;
@@ -671,7 +671,6 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
             );
         }
 
-        let pointee_type = bx.backend_type(layout);
         let pointers = args[0].immediate();
         let offsets = args[1].immediate();
         let elem_type = llret_ty.dyncast_vector().expect("vector return type").get_element_type();
@@ -680,7 +679,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 let index = bx.context.new_rvalue_from_long(bx.usize_type, i as _);
                 let pointer = bx.extract_element(pointers, index);
                 let offset = bx.extract_element(offsets, index);
-                let pointer = bx.gep(pointee_type, pointer, &[offset]);
+                let pointer = bx.gep(layout.size, pointer, offset);
                 // GCC has no pointer vectors, so the lanes are `usize`.
                 bx.ptrtoint(pointer, elem_type)
             })

compiler/rustc_codegen_llvm/src/builder.rs

@@ -160,7 +160,9 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
         val
     }
 
-    pub(crate) fn inbounds_gep(
+    /// Don't use this method: instead use `inbounds_gep` or `inbounds_ptradd`.
+    // FIXME: Remove once unused.
+    pub(crate) fn typed_inbounds_gep(
         &mut self,
         ty: &'ll Type,
         ptr: &'ll Value,
@@ -179,6 +181,33 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
         }
     }
 
+    fn build_gep(
+        &mut self,
+        stride: Size,
+        ptr: &'ll Value,
+        index: &'ll Value,
+        flags: GEPNoWrapFlags,
+    ) -> &'ll Value {
+        let ty = if stride.bytes() == 1 {
+            // LLVM prefers `i8` as the canonical form for a single-byte GEP.
+            self.cx.type_i8()
+        } else {
+            self.cx.type_array(self.cx.type_i8(), stride.bytes())
+        };
+        let indices = &[index];
+        unsafe {
+            llvm::LLVMBuildGEPWithNoWrapFlags(
+                self.llbuilder(),
+                ty,
+                ptr,
+                indices.as_ptr(),
+                indices.len() as c_uint,
+                UNNAMED,
+                flags,
+            )
+        }
+    }
+
     pub(crate) fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
         debug!("Store {:?} -> {:?}", val, ptr);
         assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
@@ -963,56 +992,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
-    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
-        unsafe {
-            llvm::LLVMBuildGEPWithNoWrapFlags(
-                self.llbuilder(),
-                ty,
-                ptr,
-                indices.as_ptr(),
-                indices.len() as c_uint,
-                UNNAMED,
-                GEPNoWrapFlags::default(),
-            )
-        }
+    fn gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
+        self.build_gep(stride, ptr, index, GEPNoWrapFlags::default())
     }
 
-    fn inbounds_gep(
-        &mut self,
-        ty: &'ll Type,
-        ptr: &'ll Value,
-        indices: &[&'ll Value],
-    ) -> &'ll Value {
-        unsafe {
-            llvm::LLVMBuildGEPWithNoWrapFlags(
-                self.llbuilder(),
-                ty,
-                ptr,
-                indices.as_ptr(),
-                indices.len() as c_uint,
-                UNNAMED,
-                GEPNoWrapFlags::InBounds,
-            )
-        }
+    fn inbounds_gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
+        self.build_gep(stride, ptr, index, GEPNoWrapFlags::InBounds)
     }
 
-    fn inbounds_nuw_gep(
-        &mut self,
-        ty: &'ll Type,
-        ptr: &'ll Value,
-        indices: &[&'ll Value],
-    ) -> &'ll Value {
-        unsafe {
-            llvm::LLVMBuildGEPWithNoWrapFlags(
-                self.llbuilder(),
-                ty,
-                ptr,
-                indices.as_ptr(),
-                indices.len() as c_uint,
-                UNNAMED,
-                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
-            )
-        }
+    fn inbounds_nuw_gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
+        self.build_gep(stride, ptr, index, GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW)
     }
 
     /* Casts */
@@ -1729,11 +1718,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             .val
             .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align));
 
-        let next = body_bx.inbounds_gep(
-            self.backend_type(cg_elem.layout),
-            current,
-            &[self.const_usize(1)],
-        );
+        let next = body_bx.inbounds_ptradd(current, self.const_usize(cg_elem.layout.size.bytes()));
         body_bx.br(header_bb);
         header_bx.add_incoming_to_phi(current, next, body_bb);
 

compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs

@@ -627,21 +627,21 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
             other => bug!("offload does not support {other:?}"),
         };
 
-        let gep = builder.inbounds_gep(cx.type_f32(), gep_base, &[i32_0]);
+        let gep = builder.typed_inbounds_gep(cx.type_f32(), gep_base, &[i32_0]);
 
         vals.push(base_val);
         geps.push(gep);
     }
 
     for i in 0..num_args {
         let idx = cx.get_const_i32(i);
-        let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, idx]);
+        let gep1 = builder.typed_inbounds_gep(ty, a1, &[i32_0, idx]);
         builder.store(vals[i as usize], gep1, Align::EIGHT);
-        let gep2 = builder.inbounds_gep(ty, a2, &[i32_0, idx]);
+        let gep2 = builder.typed_inbounds_gep(ty, a2, &[i32_0, idx]);
         builder.store(geps[i as usize], gep2, Align::EIGHT);
 
         if !matches!(metadata[i as usize].payload_size, OffloadSize::Static(_)) {
-            let gep3 = builder.inbounds_gep(ty2, a4, &[i32_0, idx]);
+            let gep3 = builder.typed_inbounds_gep(ty2, a4, &[i32_0, idx]);
             let size_val = get_runtime_size(builder, args, i as usize, &metadata[i as usize]);
             builder.store(size_val, gep3, Align::EIGHT);
         }
@@ -661,9 +661,10 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
         let cx = builder.cx;
         let i32_0 = cx.get_const_i32(0);
 
-        let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, i32_0]);
-        let gep2 = builder.inbounds_gep(ty, a2, &[i32_0, i32_0]);
-        let gep3 = if is_dynamic { builder.inbounds_gep(ty2, a4, &[i32_0, i32_0]) } else { a4 };
+        let gep1 = builder.typed_inbounds_gep(ty, a1, &[i32_0, i32_0]);
+        let gep2 = builder.typed_inbounds_gep(ty, a2, &[i32_0, i32_0]);
+        let gep3 =
+            if is_dynamic { builder.typed_inbounds_gep(ty2, a4, &[i32_0, i32_0]) } else { a4 };
         [gep1, gep2, gep3]
     }
 
@@ -710,7 +711,8 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
     // Step 3)
     // Here we fill the KernelArgsTy, see the documentation above
     for (i, value) in values.iter().enumerate() {
-        let ptr = builder.inbounds_gep(tgt_kernel_decl, a5, &[i32_0, cx.get_const_i32(i as u64)]);
+        let ptr =
+            builder.typed_inbounds_gep(tgt_kernel_decl, a5, &[i32_0, cx.get_const_i32(i as u64)]);
         let name = std::ffi::CString::new(value.1).unwrap();
         llvm::set_value_name(ptr, &name.as_bytes());
 

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -3397,7 +3397,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         }
         let offsets = args[1].immediate();
 
-        return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets]));
+        return Ok(bx.gep(layout.size, ptrs, offsets));
     }
 
     if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -207,7 +207,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let layout = bx.layout_of(ty);
                 let ptr = args[0].immediate();
                 let offset = args[1].immediate();
-                OperandValue::Immediate(bx.gep(bx.backend_type(layout), ptr, &[offset]))
+                OperandValue::Immediate(bx.gep(layout.size, ptr, offset))
             }
             sym::copy => {
                 copy_intrinsic(

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -292,7 +292,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
             layout.size
         };
 
-        let llval = bx.inbounds_nuw_gep(bx.cx().backend_type(layout), self.val.llval, &[llindex]);
+        let llval = bx.inbounds_nuw_gep(layout.size, self.val.llval, llindex);
         let align = self.val.align.restrict_for_offset(offset);
         PlaceValue::new_sized(llval, align).with_type(layout)
     }

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -970,11 +970,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     // so offsetting a pointer to ZST is a noop.
                     lhs
                 } else {
-                    let llty = bx.cx().backend_type(pointee_layout);
                     if !rhs_ty.is_signed() {
-                        bx.inbounds_nuw_gep(llty, lhs, &[rhs])
+                        bx.inbounds_nuw_gep(pointee_layout.size, lhs, rhs)
                     } else {
-                        bx.inbounds_gep(llty, lhs, &[rhs])
+                        bx.inbounds_gep(pointee_layout.size, lhs, rhs)
                     }
                 }
             }

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -350,26 +350,23 @@ pub trait BuilderMethods<'a, 'tcx>:
         size: Size,
     );
 
-    fn gep(&mut self, ty: Self::Type, ptr: Self::Value, indices: &[Self::Value]) -> Self::Value;
-    fn inbounds_gep(
-        &mut self,
-        ty: Self::Type,
-        ptr: Self::Value,
-        indices: &[Self::Value],
-    ) -> Self::Value;
+    fn gep(&mut self, stride: Size, ptr: Self::Value, index: Self::Value) -> Self::Value;
+    fn inbounds_gep(&mut self, stride: Size, ptr: Self::Value, index: Self::Value) -> Self::Value {
+        self.gep(stride, ptr, index)
+    }
     fn inbounds_nuw_gep(
         &mut self,
-        ty: Self::Type,
+        stride: Size,
         ptr: Self::Value,
-        indices: &[Self::Value],
+        index: Self::Value,
     ) -> Self::Value {
-        self.inbounds_gep(ty, ptr, indices)
+        self.inbounds_gep(stride, ptr, index)
     }
     fn ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
-        self.gep(self.cx().type_i8(), ptr, &[offset])
+        self.gep(Size::from_bytes(1), ptr, offset)
     }
     fn inbounds_ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
-        self.inbounds_gep(self.cx().type_i8(), ptr, &[offset])
+        self.inbounds_gep(Size::from_bytes(1), ptr, offset)
     }
 
     fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;

tests/codegen-llvm/intrinsics/offset.rs

@@ -18,7 +18,7 @@ pub unsafe fn offset_zst(p: *const (), d: usize) -> *const () {
 // CHECK-SAME: (ptr noundef %p, [[SIZE]] noundef %d)
 #[no_mangle]
 pub unsafe fn offset_isize(p: *const u32, d: isize) -> *const u32 {
-    // CHECK: %[[R:.*]] = getelementptr inbounds i32, ptr %p, [[SIZE]] %d
+    // CHECK: %[[R:.*]] = getelementptr inbounds [4 x i8], ptr %p, [[SIZE]] %d
     // CHECK-NEXT: ret ptr %[[R]]
     offset(p, d)
 }
@@ -27,7 +27,7 @@ pub unsafe fn offset_isize(p: *const u32, d: isize) -> *const u32 {
 // CHECK-SAME: (ptr noundef %p, [[SIZE]] noundef %d)
 #[no_mangle]
 pub unsafe fn offset_usize(p: *const u64, d: usize) -> *const u64 {
-    // CHECK: %[[R:.*]] = getelementptr inbounds{{( nuw)?}} i64, ptr %p, [[SIZE]] %d
+    // CHECK: %[[R:.*]] = getelementptr inbounds{{( nuw)?}} [8 x i8], ptr %p, [[SIZE]] %d
     // CHECK-NEXT: ret ptr %[[R]]
     offset(p, d)
 }