From feec2aeb0feb51027dfb14f81da10a8df779ba15 Mon Sep 17 00:00:00 2001 From: Chad Rosier Date: Thu, 4 Feb 2016 14:42:55 +0000 Subject: [PATCH] [AArch64] Improve load/store optimizer to handle LDUR + LDR. This patch allows the mixing of scaled and unscaled load/stores to form load/store pairs. PR24465 http://reviews.llvm.org/D12116 Many thanks to Ahmed and Michael for fixes and code review. This is a reapplication of r246769, which was reverted in r246782 due to a test-suite failure. I'm unable to reproduce the issue at this time. llvm-svn: 259790 --- .../AArch64/AArch64LoadStoreOptimizer.cpp | 99 +++++++++++++---- .../AArch64/ldp-stp-scaled-unscaled-pairs.ll | 105 ++++++++++++++++++ 2 files changed, 182 insertions(+), 22 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp index 253d76f87c0f..010557a44abb 100644 --- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -645,10 +645,24 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I, const MachineOperand &BaseRegOp = MergeForward ? getLdStBaseOp(Paired) : getLdStBaseOp(I); + int Offset = getLdStOffsetOp(I).getImm(); + int PairedOffset = getLdStOffsetOp(Paired).getImm(); + bool PairedIsUnscaled = isUnscaledLdSt(Paired->getOpcode()); + + // We're trying to pair instructions that differ in how they are scaled. + // If I is scaled then scale the offset of Paired accordingly. + // Otherwise, do the opposite (i.e., make Paired's offset unscaled). + if (IsUnscaled != PairedIsUnscaled) { + int MemSize = getMemScale(Paired); + assert(!(PairedOffset % getMemScale(Paired)) && + "Offset should be a multiple of the stride!"); + PairedOffset = + PairedIsUnscaled ? PairedOffset / MemSize : PairedOffset * MemSize; + } + // Which register is Rt and which is Rt2 depends on the offset order. MachineInstr *RtMI, *Rt2MI; - if (getLdStOffsetOp(I).getImm() == - getLdStOffsetOp(Paired).getImm() + OffsetStride) { + if (Offset == PairedOffset + OffsetStride) { RtMI = Paired; Rt2MI = I; // Here we swapped the assumption made for SExtIdx. @@ -775,9 +789,12 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I, .addImm(OffsetImm) .setMemRefs(I->mergeMemRefsWith(*Paired)); } else { - // Handle Unscaled - if (IsUnscaled) - OffsetImm /= OffsetStride; + // Scale the immediate offset, if necessary. + if (isUnscaledLdSt(RtMI->getOpcode())) { + assert(!(OffsetImm % getMemScale(RtMI)) && + "Offset should be a multiple of the stride!"); + OffsetImm /= getMemScale(RtMI); + } MIB = BuildMI(*I->getParent(), InsertionPoint, I->getDebugLoc(), TII->get(NewOpc)) .addOperand(getLdStRegOp(RtMI)) @@ -971,9 +988,13 @@ static void trackRegDefsUses(const MachineInstr *MI, BitVector &ModifiedRegs, static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) { // Convert the byte-offset used by unscaled into an "element" offset used // by the scaled pair load/store instructions. - if (IsUnscaled) + if (IsUnscaled) { + // If the byte-offset isn't a multiple of the stride, there's no point + // trying to match it. + if (Offset % OffsetStride) + return false; Offset /= OffsetStride; - + } return Offset <= 63 && Offset >= -64; } @@ -1063,6 +1084,34 @@ bool AArch64LoadStoreOpt::findMatchingStore( return false; } + +static bool canMergeOpc(unsigned Opc, unsigned PairOpc, LdStPairFlags &Flags) { + // Opcodes match: nothing more to check. + if (Opc == PairOpc) + return true; + + // Try to match a sign-extended load/store with a zero-extended load/store. + Flags.setSExtIdx(-1); + bool IsValidLdStrOpc, PairIsValidLdStrOpc; + unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc, &IsValidLdStrOpc); + assert(IsValidLdStrOpc && + "Given Opc should be a Load or Store with an immediate"); + // Opc will be the first instruction in the pair. + if (NonSExtOpc == getMatchingNonSExtOpcode(PairOpc, &PairIsValidLdStrOpc)) { + Flags.setSExtIdx(NonSExtOpc == (unsigned)Opc ? 1 : 0); + return true; + } + + // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair? + + // If the second instruction isn't even a load/store, bail out. + if (!PairIsValidLdStrOpc) + return false; + + // Try to match an unscaled load/store with a scaled load/store. + return isUnscaledLdSt(Opc) != isUnscaledLdSt(PairOpc) && + getMatchingPairOpcode(Opc) == getMatchingPairOpcode(PairOpc); +} /// findMatchingInsn - Scan the instructions looking for a load/store that can /// be combined with the current instruction into a load/store pair. MachineBasicBlock::iterator @@ -1116,19 +1165,8 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I, // Now that we know this is a real instruction, count it. ++Count; - bool CanMergeOpc = Opc == MI->getOpcode(); - Flags.setSExtIdx(-1); - if (!CanMergeOpc) { - bool IsValidLdStrOpc; - unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc, &IsValidLdStrOpc); - assert(IsValidLdStrOpc && - "Given Opc should be a Load or Store with an immediate"); - // Opc will be the first instruction in the pair. - Flags.setSExtIdx(NonSExtOpc == (unsigned)Opc ? 1 : 0); - CanMergeOpc = NonSExtOpc == getMatchingNonSExtOpcode(MI->getOpcode()); - } - - if (CanMergeOpc && getLdStOffsetOp(MI).isImm()) { + if (canMergeOpc(Opc, MI->getOpcode(), Flags) && + getLdStOffsetOp(MI).isImm()) { assert(MI->mayLoadOrStore() && "Expected memory operation."); // If we've found another instruction with the same opcode, check to see // if the base and offset are compatible with our starting instruction. @@ -1142,6 +1180,24 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I, // final offset must be in range. unsigned MIBaseReg = getLdStBaseOp(MI).getReg(); int MIOffset = getLdStOffsetOp(MI).getImm(); + + // We're trying to pair instructions that differ in how they are scaled. + // If FirstMI is scaled then scale the offset of MI accordingly. + // Otherwise, do the opposite (i.e., make MI's offset unscaled). + bool MIIsUnscaled = isUnscaledLdSt(MI); + if (IsUnscaled != MIIsUnscaled) { + int MemSize = getMemScale(MI); + if (MIIsUnscaled) { + // If the unscaled offset isn't a multiple of the MemSize, we can't + // pair the operations together: bail and keep looking. + if (MIOffset % MemSize) + continue; + MIOffset /= MemSize; + } else { + MIOffset *= MemSize; + } + } + if (BaseReg == MIBaseReg && ((Offset == MIOffset + OffsetStride) || (Offset + OffsetStride == MIOffset))) { int MinOffset = Offset < MIOffset ? Offset : MIOffset; @@ -1152,10 +1208,9 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I, return E; // If the resultant immediate offset of merging these instructions // is out of range for a pairwise instruction, bail and keep looking. - bool MIIsUnscaled = isUnscaledLdSt(MI); bool IsNarrowLoad = isNarrowLoad(MI->getOpcode()); if (!IsNarrowLoad && - !inBoundsForPair(MIIsUnscaled, MinOffset, OffsetStride)) { + !inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) { trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI); MemInsns.push_back(MI); continue; diff --git a/llvm/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll b/llvm/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll new file mode 100644 index 000000000000..b118c279c8b0 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll @@ -0,0 +1,105 @@ +; RUN: llc < %s -march=aarch64 -aarch64-neon-syntax=apple -aarch64-stp-suppress=false -verify-machineinstrs -asm-verbose=false | FileCheck %s + +; CHECK-LABEL: test_strd_sturd: +; CHECK-NEXT: stp d0, d1, [x0, #-8] +; CHECK-NEXT: ret +define void @test_strd_sturd(float* %ptr, <2 x float> %v1, <2 x float> %v2) #0 { + %tmp1 = bitcast float* %ptr to <2 x float>* + store <2 x float> %v2, <2 x float>* %tmp1, align 16 + %add.ptr = getelementptr inbounds float, float* %ptr, i64 -2 + %tmp = bitcast float* %add.ptr to <2 x float>* + store <2 x float> %v1, <2 x float>* %tmp, align 16 + ret void +} + +; CHECK-LABEL: test_sturd_strd: +; CHECK-NEXT: stp d0, d1, [x0, #-8] +; CHECK-NEXT: ret +define void @test_sturd_strd(float* %ptr, <2 x float> %v1, <2 x float> %v2) #0 { + %add.ptr = getelementptr inbounds float, float* %ptr, i64 -2 + %tmp = bitcast float* %add.ptr to <2 x float>* + store <2 x float> %v1, <2 x float>* %tmp, align 16 + %tmp1 = bitcast float* %ptr to <2 x float>* + store <2 x float> %v2, <2 x float>* %tmp1, align 16 + ret void +} + +; CHECK-LABEL: test_strq_sturq: +; CHECK-NEXT: stp q0, q1, [x0, #-16] +; CHECK-NEXT: ret +define void @test_strq_sturq(double* %ptr, <2 x double> %v1, <2 x double> %v2) #0 { + %tmp1 = bitcast double* %ptr to <2 x double>* + store <2 x double> %v2, <2 x double>* %tmp1, align 16 + %add.ptr = getelementptr inbounds double, double* %ptr, i64 -2 + %tmp = bitcast double* %add.ptr to <2 x double>* + store <2 x double> %v1, <2 x double>* %tmp, align 16 + ret void +} + +; CHECK-LABEL: test_sturq_strq: +; CHECK-NEXT: stp q0, q1, [x0, #-16] +; CHECK-NEXT: ret +define void @test_sturq_strq(double* %ptr, <2 x double> %v1, <2 x double> %v2) #0 { + %add.ptr = getelementptr inbounds double, double* %ptr, i64 -2 + %tmp = bitcast double* %add.ptr to <2 x double>* + store <2 x double> %v1, <2 x double>* %tmp, align 16 + %tmp1 = bitcast double* %ptr to <2 x double>* + store <2 x double> %v2, <2 x double>* %tmp1, align 16 + ret void +} + +; CHECK-LABEL: test_ldrx_ldurx: +; CHECK-NEXT: ldp [[V0:x[0-9]+]], [[V1:x[0-9]+]], [x0, #-8] +; CHECK-NEXT: add x0, [[V0]], [[V1]] +; CHECK-NEXT: ret +define i64 @test_ldrx_ldurx(i64* %p) #0 { + %tmp = load i64, i64* %p, align 4 + %add.ptr = getelementptr inbounds i64, i64* %p, i64 -1 + %tmp1 = load i64, i64* %add.ptr, align 4 + %add = add nsw i64 %tmp1, %tmp + ret i64 %add +} + +; CHECK-LABEL: test_ldurx_ldrx: +; CHECK-NEXT: ldp [[V0:x[0-9]+]], [[V1:x[0-9]+]], [x0, #-8] +; CHECK-NEXT: add x0, [[V0]], [[V1]] +; CHECK-NEXT: ret +define i64 @test_ldurx_ldrx(i64* %p) #0 { + %add.ptr = getelementptr inbounds i64, i64* %p, i64 -1 + %tmp1 = load i64, i64* %add.ptr, align 4 + %tmp = load i64, i64* %p, align 4 + %add = add nsw i64 %tmp1, %tmp + ret i64 %add +} + +; CHECK-LABEL: test_ldrsw_ldursw: +; CHECK-NEXT: ldpsw [[V0:x[0-9]+]], [[V1:x[0-9]+]], [x0, #-4] +; CHECK-NEXT: add x0, [[V0]], [[V1]] +; CHECK-NEXT: ret +define i64 @test_ldrsw_ldursw(i32* %p) #0 { + %tmp = load i32, i32* %p, align 4 + %add.ptr = getelementptr inbounds i32, i32* %p, i64 -1 + %tmp1 = load i32, i32* %add.ptr, align 4 + %sexttmp = sext i32 %tmp to i64 + %sexttmp1 = sext i32 %tmp1 to i64 + %add = add nsw i64 %sexttmp1, %sexttmp + ret i64 %add +} + +; Also make sure we only match valid offsets. +; CHECK-LABEL: test_ldrq_ldruq_invalidoffset: +; CHECK-NEXT: ldr q[[V0:[0-9]+]], [x0] +; CHECK-NEXT: ldur q[[V1:[0-9]+]], [x0, #24] +; CHECK-NEXT: add.2d v0, v[[V0]], v[[V1]] +; CHECK-NEXT: ret +define <2 x i64> @test_ldrq_ldruq_invalidoffset(i64* %p) nounwind { + %a1 = bitcast i64* %p to <2 x i64>* + %tmp1 = load <2 x i64>, < 2 x i64>* %a1, align 8 + %add.ptr2 = getelementptr inbounds i64, i64* %p, i64 3 + %a2 = bitcast i64* %add.ptr2 to <2 x i64>* + %tmp2 = load <2 x i64>, <2 x i64>* %a2, align 8 + %add = add nsw <2 x i64> %tmp1, %tmp2 + ret <2 x i64> %add +} + +attributes #0 = { nounwind }