diff --git a/llvm/lib/Target/R600/R600MachineScheduler.cpp b/llvm/lib/Target/R600/R600MachineScheduler.cpp index aeb2674f4e65..4c89124f46da 100644 --- a/llvm/lib/Target/R600/R600MachineScheduler.cpp +++ b/llvm/lib/Target/R600/R600MachineScheduler.cpp @@ -34,7 +34,7 @@ void R600SchedStrategy::initialize(ScheduleDAGMI *dag) { CurEmitted = 0; OccupedSlotsMask = 15; InstKindLimit[IDAlu] = TII->getMaxAlusPerClause(); - + InstKindLimit[IDOther] = 32; const AMDGPUSubtarget &ST = DAG->TM.getSubtarget(); InstKindLimit[IDFetch] = ST.getTexVTXClauseSize(); @@ -49,12 +49,12 @@ void R600SchedStrategy::MoveUnits(std::vector &QSrc, SUnit* R600SchedStrategy::pickNode(bool &IsTopNode) { SUnit *SU = 0; - IsTopNode = true; NextInstKind = IDOther; + IsTopNode = false; + // check if we might want to switch current clause type - bool AllowSwitchToAlu = (CurInstKind == IDOther) || - (CurEmitted >= InstKindLimit[CurInstKind]) || + bool AllowSwitchToAlu = (CurEmitted >= InstKindLimit[CurInstKind]) || (Available[CurInstKind].empty()); bool AllowSwitchFromAlu = (CurEmitted >= InstKindLimit[CurInstKind]) && (!Available[IDFetch].empty() || !Available[IDOther].empty()); @@ -86,10 +86,10 @@ SUnit* R600SchedStrategy::pickNode(bool &IsTopNode) { DEBUG( if (SU) { - dbgs() << "picked node: "; + dbgs() << " ** Pick node **\n"; SU->dump(DAG); } else { - dbgs() << "NO NODE "; + dbgs() << "NO NODE \n"; for (unsigned i = 0; i < DAG->SUnits.size(); i++) { const SUnit &S = DAG->SUnits[i]; if (!S.isScheduled) @@ -103,9 +103,6 @@ SUnit* R600SchedStrategy::pickNode(bool &IsTopNode) { void R600SchedStrategy::schedNode(SUnit *SU, bool IsTopNode) { - DEBUG(dbgs() << "scheduled: "); - DEBUG(SU->dump(DAG)); - if (NextInstKind != CurInstKind) { DEBUG(dbgs() << "Instruction Type Switch\n"); if (NextInstKind != IDAlu) @@ -141,19 +138,23 @@ void R600SchedStrategy::schedNode(SUnit *SU, bool IsTopNode) { if (CurInstKind != IDFetch) { MoveUnits(Pending[IDFetch], Available[IDFetch]); } - MoveUnits(Pending[IDOther], Available[IDOther]); } void R600SchedStrategy::releaseTopNode(SUnit *SU) { - int IK = getInstKind(SU); + DEBUG(dbgs() << "Top Releasing ";SU->dump(DAG);); - DEBUG(dbgs() << IK << " <= "); - DEBUG(SU->dump(DAG)); - - Pending[IK].push_back(SU); } void R600SchedStrategy::releaseBottomNode(SUnit *SU) { + DEBUG(dbgs() << "Bottom Releasing ";SU->dump(DAG);); + + int IK = getInstKind(SU); + // There is no export clause, we can schedule one as soon as its ready + if (IK == IDOther) + Available[IDOther].push_back(SU); + else + Pending[IK].push_back(SU); + } bool R600SchedStrategy::regBelongsToClass(unsigned Reg, @@ -169,18 +170,15 @@ R600SchedStrategy::AluKind R600SchedStrategy::getAluKind(SUnit *SU) const { MachineInstr *MI = SU->getInstr(); switch (MI->getOpcode()) { + case AMDGPU::PRED_X: + return AluPredX; case AMDGPU::INTERP_PAIR_XY: case AMDGPU::INTERP_PAIR_ZW: case AMDGPU::INTERP_VEC_LOAD: case AMDGPU::DOT_4: return AluT_XYZW; case AMDGPU::COPY: - if (TargetRegisterInfo::isPhysicalRegister(MI->getOperand(1).getReg())) { - // %vregX = COPY Tn_X is likely to be discarded in favor of an - // assignement of Tn_X to %vregX, don't considers it in scheduling - return AluDiscarded; - } - else if (MI->getOperand(1).isUndef()) { + if (MI->getOperand(1).isUndef()) { // MI will become a KILL, don't considers it in scheduling return AluDiscarded; } @@ -238,6 +236,7 @@ int R600SchedStrategy::getInstKind(SUnit* SU) { } switch (Opcode) { + case AMDGPU::PRED_X: case AMDGPU::COPY: case AMDGPU::CONST_COPY: case AMDGPU::INTERP_PAIR_XY: @@ -328,12 +327,18 @@ bool R600SchedStrategy::isAvailablesAluEmpty() const { return Pending[IDAlu].empty() && AvailableAlus[AluAny].empty() && AvailableAlus[AluT_XYZW].empty() && AvailableAlus[AluT_X].empty() && AvailableAlus[AluT_Y].empty() && AvailableAlus[AluT_Z].empty() && - AvailableAlus[AluT_W].empty() && AvailableAlus[AluDiscarded].empty(); + AvailableAlus[AluT_W].empty() && AvailableAlus[AluDiscarded].empty() && + AvailableAlus[AluPredX].empty(); } SUnit* R600SchedStrategy::pickAlu() { while (!isAvailablesAluEmpty()) { if (!OccupedSlotsMask) { + // Bottom up scheduling : predX must comes first + if (!AvailableAlus[AluPredX].empty()) { + OccupedSlotsMask = 15; + return PopInst(AvailableAlus[AluPredX]); + } // Flush physical reg copies (RA will discard them) if (!AvailableAlus[AluDiscarded].empty()) { OccupedSlotsMask = 15; @@ -345,7 +350,7 @@ SUnit* R600SchedStrategy::pickAlu() { return PopInst(AvailableAlus[AluT_XYZW]); } } - for (unsigned Chan = 0; Chan < 4; ++Chan) { + for (int Chan = 3; Chan > -1; --Chan) { bool isOccupied = OccupedSlotsMask & (1 << Chan); if (!isOccupied) { SUnit *SU = AttemptFillSlot(Chan); diff --git a/llvm/lib/Target/R600/R600MachineScheduler.h b/llvm/lib/Target/R600/R600MachineScheduler.h index c82ee49c78be..1db877db85d8 100644 --- a/llvm/lib/Target/R600/R600MachineScheduler.h +++ b/llvm/lib/Target/R600/R600MachineScheduler.h @@ -45,13 +45,13 @@ class R600SchedStrategy : public MachineSchedStrategy { AluT_Z, AluT_W, AluT_XYZW, + AluPredX, AluDiscarded, // LLVM Instructions that are going to be eliminated AluLast }; std::vector Available[IDLast], Pending[IDLast]; std::vector AvailableAlus[AluLast]; - std::vector FakeCopy; InstKind CurInstKind; int CurEmitted; diff --git a/llvm/lib/Target/R600/R600RegisterInfo.cpp b/llvm/lib/Target/R600/R600RegisterInfo.cpp index bbd7995d7d51..7d13420a31b2 100644 --- a/llvm/lib/Target/R600/R600RegisterInfo.cpp +++ b/llvm/lib/Target/R600/R600RegisterInfo.cpp @@ -25,7 +25,7 @@ R600RegisterInfo::R600RegisterInfo(AMDGPUTargetMachine &tm, : AMDGPURegisterInfo(tm, tii), TM(tm), TII(tii) - { } + { RCW.RegWeight = 0; RCW.WeightLimit = 0;} BitVector R600RegisterInfo::getReservedRegs(const MachineFunction &MF) const { BitVector Reserved(getNumRegs()); @@ -97,3 +97,7 @@ unsigned R600RegisterInfo::getSubRegFromChannel(unsigned Channel) const { } } +const RegClassWeight &R600RegisterInfo::getRegClassWeight( + const TargetRegisterClass *RC) const { + return RCW; +} diff --git a/llvm/lib/Target/R600/R600RegisterInfo.h b/llvm/lib/Target/R600/R600RegisterInfo.h index f9ca918f246b..1270a1eea606 100644 --- a/llvm/lib/Target/R600/R600RegisterInfo.h +++ b/llvm/lib/Target/R600/R600RegisterInfo.h @@ -26,6 +26,7 @@ class TargetInstrInfo; struct R600RegisterInfo : public AMDGPURegisterInfo { AMDGPUTargetMachine &TM; const TargetInstrInfo &TII; + RegClassWeight RCW; R600RegisterInfo(AMDGPUTargetMachine &tm, const TargetInstrInfo &tii); @@ -48,6 +49,8 @@ struct R600RegisterInfo : public AMDGPURegisterInfo { /// (e.g. getSubRegFromChannel(0) -> AMDGPU::sel_x) unsigned getSubRegFromChannel(unsigned Channel) const; + virtual const RegClassWeight &getRegClassWeight(const TargetRegisterClass *RC) const; + }; } // End namespace llvm diff --git a/llvm/test/CodeGen/R600/fabs.ll b/llvm/test/CodeGen/R600/fabs.ll index 85f2882289fa..17ac8951fb3f 100644 --- a/llvm/test/CodeGen/R600/fabs.ll +++ b/llvm/test/CodeGen/R600/fabs.ll @@ -1,6 +1,6 @@ ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s -;CHECK: MOV * T{{[0-9]+\.[XYZW], \|T[0-9]+\.[XYZW]\|}} +;CHECK: MOV * T{{[0-9]+\.[XYZW], \|PV\.[xyzw]\|}} define void @test() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/fadd.ll b/llvm/test/CodeGen/R600/fadd.ll index 9a672329e75c..821d329d615e 100644 --- a/llvm/test/CodeGen/R600/fadd.ll +++ b/llvm/test/CodeGen/R600/fadd.ll @@ -1,7 +1,7 @@ ; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s ; CHECK: @fadd_f32 -; CHECK: ADD * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +; CHECK: ADD * T{{[0-9]+\.[XYZW], PV\.[xyzw], PV\.[xyzw]}} define void @fadd_f32() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/fdiv.ll b/llvm/test/CodeGen/R600/fdiv.ll index 2e68e36be4d8..003590bb53dd 100644 --- a/llvm/test/CodeGen/R600/fdiv.ll +++ b/llvm/test/CodeGen/R600/fdiv.ll @@ -1,12 +1,12 @@ ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s ;CHECK: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +;CHECK: MUL_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} ;CHECK: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} ;CHECK: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -;CHECK: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -;CHECK: MUL_IEEE T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} ;CHECK: MUL_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} ;CHECK: MUL_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +;CHECK: RECIP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} ;CHECK: MUL_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} define void @test(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) { diff --git a/llvm/test/CodeGen/R600/floor.ll b/llvm/test/CodeGen/R600/floor.ll index 877d69a65b43..0a807b1130a7 100644 --- a/llvm/test/CodeGen/R600/floor.ll +++ b/llvm/test/CodeGen/R600/floor.ll @@ -1,6 +1,6 @@ ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s -;CHECK: FLOOR * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +;CHECK: FLOOR * T{{[0-9]+\.[XYZW], PV\.[xyzw]}} define void @test() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/fmad.ll b/llvm/test/CodeGen/R600/fmad.ll index 62001edc3aa5..861411548a15 100644 --- a/llvm/test/CodeGen/R600/fmad.ll +++ b/llvm/test/CodeGen/R600/fmad.ll @@ -1,6 +1,6 @@ ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s -;CHECK: MULADD_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +;CHECK: MULADD_IEEE * {{T[0-9]+\.[XYZW], PV\.[xyzw], PV.[xyzw], PV\.[xyzw]}} define void @test() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/fmax.ll b/llvm/test/CodeGen/R600/fmax.ll index 8b704e56484b..ef3daad226a6 100644 --- a/llvm/test/CodeGen/R600/fmax.ll +++ b/llvm/test/CodeGen/R600/fmax.ll @@ -1,6 +1,6 @@ ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s -;CHECK: MAX * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +;CHECK: MAX * T{{[0-9]+\.[XYZW], PV\.[xyzw], PV\.[xyzw]}} define void @test() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/fmin.ll b/llvm/test/CodeGen/R600/fmin.ll index 5e34b7c8902e..026481c26355 100644 --- a/llvm/test/CodeGen/R600/fmin.ll +++ b/llvm/test/CodeGen/R600/fmin.ll @@ -1,6 +1,6 @@ ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s -;CHECK: MIN * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +;CHECK: MIN * T{{[0-9]+\.[XYZW], PV\.[xyzw], PV\.[xyzw]}} define void @test() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/fmul.ll b/llvm/test/CodeGen/R600/fmul.ll index c29294632dc0..dbb6424e2cdc 100644 --- a/llvm/test/CodeGen/R600/fmul.ll +++ b/llvm/test/CodeGen/R600/fmul.ll @@ -1,7 +1,7 @@ ; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s ; CHECK: @fmul_f32 -; CHECK: MUL_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +; CHECK: MUL_IEEE * {{T[0-9]+\.[XYZW], PV\.[xyzw], PV\.[xyzw]}} define void @fmul_f32() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/fsub.ll b/llvm/test/CodeGen/R600/fsub.ll index f784cde46cd2..f88729ef352f 100644 --- a/llvm/test/CodeGen/R600/fsub.ll +++ b/llvm/test/CodeGen/R600/fsub.ll @@ -1,7 +1,7 @@ ; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s ; CHECK: @fsub_f32 -; CHECK: ADD * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], -T[0-9]+\.[XYZW]}} +; CHECK: ADD * T{{[0-9]+\.[XYZW], PV\.[xyzw], -PV\.[xyzw]}} define void @fsub_f32() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/llvm.AMDGPU.mul.ll b/llvm/test/CodeGen/R600/llvm.AMDGPU.mul.ll index cc0732b3fffd..69fbe580cb51 100644 --- a/llvm/test/CodeGen/R600/llvm.AMDGPU.mul.ll +++ b/llvm/test/CodeGen/R600/llvm.AMDGPU.mul.ll @@ -1,6 +1,6 @@ ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s -;CHECK: MUL NON-IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +;CHECK: MUL NON-IEEE * T{{[0-9]+\.[XYZW], PV\.[xyzw], PV\.[xyzw]}} define void @test() { %r0 = call float @llvm.R600.load.input(i32 0) diff --git a/llvm/test/CodeGen/R600/llvm.pow.ll b/llvm/test/CodeGen/R600/llvm.pow.ll index 91b774282906..3800abfc5c4e 100644 --- a/llvm/test/CodeGen/R600/llvm.pow.ll +++ b/llvm/test/CodeGen/R600/llvm.pow.ll @@ -1,7 +1,7 @@ ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s ;CHECK: LOG_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -;CHECK-NEXT: MUL NON-IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +;CHECK: MUL NON-IEEE * T{{[0-9]+\.[XYZW], PV\.[xyzw], T[0-9]+\.[XYZW]}} ;CHECK-NEXT: EXP_IEEE * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} define void @test() { diff --git a/llvm/test/CodeGen/R600/pv.ll b/llvm/test/CodeGen/R600/pv.ll index 37c3d9d7d6d1..5481d6d0f54c 100644 --- a/llvm/test/CodeGen/R600/pv.ll +++ b/llvm/test/CodeGen/R600/pv.ll @@ -1,7 +1,7 @@ ; RUN: llc < %s -march=r600 | FileCheck %s ;CHECK: DOT4 * T{{[0-9]\.W}} (MASKED) -;CHECK-NEXT: CNDGE T{{[0-9].[XYZW]}}, PV.x +;CHECK: CNDGE T{{[0-9].[XYZW]}}, PV.x define void @main() #0 { main_body: diff --git a/llvm/test/CodeGen/R600/r600-encoding.ll b/llvm/test/CodeGen/R600/r600-encoding.ll index c8040a1b4cd5..6ef3c3124b81 100644 --- a/llvm/test/CodeGen/R600/r600-encoding.ll +++ b/llvm/test/CodeGen/R600/r600-encoding.ll @@ -5,10 +5,10 @@ ; the VLIW4/5 GPUs. ; EG-CHECK: @test -; EG-CHECK: MUL_IEEE {{[ *TXYZW.,0-9]+}} ; encoding: [{{0x[0-9a-f]+,0x[0-9a-f]+,0x[0-9a-f]+,0x[0-9a-f]+,0x10,0x01,0x[0-9a-f]+,0x[0-9a-f]+}}] +; EG-CHECK: MUL_IEEE {{[ *TXYZWPVxyzw.,0-9]+}} ; encoding: [{{0x[0-9a-f]+,0x[0-9a-f]+,0x[0-9a-f]+,0x[0-9a-f]+,0x10,0x01,0x[0-9a-f]+,0x[0-9a-f]+}}] ; R600-CHECK: @test -; R600-CHECK: MUL_IEEE {{[ *TXYZW.,0-9]+}} ; encoding: [{{0x[0-9a-f]+,0x[0-9a-f]+,0x[0-9a-f]+,0x[0-9a-f]+,0x10,0x02,0x[0-9a-f]+,0x[0-9a-f]+}}] +; R600-CHECK: MUL_IEEE {{[ *TXYZWPVxyzw.,0-9]+}} ; encoding: [{{0x[0-9a-f]+,0x[0-9a-f]+,0x[0-9a-f]+,0x[0-9a-f]+,0x10,0x02,0x[0-9a-f]+,0x[0-9a-f]+}}] define void @test() { entry: diff --git a/llvm/test/CodeGen/R600/selectcc-opt.ll b/llvm/test/CodeGen/R600/selectcc-opt.ll index 02d935390423..7f568fc0e678 100644 --- a/llvm/test/CodeGen/R600/selectcc-opt.ll +++ b/llvm/test/CodeGen/R600/selectcc-opt.ll @@ -29,8 +29,10 @@ ENDIF: ; for the icmp instruction ; CHECK: @test_b +; CHECK: VTX_READ ; CHECK: SET{{[GTEQN]+}}_DX10 ; CHECK-NEXT: PRED_ +; CHECK-NEXT: ALU clause starting define void @test_b(i32 addrspace(1)* %out, float %in) { entry: %0 = fcmp ult float %in, 0.0 diff --git a/llvm/test/CodeGen/R600/vselect.ll b/llvm/test/CodeGen/R600/vselect.ll index 6e459df847e7..a74a114a331d 100644 --- a/llvm/test/CodeGen/R600/vselect.ll +++ b/llvm/test/CodeGen/R600/vselect.ll @@ -3,8 +3,8 @@ ; CHECK: @test_select_v4i32 ; CHECK: CNDE_INT T{{[0-9]+\.[XYZW], PV\.[xyzw], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} ; CHECK: CNDE_INT * T{{[0-9]+\.[XYZW], PV\.[xyzw], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -; CHECK: CNDE_INT * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -; CHECK: CNDE_INT * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +; CHECK: CNDE_INT T{{[0-9]+\.[XYZW], PV\.[xyzw], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} +; CHECK: CNDE_INT * T{{[0-9]+\.[XYZW], PV\.[xyzw], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} define void @test_select_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in0, <4 x i32> addrspace(1)* %in1) { entry: