[SystemZ] Simplify handling of AtomicRMW instructions. (#74789)

Let the AtomicExpand pass do more of the job of expanding
AtomicRMWInst:s in order to simplify the handling in the backend.

The only cases that the backend needs to handle itself are those of
subword size (8/16 bits) and those directly corresponding to a target
instruction.

Author: JonPsson1

llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

@@ -748,13 +748,12 @@ class SystemZTargetLowering : public TargetLowering {
                                 bool ClearEven) const;
   MachineBasicBlock *emitAtomicLoadBinary(MachineInstr &MI,
                                           MachineBasicBlock *BB,
-                                          unsigned BinOpcode, unsigned BitSize,
+                                          unsigned BinOpcode,
                                           bool Invert = false) const;
   MachineBasicBlock *emitAtomicLoadMinMax(MachineInstr &MI,
                                           MachineBasicBlock *MBB,
                                           unsigned CompareOpcode,
-                                          unsigned KeepOldMask,
-                                          unsigned BitSize) const;
+                                          unsigned KeepOldMask) const;
   MachineBasicBlock *emitAtomicCmpSwapW(MachineInstr &MI,
                                         MachineBasicBlock *BB) const;
   MachineBasicBlock *emitMemMemWrapper(MachineInstr &MI, MachineBasicBlock *BB,

llvm/lib/Target/SystemZ/SystemZISelLowering.h

@@ -5327,30 +5327,6 @@ multiclass CondStores<RegisterOperand cls, SDPatternOperator store,
   }
 }
 
-// OPERATOR is ATOMIC_SWAP or an ATOMIC_LOAD_* operation.  PAT and OPERAND
-// describe the second (non-memory) operand.
-class AtomicLoadBinary<SDPatternOperator operator, RegisterOperand cls,
-                       dag pat, DAGOperand operand>
-  : Pseudo<(outs cls:$dst), (ins bdaddr20only:$ptr, operand:$src2),
-           [(set cls:$dst, (operator bdaddr20only:$ptr, pat))]> {
-  let Defs = [CC];
-  let Has20BitOffset = 1;
-  let mayLoad = 1;
-  let mayStore = 1;
-  let usesCustomInserter = 1;
-  let hasNoSchedulingInfo = 1;
-}
-
-// Specializations of AtomicLoadWBinary.
-class AtomicLoadBinaryReg32<SDPatternOperator operator>
-  : AtomicLoadBinary<operator, GR32, (i32 GR32:$src2), GR32>;
-class AtomicLoadBinaryImm32<SDPatternOperator operator, ImmOpWithPattern imm>
-  : AtomicLoadBinary<operator, GR32, (i32 imm:$src2), imm>;
-class AtomicLoadBinaryReg64<SDPatternOperator operator>
-  : AtomicLoadBinary<operator, GR64, (i64 GR64:$src2), GR64>;
-class AtomicLoadBinaryImm64<SDPatternOperator operator, ImmOpWithPattern imm>
-  : AtomicLoadBinary<operator, GR64, (i64 imm:$src2), imm>;
-
 // OPERATOR is ATOMIC_SWAPW or an ATOMIC_LOADW_* operation.  PAT and OPERAND
 // describe the second (non-memory) operand.
 class AtomicLoadWBinary<SDPatternOperator operator, dag pat,

llvm/lib/Target/SystemZ/SystemZInstrFormats.td

@@ -1746,112 +1746,29 @@ let Predicates = [FeatureInterlockedAccess1], Defs = [CC] in {
 }
 
 def ATOMIC_SWAPW   : AtomicLoadWBinaryReg<z_atomic_swapw>;
-def ATOMIC_SWAP_32 : AtomicLoadBinaryReg32<atomic_swap_32>;
-def ATOMIC_SWAP_64 : AtomicLoadBinaryReg64<atomic_swap_64>;
 
 def ATOMIC_LOADW_AR  : AtomicLoadWBinaryReg<z_atomic_loadw_add>;
 def ATOMIC_LOADW_AFI : AtomicLoadWBinaryImm<z_atomic_loadw_add, simm32>;
-let Predicates = [FeatureNoInterlockedAccess1] in {
-  def ATOMIC_LOAD_AR   : AtomicLoadBinaryReg32<atomic_load_add_32>;
-  def ATOMIC_LOAD_AHI  : AtomicLoadBinaryImm32<atomic_load_add_32, imm32sx16>;
-  def ATOMIC_LOAD_AFI  : AtomicLoadBinaryImm32<atomic_load_add_32, simm32>;
-  def ATOMIC_LOAD_AGR  : AtomicLoadBinaryReg64<atomic_load_add_64>;
-  def ATOMIC_LOAD_AGHI : AtomicLoadBinaryImm64<atomic_load_add_64, imm64sx16>;
-  def ATOMIC_LOAD_AGFI : AtomicLoadBinaryImm64<atomic_load_add_64, imm64sx32>;
-}
 
 def ATOMIC_LOADW_SR : AtomicLoadWBinaryReg<z_atomic_loadw_sub>;
-def ATOMIC_LOAD_SR  : AtomicLoadBinaryReg32<atomic_load_sub_32>;
-def ATOMIC_LOAD_SGR : AtomicLoadBinaryReg64<atomic_load_sub_64>;
 
 def ATOMIC_LOADW_NR   : AtomicLoadWBinaryReg<z_atomic_loadw_and>;
 def ATOMIC_LOADW_NILH : AtomicLoadWBinaryImm<z_atomic_loadw_and, imm32lh16c>;
-let Predicates = [FeatureNoInterlockedAccess1] in {
-  def ATOMIC_LOAD_NR     : AtomicLoadBinaryReg32<atomic_load_and_32>;
-  def ATOMIC_LOAD_NILL   : AtomicLoadBinaryImm32<atomic_load_and_32,
-                                                 imm32ll16c>;
-  def ATOMIC_LOAD_NILH   : AtomicLoadBinaryImm32<atomic_load_and_32,
-                                                 imm32lh16c>;
-  def ATOMIC_LOAD_NILF   : AtomicLoadBinaryImm32<atomic_load_and_32, uimm32>;
-  def ATOMIC_LOAD_NGR    : AtomicLoadBinaryReg64<atomic_load_and_64>;
-  def ATOMIC_LOAD_NILL64 : AtomicLoadBinaryImm64<atomic_load_and_64,
-                                                 imm64ll16c>;
-  def ATOMIC_LOAD_NILH64 : AtomicLoadBinaryImm64<atomic_load_and_64,
-                                                 imm64lh16c>;
-  def ATOMIC_LOAD_NIHL64 : AtomicLoadBinaryImm64<atomic_load_and_64,
-                                                 imm64hl16c>;
-  def ATOMIC_LOAD_NIHH64 : AtomicLoadBinaryImm64<atomic_load_and_64,
-                                                 imm64hh16c>;
-  def ATOMIC_LOAD_NILF64 : AtomicLoadBinaryImm64<atomic_load_and_64,
-                                                 imm64lf32c>;
-  def ATOMIC_LOAD_NIHF64 : AtomicLoadBinaryImm64<atomic_load_and_64,
-                                                 imm64hf32c>;
-}
 
 def ATOMIC_LOADW_OR     : AtomicLoadWBinaryReg<z_atomic_loadw_or>;
 def ATOMIC_LOADW_OILH   : AtomicLoadWBinaryImm<z_atomic_loadw_or, imm32lh16>;
-let Predicates = [FeatureNoInterlockedAccess1] in {
-  def ATOMIC_LOAD_OR     : AtomicLoadBinaryReg32<atomic_load_or_32>;
-  def ATOMIC_LOAD_OILL   : AtomicLoadBinaryImm32<atomic_load_or_32, imm32ll16>;
-  def ATOMIC_LOAD_OILH   : AtomicLoadBinaryImm32<atomic_load_or_32, imm32lh16>;
-  def ATOMIC_LOAD_OILF   : AtomicLoadBinaryImm32<atomic_load_or_32, uimm32>;
-  def ATOMIC_LOAD_OGR    : AtomicLoadBinaryReg64<atomic_load_or_64>;
-  def ATOMIC_LOAD_OILL64 : AtomicLoadBinaryImm64<atomic_load_or_64, imm64ll16>;
-  def ATOMIC_LOAD_OILH64 : AtomicLoadBinaryImm64<atomic_load_or_64, imm64lh16>;
-  def ATOMIC_LOAD_OIHL64 : AtomicLoadBinaryImm64<atomic_load_or_64, imm64hl16>;
-  def ATOMIC_LOAD_OIHH64 : AtomicLoadBinaryImm64<atomic_load_or_64, imm64hh16>;
-  def ATOMIC_LOAD_OILF64 : AtomicLoadBinaryImm64<atomic_load_or_64, imm64lf32>;
-  def ATOMIC_LOAD_OIHF64 : AtomicLoadBinaryImm64<atomic_load_or_64, imm64hf32>;
-}
 
 def ATOMIC_LOADW_XR     : AtomicLoadWBinaryReg<z_atomic_loadw_xor>;
 def ATOMIC_LOADW_XILF   : AtomicLoadWBinaryImm<z_atomic_loadw_xor, uimm32>;
-let Predicates = [FeatureNoInterlockedAccess1] in {
-  def ATOMIC_LOAD_XR     : AtomicLoadBinaryReg32<atomic_load_xor_32>;
-  def ATOMIC_LOAD_XILF   : AtomicLoadBinaryImm32<atomic_load_xor_32, uimm32>;
-  def ATOMIC_LOAD_XGR    : AtomicLoadBinaryReg64<atomic_load_xor_64>;
-  def ATOMIC_LOAD_XILF64 : AtomicLoadBinaryImm64<atomic_load_xor_64, imm64lf32>;
-  def ATOMIC_LOAD_XIHF64 : AtomicLoadBinaryImm64<atomic_load_xor_64, imm64hf32>;
-}
 
 def ATOMIC_LOADW_NRi    : AtomicLoadWBinaryReg<z_atomic_loadw_nand>;
 def ATOMIC_LOADW_NILHi  : AtomicLoadWBinaryImm<z_atomic_loadw_nand,
                                                imm32lh16c>;
-def ATOMIC_LOAD_NRi     : AtomicLoadBinaryReg32<atomic_load_nand_32>;
-def ATOMIC_LOAD_NILLi   : AtomicLoadBinaryImm32<atomic_load_nand_32,
-                                                imm32ll16c>;
-def ATOMIC_LOAD_NILHi   : AtomicLoadBinaryImm32<atomic_load_nand_32,
-                                                imm32lh16c>;
-def ATOMIC_LOAD_NILFi   : AtomicLoadBinaryImm32<atomic_load_nand_32, uimm32>;
-def ATOMIC_LOAD_NGRi    : AtomicLoadBinaryReg64<atomic_load_nand_64>;
-def ATOMIC_LOAD_NILL64i : AtomicLoadBinaryImm64<atomic_load_nand_64,
-                                                imm64ll16c>;
-def ATOMIC_LOAD_NILH64i : AtomicLoadBinaryImm64<atomic_load_nand_64,
-                                                imm64lh16c>;
-def ATOMIC_LOAD_NIHL64i : AtomicLoadBinaryImm64<atomic_load_nand_64,
-                                                imm64hl16c>;
-def ATOMIC_LOAD_NIHH64i : AtomicLoadBinaryImm64<atomic_load_nand_64,
-                                                imm64hh16c>;
-def ATOMIC_LOAD_NILF64i : AtomicLoadBinaryImm64<atomic_load_nand_64,
-                                                imm64lf32c>;
-def ATOMIC_LOAD_NIHF64i : AtomicLoadBinaryImm64<atomic_load_nand_64,
-                                                imm64hf32c>;
 
 def ATOMIC_LOADW_MIN    : AtomicLoadWBinaryReg<z_atomic_loadw_min>;
-def ATOMIC_LOAD_MIN_32  : AtomicLoadBinaryReg32<atomic_load_min_32>;
-def ATOMIC_LOAD_MIN_64  : AtomicLoadBinaryReg64<atomic_load_min_64>;
-
 def ATOMIC_LOADW_MAX    : AtomicLoadWBinaryReg<z_atomic_loadw_max>;
-def ATOMIC_LOAD_MAX_32  : AtomicLoadBinaryReg32<atomic_load_max_32>;
-def ATOMIC_LOAD_MAX_64  : AtomicLoadBinaryReg64<atomic_load_max_64>;
-
 def ATOMIC_LOADW_UMIN   : AtomicLoadWBinaryReg<z_atomic_loadw_umin>;
-def ATOMIC_LOAD_UMIN_32 : AtomicLoadBinaryReg32<atomic_load_umin_32>;
-def ATOMIC_LOAD_UMIN_64 : AtomicLoadBinaryReg64<atomic_load_umin_64>;
-
 def ATOMIC_LOADW_UMAX   : AtomicLoadWBinaryReg<z_atomic_loadw_umax>;
-def ATOMIC_LOAD_UMAX_32 : AtomicLoadBinaryReg32<atomic_load_umax_32>;
-def ATOMIC_LOAD_UMAX_64 : AtomicLoadBinaryReg64<atomic_load_umax_64>;
 
 def ATOMIC_CMP_SWAPW
   : Pseudo<(outs GR32:$dst), (ins bdaddr20only:$addr, GR32:$cmp, GR32:$swap,

llvm/lib/Target/SystemZ/SystemZInstrInfo.td

@@ -16,96 +16,77 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) {
   ret i64 %res
 }
 
-; Check addition of 1, which can use AGHI.
+; Check addition of 1.
 define i64 @f2(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f2:
 ; CHECK: lg %r2, 0(%r3)
 ; CHECK: [[LABEL:\.[^:]*]]:
-; CHECK: lgr %r0, %r2
-; CHECK: aghi %r0, 1
+; CHECK: la %r0, 1(%r2)
 ; CHECK: csg %r2, %r0, 0(%r3)
 ; CHECK: jl [[LABEL]]
 ; CHECK: br %r14
   %res = atomicrmw add ptr %src, i64 1 seq_cst
   ret i64 %res
 }
 
-; Check the high end of the AGHI range.
+; Check use of LAY.
 define i64 @f3(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f3:
-; CHECK: aghi %r0, 32767
+; CHECK: lay %r0, 32767(%r2)
 ; CHECK: br %r14
   %res = atomicrmw add ptr %src, i64 32767 seq_cst
   ret i64 %res
 }
 
-; Check the next value up, which must use AGFI.
+; Check the high end of the AGFI range.
 define i64 @f4(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f4:
-; CHECK: agfi %r0, 32768
+; CHECK: agfi %r0, 2147483647
 ; CHECK: br %r14
-  %res = atomicrmw add ptr %src, i64 32768 seq_cst
+  %res = atomicrmw add ptr %src, i64 2147483647 seq_cst
   ret i64 %res
 }
 
-; Check the high end of the AGFI range.
+; Check the next value up, which uses an ALGFI.
 define i64 @f5(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f5:
-; CHECK: agfi %r0, 2147483647
+; CHECK: algfi %r0, 2147483648
 ; CHECK: br %r14
-  %res = atomicrmw add ptr %src, i64 2147483647 seq_cst
+  %res = atomicrmw add ptr %src, i64 2147483648 seq_cst
   ret i64 %res
 }
 
-; Check the next value up, which must use a register addition.
+; Check addition of -1, which can use LAY.
 define i64 @f6(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f6:
-; CHECK: agr
+; CHECK: lay %r0, -1(%r2)
 ; CHECK: br %r14
-  %res = atomicrmw add ptr %src, i64 2147483648 seq_cst
+  %res = atomicrmw add ptr %src, i64 -1 seq_cst
   ret i64 %res
 }
 
-; Check addition of -1, which can use AGHI.
+; LAY still OK.
 define i64 @f7(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f7:
-; CHECK: aghi %r0, -1
-; CHECK: br %r14
-  %res = atomicrmw add ptr %src, i64 -1 seq_cst
-  ret i64 %res
-}
-
-; Check the low end of the AGHI range.
-define i64 @f8(i64 %dummy, ptr %src) {
-; CHECK-LABEL: f8:
-; CHECK: aghi %r0, -32768
-; CHECK: br %r14
-  %res = atomicrmw add ptr %src, i64 -32768 seq_cst
-  ret i64 %res
-}
-
-; Check the next value down, which must use AGFI instead.
-define i64 @f9(i64 %dummy, ptr %src) {
-; CHECK-LABEL: f9:
-; CHECK: agfi %r0, -32769
+; CHECK: lay %r0, -32769(%r2)
 ; CHECK: br %r14
   %res = atomicrmw add ptr %src, i64 -32769 seq_cst
   ret i64 %res
 }
 
 ; Check the low end of the AGFI range.
-define i64 @f10(i64 %dummy, ptr %src) {
-; CHECK-LABEL: f10:
+define i64 @f8(i64 %dummy, ptr %src) {
+; CHECK-LABEL: f8:
 ; CHECK: agfi %r0, -2147483648
 ; CHECK: br %r14
   %res = atomicrmw add ptr %src, i64 -2147483648 seq_cst
   ret i64 %res
 }
 
-; Check the next value down, which must use a register addition.
-define i64 @f11(i64 %dummy, ptr %src) {
-; CHECK-LABEL: f11:
-; CHECK: agr
+; Check the next value down, which uses an SLGFI.
+define i64 @f9(i64 %dummy, ptr %src) {
+; CHECK-LABEL: f9:
+; CHECK: slgfi %r0, 2147483649
 ; CHECK: br %r14
   %res = atomicrmw add ptr %src, i64 -2147483649 seq_cst
   ret i64 %res

llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll

@@ -33,7 +33,7 @@ define i32 @f2(i32 %dummy, ptr %src) {
 ; Check ANDs of the low end of the NILH range.
 define i32 @f3(i32 %dummy, ptr %src) {
 ; CHECK-LABEL: f3:
-; CHECK: nilh %r0, 0
+; CHECK: llhr %r0, %r2
 ; CHECK: br %r14
   %res = atomicrmw and ptr %src, i32 65535 seq_cst
   ret i32 %res

llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll

@@ -16,11 +16,10 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) {
   ret i64 %res
 }
 
-; Check ANDs of 1, which are done using a register.  (We could use RISBG
-; instead, but that isn't implemented yet.)
+; Check ANDs of 1, which are done using a register.
 define i64 @f2(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f2:
-; CHECK: ngr
+; CHECK: risbg
 ; CHECK: br %r14
   %res = atomicrmw and ptr %src, i64 1 seq_cst
   ret i64 %res
@@ -56,7 +55,7 @@ define i64 @f4(i64 %dummy, ptr %src) {
 ; Check the next value up, which must use a register.
 define i64 @f5(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f5:
-; CHECK: ngr
+; CHECK: risbg
 ; CHECK: br %r14
   %res = atomicrmw and ptr %src, i64 12884901888 seq_cst
   ret i64 %res
@@ -74,7 +73,7 @@ define i64 @f6(i64 %dummy, ptr %src) {
 ; Check the next value up, which must use a register.
 define i64 @f7(i64 %dummy, ptr %src) {
 ; CHECK-LABEL: f7:
-; CHECK: ngr
+; CHECK: risbg
 ; CHECK: br %r14
   %res = atomicrmw and ptr %src, i64 281474976710656 seq_cst
   ret i64 %res