[RFC] Introducing __builtin_consistent to generate AArch64 BC.cond …

bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp

@@ -962,7 +962,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     if (isTB(Inst) || isCB(Inst)) {
       Inst.setOpcode(getInvertedBranchOpcode(Inst.getOpcode()));
       assert(Inst.getOpcode() != 0 && "Invalid branch instruction");
-    } else if (Inst.getOpcode() == AArch64::Bcc) {
+    } else if (Inst.getOpcode() == AArch64::Bcc ||
+               Inst.getOpcode() == AArch64::BCcc) {
       Inst.getOperand(0).setImm(AArch64CC::getInvertedCondCode(
           static_cast<AArch64CC::CondCode>(Inst.getOperand(0).getImm())));
       assert(Inst.getOperand(0).getImm() != AArch64CC::AL &&
@@ -991,6 +992,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     case AArch64::B:        return 28;
     case AArch64::BL:       return 28;
     case AArch64::Bcc:      return 21;
+    case AArch64::BCcc:
+      return 21;
     }
   }
 

clang/docs/LanguageExtensions.rst

@@ -3102,6 +3102,35 @@ flow conditions such as in ``if`` and ``switch`` statements.
 
 Query for this feature with ``__has_builtin(__builtin_unpredictable)``.
 
+``__builtin_consistent``
+------------------------
+
+``__builtin_consistent`` is used to indicate that the value of an expression is
+very likely to be consistent, e.g. branch having expression as condition will
+behave very consistently and is very unlikely to change direction.
+
+**Syntax**:
+
+.. code-block:: c++
+
+    __builtin_consistent(long long)
+
+**Example of use**:
+
+.. code-block:: c++
+
+  if (__builtin_consistent(x > 0)) {
+    foo();
+  }
+
+**Description**:
+
+The ``__builtin_consistent()`` builtin used with control flow conditions will
+provide information about branch consistently behavior which can be used to
+generate more efficient conditional branch instructions if target supports them
+(like AArch64 FEAT_HBC ``BC.cond``).
+
+Query for this feature with ``__has_builtin(__builtin_consistent)``.
 
 ``__builtin_expect``
 --------------------

clang/include/clang/Basic/Builtins.def

@@ -667,8 +667,9 @@ BUILTIN(__builtin___printf_chk, "iicC*R.", "Fp:1:")
 BUILTIN(__builtin___vfprintf_chk, "iP*RicC*Ra", "FP:2:")
 BUILTIN(__builtin___vprintf_chk, "iicC*Ra", "FP:1:")
 
-BUILTIN(__builtin_unpredictable, "LiLi"   , "nc")
-BUILTIN(__builtin_expect, "LiLiLi"   , "ncE")
+BUILTIN(__builtin_unpredictable, "LiLi", "nc")
+BUILTIN(__builtin_consistent, "LiLi", "nc")
+BUILTIN(__builtin_expect, "LiLiLi", "ncE")
 BUILTIN(__builtin_expect_with_probability, "LiLiLid", "ncE")
 BUILTIN(__builtin_prefetch, "vvC*.", "nc")
 BUILTIN(__builtin_readcyclecounter, "ULLi", "n")

clang/lib/Analysis/CalledOnceCheck.cpp

@@ -365,6 +365,7 @@ class DeclRefFinder
     }
 
     case Builtin::BI__builtin_unpredictable:
+    case Builtin::BI__builtin_consistent:
       return Visit(CE->getArg(0));
 
     default:

clang/lib/CodeGen/CGBuiltin.cpp

@@ -3137,10 +3137,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                                      "cast");
     return RValue::get(Result);
   }
-  case Builtin::BI__builtin_unpredictable: {
-    // Always return the argument of __builtin_unpredictable. LLVM does not
-    // handle this builtin. Metadata for this builtin should be added directly
-    // to instructions such as branches or switches that use it.
+  case Builtin::BI__builtin_unpredictable:
+  case Builtin::BI__builtin_consistent: {
+    // Always return the argument of __builtin_unpredictable and
+    // __builtin_consistent. LLVM does not handle these builtins. Metadata for
+    // these builtins should be added directly to instructions such as branches
+    // or switches that use it.
     return RValue::get(EmitScalarExpr(E->getArg(0)));
   }
   case Builtin::BI__builtin_expect: {

clang/lib/CodeGen/CGStmt.cpp

@@ -2059,16 +2059,21 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
   EmitBlock(SwitchExit.getBlock(), true);
   incrementProfileCounter(&S);
 
-  // If the switch has a condition wrapped by __builtin_unpredictable,
-  // create metadata that specifies that the switch is unpredictable.
-  // Don't bother if not optimizing because that metadata would not be used.
+  // If the switch has a condition wrapped by __builtin_unpredictable or
+  // __builtin_consistent, create metadata that specifies that the switch is
+  // unpredictable or consistent correspondingly. Don't bother if not optimizing
+  // because that metadata would not be used.
   auto *Call = dyn_cast<CallExpr>(S.getCond());
   if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
     auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
-    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
+    if (FD) {
       llvm::MDBuilder MDHelper(getLLVMContext());
-      SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable,
-                              MDHelper.createUnpredictable());
+      if (FD->getBuiltinID() == Builtin::BI__builtin_unpredictable)
+        SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable,
+                                MDHelper.createUnpredictable());
+      if (FD->getBuiltinID() == Builtin::BI__builtin_consistent)
+        SwitchInsn->setMetadata(llvm::LLVMContext::MD_consistent,
+                                MDHelper.createConsistent());
     }
   }
 

clang/lib/CodeGen/CodeGenFunction.cpp

@@ -1908,16 +1908,21 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
 
   llvm::MDNode *Weights = nullptr;
   llvm::MDNode *Unpredictable = nullptr;
+  llvm::MDNode *Consistent = nullptr;
 
-  // If the branch has a condition wrapped by __builtin_unpredictable,
-  // create metadata that specifies that the branch is unpredictable.
-  // Don't bother if not optimizing because that metadata would not be used.
+  // If the branch has a condition wrapped by __builtin_unpredictable or
+  // __builtin_consistent, create metadata that specifies that the branch is
+  // unpredictable or consistent correspondingly. Don't bother if not optimizing
+  // because that metadata would not be used.
   auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
   if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
     auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
-    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
+    if (FD) {
       llvm::MDBuilder MDHelper(getLLVMContext());
-      Unpredictable = MDHelper.createUnpredictable();
+      if (FD->getBuiltinID() == Builtin::BI__builtin_unpredictable)
+        Unpredictable = MDHelper.createUnpredictable();
+      if (FD->getBuiltinID() == Builtin::BI__builtin_consistent)
+        Consistent = MDHelper.createConsistent();
     }
   }
 
@@ -1932,7 +1937,8 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
     Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
   }
 
-  Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
+  Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable,
+                       Consistent);
 }
 
 /// ErrorUnsupported - Print out an error that codegen doesn't support the

clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp

@@ -63,12 +63,13 @@ bool BuiltinFunctionChecker::evalCall(const CallEvent &Call,
   }
 
   case Builtin::BI__builtin_unpredictable:
+  case Builtin::BI__builtin_consistent:
   case Builtin::BI__builtin_expect:
   case Builtin::BI__builtin_expect_with_probability:
   case Builtin::BI__builtin_assume_aligned:
   case Builtin::BI__builtin_addressof:
   case Builtin::BI__builtin_function_start: {
-    // For __builtin_unpredictable, __builtin_expect,
+    // For __builtin_unpredictable, __builtin_consistent, __builtin_expect,
     // __builtin_expect_with_probability and __builtin_assume_aligned,
     // just return the value of the subexpression.
     // __builtin_addressof is going from a reference to a pointer, but those

clang/test/CodeGen/builtin-consistent.c

@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple aarch64-unknown-unknown -emit-llvm -disable-llvm-passes -o - %s -O1 | FileCheck %s 
+// RUN: %clang_cc1 -triple aarch64-unknown-unknown -emit-llvm -o - %s -O0 | FileCheck %s --check-prefix=CHECK_O0
+
+void f(void);
+void g(void);
+void consistent_branch(int x) {
+// CHECK-LABEL: define{{.*}} void @consistent_branch(
+// CHECK-NOT: builtin_consistent
+// CHECK: !consistent [[METADATA:.+]]
+// CHECK_O0-NOT: builtin_consistent
+// CHECK_O0-NOT: !consistent 
+  if (__builtin_consistent(x > 0))
+    f();
+
+  if (x || __builtin_consistent(x != 0))
+    g();
+}
+
+int consistent_switch(int x) {
+// CHECK-LABEL: @consistent_switch(
+// CHECK-NOT: builtin_consistent
+// CHECK: !consistent [[METADATA:.+]]
+// CHECK_O0-NOT: builtin_consistent
+// CHECK_O0-NOT: !consistent 
+  switch(__builtin_consistent(x)) {
+  default:
+    return x;
+  case 0:
+  case 1:
+  case 2:
+    return 1;
+  case 3:
+    return x-1;
+  };
+}
+// CHECK: [[METADATA]] = !{i1 true}
+

llvm/include/llvm/CodeGen/MachineInstr.h

@@ -114,6 +114,7 @@ class MachineInstr
                              // this instruction.
     Unpredictable = 1 << 16, // Instruction with unpredictable condition.
     NoConvergent = 1 << 17,  // Call does not require convergence guarantees.
+    Consistent = 1 << 18,    // Instruction condition behaves consistently.
   };
 
 private:

llvm/include/llvm/CodeGen/SelectionDAGNodes.h

@@ -398,14 +398,17 @@ struct SDNodeFlags {
   bool NoFPExcept : 1;
   // Instructions with attached 'unpredictable' metadata on IR level.
   bool Unpredictable : 1;
+  // Instructions with attached 'consistent' metadata on IR level.
+  bool Consistent : 1;
 
 public:
   /// Default constructor turns off all optimization flags.
   SDNodeFlags()
       : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NonNeg(false),
         NoNaNs(false), NoInfs(false), NoSignedZeros(false),
         AllowReciprocal(false), AllowContract(false), ApproximateFuncs(false),
-        AllowReassociation(false), NoFPExcept(false), Unpredictable(false) {}
+        AllowReassociation(false), NoFPExcept(false), Unpredictable(false),
+        Consistent(false) {}
 
   /// Propagate the fast-math-flags from an IR FPMathOperator.
   void copyFMF(const FPMathOperator &FPMO) {
@@ -432,6 +435,7 @@ struct SDNodeFlags {
   void setAllowReassociation(bool b) { AllowReassociation = b; }
   void setNoFPExcept(bool b) { NoFPExcept = b; }
   void setUnpredictable(bool b) { Unpredictable = b; }
+  void setConsistent(bool b) { Consistent = b; }
 
   // These are accessors for each flag.
   bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
@@ -447,6 +451,7 @@ struct SDNodeFlags {
   bool hasAllowReassociation() const { return AllowReassociation; }
   bool hasNoFPExcept() const { return NoFPExcept; }
   bool hasUnpredictable() const { return Unpredictable; }
+  bool hasConsistent() const { return Consistent; }
 
   /// Clear any flags in this flag set that aren't also set in Flags. All
   /// flags will be cleared if Flags are undefined.
@@ -464,6 +469,7 @@ struct SDNodeFlags {
     AllowReassociation &= Flags.AllowReassociation;
     NoFPExcept &= Flags.NoFPExcept;
     Unpredictable &= Flags.Unpredictable;
+    Consistent &= Flags.Consistent;
   }
 };
 

llvm/include/llvm/CodeGen/TargetInstrInfo.h

@@ -698,7 +698,8 @@ class TargetInstrInfo : public MCInstrInfo {
   /// If \p BytesRemoved is non-null, report the change in code size from the
   /// removed instructions.
   virtual unsigned removeBranch(MachineBasicBlock &MBB,
-                                int *BytesRemoved = nullptr) const {
+                                int *BytesRemoved = nullptr,
+                                bool *IsConsistent = nullptr) const {
     llvm_unreachable("Target didn't implement TargetInstrInfo::removeBranch!");
   }
 
@@ -718,8 +719,8 @@ class TargetInstrInfo : public MCInstrInfo {
   virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                                 MachineBasicBlock *FBB,
                                 ArrayRef<MachineOperand> Cond,
-                                const DebugLoc &DL,
-                                int *BytesAdded = nullptr) const {
+                                const DebugLoc &DL, int *BytesAdded = nullptr,
+                                bool IsConsistent = false) const {
     llvm_unreachable("Target didn't implement TargetInstrInfo::insertBranch!");
   }
 

llvm/include/llvm/IR/FixedMetadataKinds.def

@@ -51,3 +51,4 @@ LLVM_FIXED_MD_KIND(MD_kcfi_type, "kcfi_type", 36)
 LLVM_FIXED_MD_KIND(MD_pcsections, "pcsections", 37)
 LLVM_FIXED_MD_KIND(MD_DIAssignID, "DIAssignID", 38)
 LLVM_FIXED_MD_KIND(MD_coro_outside_frame, "coro.outside.frame", 39)
+LLVM_FIXED_MD_KIND(MD_consistent, "consistent", 40)

llvm/include/llvm/IR/IRBuilder.h

@@ -1064,15 +1064,18 @@ class IRBuilderBase {
   //===--------------------------------------------------------------------===//
 
 private:
-  /// Helper to add branch weight and unpredictable metadata onto an
-  /// instruction.
+  /// Helper to add branch weight, unpredictable and consistent metadata onto
+  /// an instruction.
   /// \returns The annotated instruction.
   template <typename InstTy>
-  InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) {
+  InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpred,
+                            MDNode *Consist) {
     if (Weights)
       I->setMetadata(LLVMContext::MD_prof, Weights);
-    if (Unpredictable)
-      I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
+    if (Unpred)
+      I->setMetadata(LLVMContext::MD_unpredictable, Unpred);
+    if (Consist)
+      I->setMetadata(LLVMContext::MD_consistent, Consist);
     return I;
   }
 
@@ -1110,9 +1113,10 @@ class IRBuilderBase {
   /// instruction.
   BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
                            MDNode *BranchWeights = nullptr,
-                           MDNode *Unpredictable = nullptr) {
+                           MDNode *Unpredictable = nullptr,
+                           MDNode *Consistent = nullptr) {
     return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond),
-                                    BranchWeights, Unpredictable));
+                                    BranchWeights, Unpredictable, Consistent));
   }
 
   /// Create a conditional 'br Cond, TrueDest, FalseDest'
@@ -1121,7 +1125,8 @@ class IRBuilderBase {
                            Instruction *MDSrc) {
     BranchInst *Br = BranchInst::Create(True, False, Cond);
     if (MDSrc) {
-      unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
+      unsigned WL[5] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
+                        LLVMContext::MD_consistent,
                         LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
       Br->copyMetadata(*MDSrc, WL);
     }
@@ -1133,9 +1138,10 @@ class IRBuilderBase {
   /// allocation).
   SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10,
                            MDNode *BranchWeights = nullptr,
-                           MDNode *Unpredictable = nullptr) {
+                           MDNode *Unpredictable = nullptr,
+                           MDNode *Consistent = nullptr) {
     return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases),
-                                    BranchWeights, Unpredictable));
+                                    BranchWeights, Unpredictable, Consistent));
   }
 
   /// Create an indirect branch instruction with the specified address

llvm/include/llvm/IR/MDBuilder.h

@@ -67,6 +67,9 @@ class MDBuilder {
   /// Return metadata specifying that a branch or switch is unpredictable.
   MDNode *createUnpredictable();
 
+  /// Return metadata specifying that a branch or switch behaves consistently.
+  MDNode *createConsistent();
+
   /// Return metadata containing the entry \p Count for a function, a boolean
   /// \Synthetic indicating whether the counts were synthetized, and the
   /// GUIDs stored in \p Imports that need to be imported for sample PGO, to