@@ -5434,29 +5434,37 @@ void LoopVectorizationLegality::collectLoopUniforms() {
5434
5434
}
5435
5435
5436
5436
// For an instruction to be added into Worklist above, all its users inside
5437
- // the current loop should be already added into Worklist. This condition
5438
- // cannot be true for phi instructions which is always in a dependence loop.
5439
- // Because any instruction in the dependence cycle always depends on others
5440
- // in the cycle to be added into Worklist first, the result is no ones in
5441
- // the cycle will be added into Worklist in the end.
5442
- // That is why we process PHI separately.
5443
- for (auto &Induction : *getInductionVars ()) {
5444
- auto *PN = Induction.first ;
5445
- auto *UpdateV = PN->getIncomingValueForBlock (TheLoop->getLoopLatch ());
5446
- if (all_of (PN->users (),
5447
- [&](User *U) -> bool {
5448
- return U == UpdateV || isOutOfScope (U) ||
5449
- Worklist.count (cast<Instruction>(U));
5450
- }) &&
5451
- all_of (UpdateV->users (), [&](User *U) -> bool {
5452
- return U == PN || isOutOfScope (U) ||
5453
- Worklist.count (cast<Instruction>(U));
5454
- })) {
5455
- Worklist.insert (cast<Instruction>(PN));
5456
- Worklist.insert (cast<Instruction>(UpdateV));
5457
- DEBUG (dbgs () << " LV: Found uniform instruction: " << *PN << " \n " );
5458
- DEBUG (dbgs () << " LV: Found uniform instruction: " << *UpdateV << " \n " );
5459
- }
5437
+ // the loop should also be in Worklist. However, this condition cannot be
5438
+ // true for phi nodes that form a cyclic dependence. We must process phi
5439
+ // nodes separately. An induction variable will remain uniform if all users
5440
+ // of the induction variable and induction variable update remain uniform.
5441
+ for (auto &Induction : Inductions) {
5442
+ auto *Ind = Induction.first ;
5443
+ auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock (Latch));
5444
+
5445
+ // Determine if all users of the induction variable are uniform after
5446
+ // vectorization.
5447
+ auto UniformInd = all_of (Ind->users (), [&](User *U) -> bool {
5448
+ auto *I = cast<Instruction>(U);
5449
+ return I == IndUpdate || !TheLoop->contains (I) || Worklist.count (I);
5450
+ });
5451
+ if (!UniformInd)
5452
+ continue ;
5453
+
5454
+ // Determine if all users of the induction variable update instruction are
5455
+ // uniform after vectorization.
5456
+ auto UniformIndUpdate = all_of (IndUpdate->users (), [&](User *U) -> bool {
5457
+ auto *I = cast<Instruction>(U);
5458
+ return I == Ind || !TheLoop->contains (I) || Worklist.count (I);
5459
+ });
5460
+ if (!UniformIndUpdate)
5461
+ continue ;
5462
+
5463
+ // The induction variable and its update instruction will remain uniform.
5464
+ Worklist.insert (Ind);
5465
+ Worklist.insert (IndUpdate);
5466
+ DEBUG (dbgs () << " LV: Found uniform instruction: " << *Ind << " \n " );
5467
+ DEBUG (dbgs () << " LV: Found uniform instruction: " << *IndUpdate << " \n " );
5460
5468
}
5461
5469
5462
5470
Uniforms.insert (Worklist.begin (), Worklist.end ());
0 commit comments