1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the code for emitting atomic operations.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCall.h"
14 #include "CGRecordLayout.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "clang/Frontend/FrontendDiagnostic.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30   class AtomicInfo {
31     CodeGenFunction &CGF;
32     QualType AtomicTy;
33     QualType ValueTy;
34     uint64_t AtomicSizeInBits;
35     uint64_t ValueSizeInBits;
36     CharUnits AtomicAlign;
37     CharUnits ValueAlign;
38     TypeEvaluationKind EvaluationKind;
39     bool UseLibcall;
40     LValue LVal;
41     CGBitFieldInfo BFI;
42   public:
43     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45           EvaluationKind(TEK_Scalar), UseLibcall(true) {
46       assert(!lvalue.isGlobalReg());
47       ASTContext &C = CGF.getContext();
48       if (lvalue.isSimple()) {
49         AtomicTy = lvalue.getType();
50         if (auto *ATy = AtomicTy->getAs<AtomicType>())
51           ValueTy = ATy->getValueType();
52         else
53           ValueTy = AtomicTy;
54         EvaluationKind = CGF.getEvaluationKind(ValueTy);
55 
56         uint64_t ValueAlignInBits;
57         uint64_t AtomicAlignInBits;
58         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59         ValueSizeInBits = ValueTI.Width;
60         ValueAlignInBits = ValueTI.Align;
61 
62         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63         AtomicSizeInBits = AtomicTI.Width;
64         AtomicAlignInBits = AtomicTI.Align;
65 
66         assert(ValueSizeInBits <= AtomicSizeInBits);
67         assert(ValueAlignInBits <= AtomicAlignInBits);
68 
69         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71         if (lvalue.getAlignment().isZero())
72           lvalue.setAlignment(AtomicAlign);
73 
74         LVal = lvalue;
75       } else if (lvalue.isBitField()) {
76         ValueTy = lvalue.getType();
77         ValueSizeInBits = C.getTypeSize(ValueTy);
78         auto &OrigBFI = lvalue.getBitFieldInfo();
79         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80         AtomicSizeInBits = C.toBits(
81             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82                 .alignTo(lvalue.getAlignment()));
83         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84         auto OffsetInChars =
85             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86             lvalue.getAlignment();
87         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88             CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
89         llvm::Type *IntTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
90         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
91             VoidPtrAddr, IntTy->getPointerTo(), "atomic_bitfield_base");
92         BFI = OrigBFI;
93         BFI.Offset = Offset;
94         BFI.StorageSize = AtomicSizeInBits;
95         BFI.StorageOffset += OffsetInChars;
96         LVal = LValue::MakeBitfield(Address(Addr, IntTy, lvalue.getAlignment()),
97                                     BFI, lvalue.getType(), lvalue.getBaseInfo(),
98                                     lvalue.getTBAAInfo());
99         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100         if (AtomicTy.isNull()) {
101           llvm::APInt Size(
102               /*numBits=*/32,
103               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104           AtomicTy =
105               C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
106                                      /*IndexTypeQuals=*/0);
107         }
108         AtomicAlign = ValueAlign = lvalue.getAlignment();
109       } else if (lvalue.isVectorElt()) {
110         ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
111         ValueSizeInBits = C.getTypeSize(ValueTy);
112         AtomicTy = lvalue.getType();
113         AtomicSizeInBits = C.getTypeSize(AtomicTy);
114         AtomicAlign = ValueAlign = lvalue.getAlignment();
115         LVal = lvalue;
116       } else {
117         assert(lvalue.isExtVectorElt());
118         ValueTy = lvalue.getType();
119         ValueSizeInBits = C.getTypeSize(ValueTy);
120         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121             lvalue.getType(), cast<llvm::FixedVectorType>(
122                                   lvalue.getExtVectorAddress().getElementType())
123                                   ->getNumElements());
124         AtomicSizeInBits = C.getTypeSize(AtomicTy);
125         AtomicAlign = ValueAlign = lvalue.getAlignment();
126         LVal = lvalue;
127       }
128       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130     }
131 
132     QualType getAtomicType() const { return AtomicTy; }
133     QualType getValueType() const { return ValueTy; }
134     CharUnits getAtomicAlignment() const { return AtomicAlign; }
135     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
136     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
137     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
138     bool shouldUseLibcall() const { return UseLibcall; }
139     const LValue &getAtomicLValue() const { return LVal; }
140     llvm::Value *getAtomicPointer() const {
141       if (LVal.isSimple())
142         return LVal.getPointer(CGF);
143       else if (LVal.isBitField())
144         return LVal.getBitFieldPointer();
145       else if (LVal.isVectorElt())
146         return LVal.getVectorPointer();
147       assert(LVal.isExtVectorElt());
148       return LVal.getExtVectorPointer();
149     }
150     Address getAtomicAddress() const {
151       llvm::Type *ElTy;
152       if (LVal.isSimple())
153         ElTy = LVal.getAddress(CGF).getElementType();
154       else if (LVal.isBitField())
155         ElTy = LVal.getBitFieldAddress().getElementType();
156       else if (LVal.isVectorElt())
157         ElTy = LVal.getVectorAddress().getElementType();
158       else
159         ElTy = LVal.getExtVectorAddress().getElementType();
160       return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
161     }
162 
163     Address getAtomicAddressAsAtomicIntPointer() const {
164       return emitCastToAtomicIntPointer(getAtomicAddress());
165     }
166 
167     /// Is the atomic size larger than the underlying value type?
168     ///
169     /// Note that the absence of padding does not mean that atomic
170     /// objects are completely interchangeable with non-atomic
171     /// objects: we might have promoted the alignment of a type
172     /// without making it bigger.
173     bool hasPadding() const {
174       return (ValueSizeInBits != AtomicSizeInBits);
175     }
176 
177     bool emitMemSetZeroIfNecessary() const;
178 
179     llvm::Value *getAtomicSizeValue() const {
180       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
181       return CGF.CGM.getSize(size);
182     }
183 
184     /// Cast the given pointer to an integer pointer suitable for atomic
185     /// operations if the source.
186     Address emitCastToAtomicIntPointer(Address Addr) const;
187 
188     /// If Addr is compatible with the iN that will be used for an atomic
189     /// operation, bitcast it. Otherwise, create a temporary that is suitable
190     /// and copy the value across.
191     Address convertToAtomicIntPointer(Address Addr) const;
192 
193     /// Turn an atomic-layout object into an r-value.
194     RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
195                                      SourceLocation loc, bool AsValue) const;
196 
197     /// Converts a rvalue to integer value.
198     llvm::Value *convertRValueToInt(RValue RVal) const;
199 
200     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
201                                      AggValueSlot ResultSlot,
202                                      SourceLocation Loc, bool AsValue) const;
203 
204     /// Copy an atomic r-value into atomic-layout memory.
205     void emitCopyIntoMemory(RValue rvalue) const;
206 
207     /// Project an l-value down to the value field.
208     LValue projectValue() const {
209       assert(LVal.isSimple());
210       Address addr = getAtomicAddress();
211       if (hasPadding())
212         addr = CGF.Builder.CreateStructGEP(addr, 0);
213 
214       return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
215                               LVal.getBaseInfo(), LVal.getTBAAInfo());
216     }
217 
218     /// Emits atomic load.
219     /// \returns Loaded value.
220     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
221                           bool AsValue, llvm::AtomicOrdering AO,
222                           bool IsVolatile);
223 
224     /// Emits atomic compare-and-exchange sequence.
225     /// \param Expected Expected value.
226     /// \param Desired Desired value.
227     /// \param Success Atomic ordering for success operation.
228     /// \param Failure Atomic ordering for failed operation.
229     /// \param IsWeak true if atomic operation is weak, false otherwise.
230     /// \returns Pair of values: previous value from storage (value type) and
231     /// boolean flag (i1 type) with true if success and false otherwise.
232     std::pair<RValue, llvm::Value *>
233     EmitAtomicCompareExchange(RValue Expected, RValue Desired,
234                               llvm::AtomicOrdering Success =
235                                   llvm::AtomicOrdering::SequentiallyConsistent,
236                               llvm::AtomicOrdering Failure =
237                                   llvm::AtomicOrdering::SequentiallyConsistent,
238                               bool IsWeak = false);
239 
240     /// Emits atomic update.
241     /// \param AO Atomic ordering.
242     /// \param UpdateOp Update operation for the current lvalue.
243     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
244                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
245                           bool IsVolatile);
246     /// Emits atomic update.
247     /// \param AO Atomic ordering.
248     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
249                           bool IsVolatile);
250 
251     /// Materialize an atomic r-value in atomic-layout memory.
252     Address materializeRValue(RValue rvalue) const;
253 
254     /// Creates temp alloca for intermediate operations on atomic value.
255     Address CreateTempAlloca() const;
256   private:
257     bool requiresMemSetZero(llvm::Type *type) const;
258 
259 
260     /// Emits atomic load as a libcall.
261     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
262                                llvm::AtomicOrdering AO, bool IsVolatile);
263     /// Emits atomic load as LLVM instruction.
264     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
265     /// Emits atomic compare-and-exchange op as a libcall.
266     llvm::Value *EmitAtomicCompareExchangeLibcall(
267         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
268         llvm::AtomicOrdering Success =
269             llvm::AtomicOrdering::SequentiallyConsistent,
270         llvm::AtomicOrdering Failure =
271             llvm::AtomicOrdering::SequentiallyConsistent);
272     /// Emits atomic compare-and-exchange op as LLVM instruction.
273     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
274         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
275         llvm::AtomicOrdering Success =
276             llvm::AtomicOrdering::SequentiallyConsistent,
277         llvm::AtomicOrdering Failure =
278             llvm::AtomicOrdering::SequentiallyConsistent,
279         bool IsWeak = false);
280     /// Emit atomic update as libcalls.
281     void
282     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
283                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
284                             bool IsVolatile);
285     /// Emit atomic update as LLVM instructions.
286     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
287                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
288                             bool IsVolatile);
289     /// Emit atomic update as libcalls.
290     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
291                                  bool IsVolatile);
292     /// Emit atomic update as LLVM instructions.
293     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
294                             bool IsVolatile);
295   };
296 }
297 
298 Address AtomicInfo::CreateTempAlloca() const {
299   Address TempAlloca = CGF.CreateMemTemp(
300       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
301                                                                 : AtomicTy,
302       getAtomicAlignment(),
303       "atomic-temp");
304   // Cast to pointer to value type for bitfields.
305   if (LVal.isBitField())
306     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
307         TempAlloca, getAtomicAddress().getType());
308   return TempAlloca;
309 }
310 
311 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
312                                 StringRef fnName,
313                                 QualType resultType,
314                                 CallArgList &args) {
315   const CGFunctionInfo &fnInfo =
316     CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
317   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
318   llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
319   fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
320   fnAttrB.addAttribute(llvm::Attribute::WillReturn);
321   llvm::AttributeList fnAttrs = llvm::AttributeList::get(
322       CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
323 
324   llvm::FunctionCallee fn =
325       CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
326   auto callee = CGCallee::forDirect(fn);
327   return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
328 }
329 
330 /// Does a store of the given IR type modify the full expected width?
331 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
332                            uint64_t expectedSize) {
333   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
334 }
335 
336 /// Does the atomic type require memsetting to zero before initialization?
337 ///
338 /// The IR type is provided as a way of making certain queries faster.
339 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
340   // If the atomic type has size padding, we definitely need a memset.
341   if (hasPadding()) return true;
342 
343   // Otherwise, do some simple heuristics to try to avoid it:
344   switch (getEvaluationKind()) {
345   // For scalars and complexes, check whether the store size of the
346   // type uses the full size.
347   case TEK_Scalar:
348     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
349   case TEK_Complex:
350     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
351                            AtomicSizeInBits / 2);
352 
353   // Padding in structs has an undefined bit pattern.  User beware.
354   case TEK_Aggregate:
355     return false;
356   }
357   llvm_unreachable("bad evaluation kind");
358 }
359 
360 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
361   assert(LVal.isSimple());
362   Address addr = LVal.getAddress(CGF);
363   if (!requiresMemSetZero(addr.getElementType()))
364     return false;
365 
366   CGF.Builder.CreateMemSet(
367       addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
368       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
369       LVal.getAlignment().getAsAlign());
370   return true;
371 }
372 
373 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
374                               Address Dest, Address Ptr,
375                               Address Val1, Address Val2,
376                               uint64_t Size,
377                               llvm::AtomicOrdering SuccessOrder,
378                               llvm::AtomicOrdering FailureOrder,
379                               llvm::SyncScope::ID Scope) {
380   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
381   llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
382   llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
383 
384   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
385       Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
386       Scope);
387   Pair->setVolatile(E->isVolatile());
388   Pair->setWeak(IsWeak);
389 
390   // Cmp holds the result of the compare-exchange operation: true on success,
391   // false on failure.
392   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
393   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
394 
395   // This basic block is used to hold the store instruction if the operation
396   // failed.
397   llvm::BasicBlock *StoreExpectedBB =
398       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
399 
400   // This basic block is the exit point of the operation, we should end up
401   // here regardless of whether or not the operation succeeded.
402   llvm::BasicBlock *ContinueBB =
403       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
404 
405   // Update Expected if Expected isn't equal to Old, otherwise branch to the
406   // exit point.
407   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
408 
409   CGF.Builder.SetInsertPoint(StoreExpectedBB);
410   // Update the memory at Expected with Old's value.
411   CGF.Builder.CreateStore(Old, Val1);
412   // Finally, branch to the exit point.
413   CGF.Builder.CreateBr(ContinueBB);
414 
415   CGF.Builder.SetInsertPoint(ContinueBB);
416   // Update the memory at Dest with Cmp's value.
417   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
418 }
419 
420 /// Given an ordering required on success, emit all possible cmpxchg
421 /// instructions to cope with the provided (but possibly only dynamically known)
422 /// FailureOrder.
423 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
424                                         bool IsWeak, Address Dest, Address Ptr,
425                                         Address Val1, Address Val2,
426                                         llvm::Value *FailureOrderVal,
427                                         uint64_t Size,
428                                         llvm::AtomicOrdering SuccessOrder,
429                                         llvm::SyncScope::ID Scope) {
430   llvm::AtomicOrdering FailureOrder;
431   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
432     auto FOS = FO->getSExtValue();
433     if (!llvm::isValidAtomicOrderingCABI(FOS))
434       FailureOrder = llvm::AtomicOrdering::Monotonic;
435     else
436       switch ((llvm::AtomicOrderingCABI)FOS) {
437       case llvm::AtomicOrderingCABI::relaxed:
438       // 31.7.2.18: "The failure argument shall not be memory_order_release
439       // nor memory_order_acq_rel". Fallback to monotonic.
440       case llvm::AtomicOrderingCABI::release:
441       case llvm::AtomicOrderingCABI::acq_rel:
442         FailureOrder = llvm::AtomicOrdering::Monotonic;
443         break;
444       case llvm::AtomicOrderingCABI::consume:
445       case llvm::AtomicOrderingCABI::acquire:
446         FailureOrder = llvm::AtomicOrdering::Acquire;
447         break;
448       case llvm::AtomicOrderingCABI::seq_cst:
449         FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
450         break;
451       }
452     // Prior to c++17, "the failure argument shall be no stronger than the
453     // success argument". This condition has been lifted and the only
454     // precondition is 31.7.2.18. Effectively treat this as a DR and skip
455     // language version checks.
456     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
457                       FailureOrder, Scope);
458     return;
459   }
460 
461   // Create all the relevant BB's
462   auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
463   auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
464   auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
465   auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
466 
467   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
468   // doesn't matter unless someone is crazy enough to use something that
469   // doesn't fold to a constant for the ordering.
470   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
471   // Implemented as acquire, since it's the closest in LLVM.
472   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
473               AcquireBB);
474   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
475               AcquireBB);
476   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
477               SeqCstBB);
478 
479   // Emit all the different atomics
480   CGF.Builder.SetInsertPoint(MonotonicBB);
481   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
482                     Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
483   CGF.Builder.CreateBr(ContBB);
484 
485   CGF.Builder.SetInsertPoint(AcquireBB);
486   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
487                     llvm::AtomicOrdering::Acquire, Scope);
488   CGF.Builder.CreateBr(ContBB);
489 
490   CGF.Builder.SetInsertPoint(SeqCstBB);
491   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
492                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);
493   CGF.Builder.CreateBr(ContBB);
494 
495   CGF.Builder.SetInsertPoint(ContBB);
496 }
497 
498 /// Duplicate the atomic min/max operation in conventional IR for the builtin
499 /// variants that return the new rather than the original value.
500 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
501                                          AtomicExpr::AtomicOp Op,
502                                          bool IsSigned,
503                                          llvm::Value *OldVal,
504                                          llvm::Value *RHS) {
505   llvm::CmpInst::Predicate Pred;
506   switch (Op) {
507   default:
508     llvm_unreachable("Unexpected min/max operation");
509   case AtomicExpr::AO__atomic_max_fetch:
510     Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
511     break;
512   case AtomicExpr::AO__atomic_min_fetch:
513     Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
514     break;
515   }
516   llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
517   return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
518 }
519 
520 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
521                          Address Ptr, Address Val1, Address Val2,
522                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
523                          uint64_t Size, llvm::AtomicOrdering Order,
524                          llvm::SyncScope::ID Scope) {
525   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
526   bool PostOpMinMax = false;
527   unsigned PostOp = 0;
528 
529   switch (E->getOp()) {
530   case AtomicExpr::AO__c11_atomic_init:
531   case AtomicExpr::AO__opencl_atomic_init:
532     llvm_unreachable("Already handled!");
533 
534   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
535   case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
536   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
537     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
538                                 FailureOrder, Size, Order, Scope);
539     return;
540   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
541   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
542   case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
543     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
544                                 FailureOrder, Size, Order, Scope);
545     return;
546   case AtomicExpr::AO__atomic_compare_exchange:
547   case AtomicExpr::AO__atomic_compare_exchange_n: {
548     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
549       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
550                                   Val1, Val2, FailureOrder, Size, Order, Scope);
551     } else {
552       // Create all the relevant BB's
553       llvm::BasicBlock *StrongBB =
554           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
555       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
556       llvm::BasicBlock *ContBB =
557           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
558 
559       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
560       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
561 
562       CGF.Builder.SetInsertPoint(StrongBB);
563       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
564                                   FailureOrder, Size, Order, Scope);
565       CGF.Builder.CreateBr(ContBB);
566 
567       CGF.Builder.SetInsertPoint(WeakBB);
568       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
569                                   FailureOrder, Size, Order, Scope);
570       CGF.Builder.CreateBr(ContBB);
571 
572       CGF.Builder.SetInsertPoint(ContBB);
573     }
574     return;
575   }
576   case AtomicExpr::AO__c11_atomic_load:
577   case AtomicExpr::AO__opencl_atomic_load:
578   case AtomicExpr::AO__hip_atomic_load:
579   case AtomicExpr::AO__atomic_load_n:
580   case AtomicExpr::AO__atomic_load: {
581     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
582     Load->setAtomic(Order, Scope);
583     Load->setVolatile(E->isVolatile());
584     CGF.Builder.CreateStore(Load, Dest);
585     return;
586   }
587 
588   case AtomicExpr::AO__c11_atomic_store:
589   case AtomicExpr::AO__opencl_atomic_store:
590   case AtomicExpr::AO__hip_atomic_store:
591   case AtomicExpr::AO__atomic_store:
592   case AtomicExpr::AO__atomic_store_n: {
593     llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
594     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
595     Store->setAtomic(Order, Scope);
596     Store->setVolatile(E->isVolatile());
597     return;
598   }
599 
600   case AtomicExpr::AO__c11_atomic_exchange:
601   case AtomicExpr::AO__hip_atomic_exchange:
602   case AtomicExpr::AO__opencl_atomic_exchange:
603   case AtomicExpr::AO__atomic_exchange_n:
604   case AtomicExpr::AO__atomic_exchange:
605     Op = llvm::AtomicRMWInst::Xchg;
606     break;
607 
608   case AtomicExpr::AO__atomic_add_fetch:
609     PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
610                                                  : llvm::Instruction::Add;
611     LLVM_FALLTHROUGH;
612   case AtomicExpr::AO__c11_atomic_fetch_add:
613   case AtomicExpr::AO__hip_atomic_fetch_add:
614   case AtomicExpr::AO__opencl_atomic_fetch_add:
615   case AtomicExpr::AO__atomic_fetch_add:
616     Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
617                                              : llvm::AtomicRMWInst::Add;
618     break;
619 
620   case AtomicExpr::AO__atomic_sub_fetch:
621     PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
622                                                  : llvm::Instruction::Sub;
623     LLVM_FALLTHROUGH;
624   case AtomicExpr::AO__c11_atomic_fetch_sub:
625   case AtomicExpr::AO__opencl_atomic_fetch_sub:
626   case AtomicExpr::AO__atomic_fetch_sub:
627     Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
628                                              : llvm::AtomicRMWInst::Sub;
629     break;
630 
631   case AtomicExpr::AO__atomic_min_fetch:
632     PostOpMinMax = true;
633     LLVM_FALLTHROUGH;
634   case AtomicExpr::AO__c11_atomic_fetch_min:
635   case AtomicExpr::AO__hip_atomic_fetch_min:
636   case AtomicExpr::AO__opencl_atomic_fetch_min:
637   case AtomicExpr::AO__atomic_fetch_min:
638     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
639                                                   : llvm::AtomicRMWInst::UMin;
640     break;
641 
642   case AtomicExpr::AO__atomic_max_fetch:
643     PostOpMinMax = true;
644     LLVM_FALLTHROUGH;
645   case AtomicExpr::AO__c11_atomic_fetch_max:
646   case AtomicExpr::AO__hip_atomic_fetch_max:
647   case AtomicExpr::AO__opencl_atomic_fetch_max:
648   case AtomicExpr::AO__atomic_fetch_max:
649     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
650                                                   : llvm::AtomicRMWInst::UMax;
651     break;
652 
653   case AtomicExpr::AO__atomic_and_fetch:
654     PostOp = llvm::Instruction::And;
655     LLVM_FALLTHROUGH;
656   case AtomicExpr::AO__c11_atomic_fetch_and:
657   case AtomicExpr::AO__hip_atomic_fetch_and:
658   case AtomicExpr::AO__opencl_atomic_fetch_and:
659   case AtomicExpr::AO__atomic_fetch_and:
660     Op = llvm::AtomicRMWInst::And;
661     break;
662 
663   case AtomicExpr::AO__atomic_or_fetch:
664     PostOp = llvm::Instruction::Or;
665     LLVM_FALLTHROUGH;
666   case AtomicExpr::AO__c11_atomic_fetch_or:
667   case AtomicExpr::AO__hip_atomic_fetch_or:
668   case AtomicExpr::AO__opencl_atomic_fetch_or:
669   case AtomicExpr::AO__atomic_fetch_or:
670     Op = llvm::AtomicRMWInst::Or;
671     break;
672 
673   case AtomicExpr::AO__atomic_xor_fetch:
674     PostOp = llvm::Instruction::Xor;
675     LLVM_FALLTHROUGH;
676   case AtomicExpr::AO__c11_atomic_fetch_xor:
677   case AtomicExpr::AO__hip_atomic_fetch_xor:
678   case AtomicExpr::AO__opencl_atomic_fetch_xor:
679   case AtomicExpr::AO__atomic_fetch_xor:
680     Op = llvm::AtomicRMWInst::Xor;
681     break;
682 
683   case AtomicExpr::AO__atomic_nand_fetch:
684     PostOp = llvm::Instruction::And; // the NOT is special cased below
685     LLVM_FALLTHROUGH;
686   case AtomicExpr::AO__c11_atomic_fetch_nand:
687   case AtomicExpr::AO__atomic_fetch_nand:
688     Op = llvm::AtomicRMWInst::Nand;
689     break;
690   }
691 
692   llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
693   llvm::AtomicRMWInst *RMWI =
694       CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
695   RMWI->setVolatile(E->isVolatile());
696 
697   // For __atomic_*_fetch operations, perform the operation again to
698   // determine the value which was written.
699   llvm::Value *Result = RMWI;
700   if (PostOpMinMax)
701     Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
702                                   E->getValueType()->isSignedIntegerType(),
703                                   RMWI, LoadVal1);
704   else if (PostOp)
705     Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
706                                      LoadVal1);
707   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
708     Result = CGF.Builder.CreateNot(Result);
709   CGF.Builder.CreateStore(Result, Dest);
710 }
711 
712 // This function emits any expression (scalar, complex, or aggregate)
713 // into a temporary alloca.
714 static Address
715 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
716   Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
717   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
718                        /*Init*/ true);
719   return DeclPtr;
720 }
721 
722 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
723                          Address Ptr, Address Val1, Address Val2,
724                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
725                          uint64_t Size, llvm::AtomicOrdering Order,
726                          llvm::Value *Scope) {
727   auto ScopeModel = Expr->getScopeModel();
728 
729   // LLVM atomic instructions always have synch scope. If clang atomic
730   // expression has no scope operand, use default LLVM synch scope.
731   if (!ScopeModel) {
732     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
733                  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
734     return;
735   }
736 
737   // Handle constant scope.
738   if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
739     auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
740         CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
741         Order, CGF.CGM.getLLVMContext());
742     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
743                  Order, SCID);
744     return;
745   }
746 
747   // Handle non-constant scope.
748   auto &Builder = CGF.Builder;
749   auto Scopes = ScopeModel->getRuntimeValues();
750   llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
751   for (auto S : Scopes)
752     BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
753 
754   llvm::BasicBlock *ContBB =
755       CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
756 
757   auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
758   // If unsupported synch scope is encountered at run time, assume a fallback
759   // synch scope value.
760   auto FallBack = ScopeModel->getFallBackValue();
761   llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
762   for (auto S : Scopes) {
763     auto *B = BB[S];
764     if (S != FallBack)
765       SI->addCase(Builder.getInt32(S), B);
766 
767     Builder.SetInsertPoint(B);
768     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
769                  Order,
770                  CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
771                                                          ScopeModel->map(S),
772                                                          Order,
773                                                          CGF.getLLVMContext()));
774     Builder.CreateBr(ContBB);
775   }
776 
777   Builder.SetInsertPoint(ContBB);
778 }
779 
780 static void
781 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
782                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
783                   SourceLocation Loc, CharUnits SizeInChars) {
784   if (UseOptimizedLibcall) {
785     // Load value and pass it to the function directly.
786     CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
787     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
788     ValTy =
789         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
790     llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
791     Address Ptr = Address(CGF.Builder.CreateBitCast(Val, ITy->getPointerTo()),
792                           ITy, Align);
793     Val = CGF.EmitLoadOfScalar(Ptr, false,
794                                CGF.getContext().getPointerType(ValTy),
795                                Loc);
796     // Coerce the value into an appropriately sized integer type.
797     Args.add(RValue::get(Val), ValTy);
798   } else {
799     // Non-optimized functions always take a reference.
800     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
801                          CGF.getContext().VoidPtrTy);
802   }
803 }
804 
805 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
806   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
807   QualType MemTy = AtomicTy;
808   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
809     MemTy = AT->getValueType();
810   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
811 
812   Address Val1 = Address::invalid();
813   Address Val2 = Address::invalid();
814   Address Dest = Address::invalid();
815   Address Ptr = EmitPointerWithAlignment(E->getPtr());
816 
817   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
818       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
819     LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
820     EmitAtomicInit(E->getVal1(), lvalue);
821     return RValue::get(nullptr);
822   }
823 
824   auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
825   uint64_t Size = TInfo.Width.getQuantity();
826   unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
827 
828   bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
829   bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
830   bool UseLibcall = Misaligned | Oversized;
831   bool ShouldCastToIntPtrTy = true;
832 
833   CharUnits MaxInlineWidth =
834       getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
835 
836   DiagnosticsEngine &Diags = CGM.getDiags();
837 
838   if (Misaligned) {
839     Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
840         << (int)TInfo.Width.getQuantity()
841         << (int)Ptr.getAlignment().getQuantity();
842   }
843 
844   if (Oversized) {
845     Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
846         << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
847   }
848 
849   llvm::Value *Order = EmitScalarExpr(E->getOrder());
850   llvm::Value *Scope =
851       E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
852 
853   switch (E->getOp()) {
854   case AtomicExpr::AO__c11_atomic_init:
855   case AtomicExpr::AO__opencl_atomic_init:
856     llvm_unreachable("Already handled above with EmitAtomicInit!");
857 
858   case AtomicExpr::AO__c11_atomic_load:
859   case AtomicExpr::AO__opencl_atomic_load:
860   case AtomicExpr::AO__hip_atomic_load:
861   case AtomicExpr::AO__atomic_load_n:
862     break;
863 
864   case AtomicExpr::AO__atomic_load:
865     Dest = EmitPointerWithAlignment(E->getVal1());
866     break;
867 
868   case AtomicExpr::AO__atomic_store:
869     Val1 = EmitPointerWithAlignment(E->getVal1());
870     break;
871 
872   case AtomicExpr::AO__atomic_exchange:
873     Val1 = EmitPointerWithAlignment(E->getVal1());
874     Dest = EmitPointerWithAlignment(E->getVal2());
875     break;
876 
877   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
878   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
879   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
880   case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
881   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
882   case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
883   case AtomicExpr::AO__atomic_compare_exchange_n:
884   case AtomicExpr::AO__atomic_compare_exchange:
885     Val1 = EmitPointerWithAlignment(E->getVal1());
886     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
887       Val2 = EmitPointerWithAlignment(E->getVal2());
888     else
889       Val2 = EmitValToTemp(*this, E->getVal2());
890     OrderFail = EmitScalarExpr(E->getOrderFail());
891     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
892         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
893       IsWeak = EmitScalarExpr(E->getWeak());
894     break;
895 
896   case AtomicExpr::AO__c11_atomic_fetch_add:
897   case AtomicExpr::AO__c11_atomic_fetch_sub:
898   case AtomicExpr::AO__hip_atomic_fetch_add:
899   case AtomicExpr::AO__opencl_atomic_fetch_add:
900   case AtomicExpr::AO__opencl_atomic_fetch_sub:
901     if (MemTy->isPointerType()) {
902       // For pointer arithmetic, we're required to do a bit of math:
903       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
904       // ... but only for the C11 builtins. The GNU builtins expect the
905       // user to multiply by sizeof(T).
906       QualType Val1Ty = E->getVal1()->getType();
907       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
908       CharUnits PointeeIncAmt =
909           getContext().getTypeSizeInChars(MemTy->getPointeeType());
910       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
911       auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
912       Val1 = Temp;
913       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
914       break;
915     }
916     LLVM_FALLTHROUGH;
917   case AtomicExpr::AO__atomic_fetch_add:
918   case AtomicExpr::AO__atomic_fetch_sub:
919   case AtomicExpr::AO__atomic_add_fetch:
920   case AtomicExpr::AO__atomic_sub_fetch:
921     ShouldCastToIntPtrTy = !MemTy->isFloatingType();
922     LLVM_FALLTHROUGH;
923 
924   case AtomicExpr::AO__c11_atomic_store:
925   case AtomicExpr::AO__c11_atomic_exchange:
926   case AtomicExpr::AO__opencl_atomic_store:
927   case AtomicExpr::AO__hip_atomic_store:
928   case AtomicExpr::AO__opencl_atomic_exchange:
929   case AtomicExpr::AO__hip_atomic_exchange:
930   case AtomicExpr::AO__atomic_store_n:
931   case AtomicExpr::AO__atomic_exchange_n:
932   case AtomicExpr::AO__c11_atomic_fetch_and:
933   case AtomicExpr::AO__c11_atomic_fetch_or:
934   case AtomicExpr::AO__c11_atomic_fetch_xor:
935   case AtomicExpr::AO__c11_atomic_fetch_nand:
936   case AtomicExpr::AO__c11_atomic_fetch_max:
937   case AtomicExpr::AO__c11_atomic_fetch_min:
938   case AtomicExpr::AO__opencl_atomic_fetch_and:
939   case AtomicExpr::AO__opencl_atomic_fetch_or:
940   case AtomicExpr::AO__opencl_atomic_fetch_xor:
941   case AtomicExpr::AO__opencl_atomic_fetch_min:
942   case AtomicExpr::AO__opencl_atomic_fetch_max:
943   case AtomicExpr::AO__atomic_fetch_and:
944   case AtomicExpr::AO__hip_atomic_fetch_and:
945   case AtomicExpr::AO__atomic_fetch_or:
946   case AtomicExpr::AO__hip_atomic_fetch_or:
947   case AtomicExpr::AO__atomic_fetch_xor:
948   case AtomicExpr::AO__hip_atomic_fetch_xor:
949   case AtomicExpr::AO__atomic_fetch_nand:
950   case AtomicExpr::AO__atomic_and_fetch:
951   case AtomicExpr::AO__atomic_or_fetch:
952   case AtomicExpr::AO__atomic_xor_fetch:
953   case AtomicExpr::AO__atomic_nand_fetch:
954   case AtomicExpr::AO__atomic_max_fetch:
955   case AtomicExpr::AO__atomic_min_fetch:
956   case AtomicExpr::AO__atomic_fetch_max:
957   case AtomicExpr::AO__hip_atomic_fetch_max:
958   case AtomicExpr::AO__atomic_fetch_min:
959   case AtomicExpr::AO__hip_atomic_fetch_min:
960     Val1 = EmitValToTemp(*this, E->getVal1());
961     break;
962   }
963 
964   QualType RValTy = E->getType().getUnqualifiedType();
965 
966   // The inlined atomics only function on iN types, where N is a power of 2. We
967   // need to make sure (via temporaries if necessary) that all incoming values
968   // are compatible.
969   LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
970   AtomicInfo Atomics(*this, AtomicVal);
971 
972   if (ShouldCastToIntPtrTy) {
973     Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
974     if (Val1.isValid())
975       Val1 = Atomics.convertToAtomicIntPointer(Val1);
976     if (Val2.isValid())
977       Val2 = Atomics.convertToAtomicIntPointer(Val2);
978   }
979   if (Dest.isValid()) {
980     if (ShouldCastToIntPtrTy)
981       Dest = Atomics.emitCastToAtomicIntPointer(Dest);
982   } else if (E->isCmpXChg())
983     Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
984   else if (!RValTy->isVoidType()) {
985     Dest = Atomics.CreateTempAlloca();
986     if (ShouldCastToIntPtrTy)
987       Dest = Atomics.emitCastToAtomicIntPointer(Dest);
988   }
989 
990   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
991   if (UseLibcall) {
992     bool UseOptimizedLibcall = false;
993     switch (E->getOp()) {
994     case AtomicExpr::AO__c11_atomic_init:
995     case AtomicExpr::AO__opencl_atomic_init:
996       llvm_unreachable("Already handled above with EmitAtomicInit!");
997 
998     case AtomicExpr::AO__c11_atomic_fetch_add:
999     case AtomicExpr::AO__opencl_atomic_fetch_add:
1000     case AtomicExpr::AO__atomic_fetch_add:
1001     case AtomicExpr::AO__hip_atomic_fetch_add:
1002     case AtomicExpr::AO__c11_atomic_fetch_and:
1003     case AtomicExpr::AO__opencl_atomic_fetch_and:
1004     case AtomicExpr::AO__hip_atomic_fetch_and:
1005     case AtomicExpr::AO__atomic_fetch_and:
1006     case AtomicExpr::AO__c11_atomic_fetch_or:
1007     case AtomicExpr::AO__opencl_atomic_fetch_or:
1008     case AtomicExpr::AO__hip_atomic_fetch_or:
1009     case AtomicExpr::AO__atomic_fetch_or:
1010     case AtomicExpr::AO__c11_atomic_fetch_nand:
1011     case AtomicExpr::AO__atomic_fetch_nand:
1012     case AtomicExpr::AO__c11_atomic_fetch_sub:
1013     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1014     case AtomicExpr::AO__atomic_fetch_sub:
1015     case AtomicExpr::AO__c11_atomic_fetch_xor:
1016     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1017     case AtomicExpr::AO__opencl_atomic_fetch_min:
1018     case AtomicExpr::AO__opencl_atomic_fetch_max:
1019     case AtomicExpr::AO__atomic_fetch_xor:
1020     case AtomicExpr::AO__hip_atomic_fetch_xor:
1021     case AtomicExpr::AO__c11_atomic_fetch_max:
1022     case AtomicExpr::AO__c11_atomic_fetch_min:
1023     case AtomicExpr::AO__atomic_add_fetch:
1024     case AtomicExpr::AO__atomic_and_fetch:
1025     case AtomicExpr::AO__atomic_nand_fetch:
1026     case AtomicExpr::AO__atomic_or_fetch:
1027     case AtomicExpr::AO__atomic_sub_fetch:
1028     case AtomicExpr::AO__atomic_xor_fetch:
1029     case AtomicExpr::AO__atomic_fetch_max:
1030     case AtomicExpr::AO__hip_atomic_fetch_max:
1031     case AtomicExpr::AO__atomic_fetch_min:
1032     case AtomicExpr::AO__hip_atomic_fetch_min:
1033     case AtomicExpr::AO__atomic_max_fetch:
1034     case AtomicExpr::AO__atomic_min_fetch:
1035       // For these, only library calls for certain sizes exist.
1036       UseOptimizedLibcall = true;
1037       break;
1038 
1039     case AtomicExpr::AO__atomic_load:
1040     case AtomicExpr::AO__atomic_store:
1041     case AtomicExpr::AO__atomic_exchange:
1042     case AtomicExpr::AO__atomic_compare_exchange:
1043       // Use the generic version if we don't know that the operand will be
1044       // suitably aligned for the optimized version.
1045       if (Misaligned)
1046         break;
1047       LLVM_FALLTHROUGH;
1048     case AtomicExpr::AO__c11_atomic_load:
1049     case AtomicExpr::AO__c11_atomic_store:
1050     case AtomicExpr::AO__c11_atomic_exchange:
1051     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1052     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1053     case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1054     case AtomicExpr::AO__opencl_atomic_load:
1055     case AtomicExpr::AO__hip_atomic_load:
1056     case AtomicExpr::AO__opencl_atomic_store:
1057     case AtomicExpr::AO__hip_atomic_store:
1058     case AtomicExpr::AO__opencl_atomic_exchange:
1059     case AtomicExpr::AO__hip_atomic_exchange:
1060     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1061     case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1062     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1063     case AtomicExpr::AO__atomic_load_n:
1064     case AtomicExpr::AO__atomic_store_n:
1065     case AtomicExpr::AO__atomic_exchange_n:
1066     case AtomicExpr::AO__atomic_compare_exchange_n:
1067       // Only use optimized library calls for sizes for which they exist.
1068       // FIXME: Size == 16 optimized library functions exist too.
1069       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1070         UseOptimizedLibcall = true;
1071       break;
1072     }
1073 
1074     CallArgList Args;
1075     if (!UseOptimizedLibcall) {
1076       // For non-optimized library calls, the size is the first parameter
1077       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1078                getContext().getSizeType());
1079     }
1080     // Atomic address is the first or second parameter
1081     // The OpenCL atomic library functions only accept pointer arguments to
1082     // generic address space.
1083     auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1084       if (!E->isOpenCL())
1085         return V;
1086       auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1087       if (AS == LangAS::opencl_generic)
1088         return V;
1089       auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1090       auto T = llvm::cast<llvm::PointerType>(V->getType());
1091       auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS);
1092 
1093       return getTargetHooks().performAddrSpaceCast(
1094           *this, V, AS, LangAS::opencl_generic, DestType, false);
1095     };
1096 
1097     Args.add(RValue::get(CastToGenericAddrSpace(
1098                  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1099              getContext().VoidPtrTy);
1100 
1101     std::string LibCallName;
1102     QualType LoweredMemTy =
1103       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1104     QualType RetTy;
1105     bool HaveRetTy = false;
1106     llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1107     bool PostOpMinMax = false;
1108     switch (E->getOp()) {
1109     case AtomicExpr::AO__c11_atomic_init:
1110     case AtomicExpr::AO__opencl_atomic_init:
1111       llvm_unreachable("Already handled!");
1112 
1113     // There is only one libcall for compare an exchange, because there is no
1114     // optimisation benefit possible from a libcall version of a weak compare
1115     // and exchange.
1116     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1117     //                                void *desired, int success, int failure)
1118     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1119     //                                  int success, int failure)
1120     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1121     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1122     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1123     case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1124     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1125     case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1126     case AtomicExpr::AO__atomic_compare_exchange:
1127     case AtomicExpr::AO__atomic_compare_exchange_n:
1128       LibCallName = "__atomic_compare_exchange";
1129       RetTy = getContext().BoolTy;
1130       HaveRetTy = true;
1131       Args.add(
1132           RValue::get(CastToGenericAddrSpace(
1133               EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1134           getContext().VoidPtrTy);
1135       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1136                         MemTy, E->getExprLoc(), TInfo.Width);
1137       Args.add(RValue::get(Order), getContext().IntTy);
1138       Order = OrderFail;
1139       break;
1140     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1141     //                        int order)
1142     // T __atomic_exchange_N(T *mem, T val, int order)
1143     case AtomicExpr::AO__c11_atomic_exchange:
1144     case AtomicExpr::AO__opencl_atomic_exchange:
1145     case AtomicExpr::AO__atomic_exchange_n:
1146     case AtomicExpr::AO__atomic_exchange:
1147     case AtomicExpr::AO__hip_atomic_exchange:
1148       LibCallName = "__atomic_exchange";
1149       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1150                         MemTy, E->getExprLoc(), TInfo.Width);
1151       break;
1152     // void __atomic_store(size_t size, void *mem, void *val, int order)
1153     // void __atomic_store_N(T *mem, T val, int order)
1154     case AtomicExpr::AO__c11_atomic_store:
1155     case AtomicExpr::AO__opencl_atomic_store:
1156     case AtomicExpr::AO__hip_atomic_store:
1157     case AtomicExpr::AO__atomic_store:
1158     case AtomicExpr::AO__atomic_store_n:
1159       LibCallName = "__atomic_store";
1160       RetTy = getContext().VoidTy;
1161       HaveRetTy = true;
1162       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1163                         MemTy, E->getExprLoc(), TInfo.Width);
1164       break;
1165     // void __atomic_load(size_t size, void *mem, void *return, int order)
1166     // T __atomic_load_N(T *mem, int order)
1167     case AtomicExpr::AO__c11_atomic_load:
1168     case AtomicExpr::AO__opencl_atomic_load:
1169     case AtomicExpr::AO__hip_atomic_load:
1170     case AtomicExpr::AO__atomic_load:
1171     case AtomicExpr::AO__atomic_load_n:
1172       LibCallName = "__atomic_load";
1173       break;
1174     // T __atomic_add_fetch_N(T *mem, T val, int order)
1175     // T __atomic_fetch_add_N(T *mem, T val, int order)
1176     case AtomicExpr::AO__atomic_add_fetch:
1177       PostOp = llvm::Instruction::Add;
1178       LLVM_FALLTHROUGH;
1179     case AtomicExpr::AO__c11_atomic_fetch_add:
1180     case AtomicExpr::AO__opencl_atomic_fetch_add:
1181     case AtomicExpr::AO__atomic_fetch_add:
1182     case AtomicExpr::AO__hip_atomic_fetch_add:
1183       LibCallName = "__atomic_fetch_add";
1184       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1185                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1186       break;
1187     // T __atomic_and_fetch_N(T *mem, T val, int order)
1188     // T __atomic_fetch_and_N(T *mem, T val, int order)
1189     case AtomicExpr::AO__atomic_and_fetch:
1190       PostOp = llvm::Instruction::And;
1191       LLVM_FALLTHROUGH;
1192     case AtomicExpr::AO__c11_atomic_fetch_and:
1193     case AtomicExpr::AO__opencl_atomic_fetch_and:
1194     case AtomicExpr::AO__hip_atomic_fetch_and:
1195     case AtomicExpr::AO__atomic_fetch_and:
1196       LibCallName = "__atomic_fetch_and";
1197       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1198                         MemTy, E->getExprLoc(), TInfo.Width);
1199       break;
1200     // T __atomic_or_fetch_N(T *mem, T val, int order)
1201     // T __atomic_fetch_or_N(T *mem, T val, int order)
1202     case AtomicExpr::AO__atomic_or_fetch:
1203       PostOp = llvm::Instruction::Or;
1204       LLVM_FALLTHROUGH;
1205     case AtomicExpr::AO__c11_atomic_fetch_or:
1206     case AtomicExpr::AO__opencl_atomic_fetch_or:
1207     case AtomicExpr::AO__hip_atomic_fetch_or:
1208     case AtomicExpr::AO__atomic_fetch_or:
1209       LibCallName = "__atomic_fetch_or";
1210       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1211                         MemTy, E->getExprLoc(), TInfo.Width);
1212       break;
1213     // T __atomic_sub_fetch_N(T *mem, T val, int order)
1214     // T __atomic_fetch_sub_N(T *mem, T val, int order)
1215     case AtomicExpr::AO__atomic_sub_fetch:
1216       PostOp = llvm::Instruction::Sub;
1217       LLVM_FALLTHROUGH;
1218     case AtomicExpr::AO__c11_atomic_fetch_sub:
1219     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1220     case AtomicExpr::AO__atomic_fetch_sub:
1221       LibCallName = "__atomic_fetch_sub";
1222       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1223                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1224       break;
1225     // T __atomic_xor_fetch_N(T *mem, T val, int order)
1226     // T __atomic_fetch_xor_N(T *mem, T val, int order)
1227     case AtomicExpr::AO__atomic_xor_fetch:
1228       PostOp = llvm::Instruction::Xor;
1229       LLVM_FALLTHROUGH;
1230     case AtomicExpr::AO__c11_atomic_fetch_xor:
1231     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1232     case AtomicExpr::AO__hip_atomic_fetch_xor:
1233     case AtomicExpr::AO__atomic_fetch_xor:
1234       LibCallName = "__atomic_fetch_xor";
1235       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1236                         MemTy, E->getExprLoc(), TInfo.Width);
1237       break;
1238     case AtomicExpr::AO__atomic_min_fetch:
1239       PostOpMinMax = true;
1240       LLVM_FALLTHROUGH;
1241     case AtomicExpr::AO__c11_atomic_fetch_min:
1242     case AtomicExpr::AO__atomic_fetch_min:
1243     case AtomicExpr::AO__hip_atomic_fetch_min:
1244     case AtomicExpr::AO__opencl_atomic_fetch_min:
1245       LibCallName = E->getValueType()->isSignedIntegerType()
1246                         ? "__atomic_fetch_min"
1247                         : "__atomic_fetch_umin";
1248       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1249                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1250       break;
1251     case AtomicExpr::AO__atomic_max_fetch:
1252       PostOpMinMax = true;
1253       LLVM_FALLTHROUGH;
1254     case AtomicExpr::AO__c11_atomic_fetch_max:
1255     case AtomicExpr::AO__atomic_fetch_max:
1256     case AtomicExpr::AO__hip_atomic_fetch_max:
1257     case AtomicExpr::AO__opencl_atomic_fetch_max:
1258       LibCallName = E->getValueType()->isSignedIntegerType()
1259                         ? "__atomic_fetch_max"
1260                         : "__atomic_fetch_umax";
1261       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1262                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1263       break;
1264     // T __atomic_nand_fetch_N(T *mem, T val, int order)
1265     // T __atomic_fetch_nand_N(T *mem, T val, int order)
1266     case AtomicExpr::AO__atomic_nand_fetch:
1267       PostOp = llvm::Instruction::And; // the NOT is special cased below
1268       LLVM_FALLTHROUGH;
1269     case AtomicExpr::AO__c11_atomic_fetch_nand:
1270     case AtomicExpr::AO__atomic_fetch_nand:
1271       LibCallName = "__atomic_fetch_nand";
1272       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1273                         MemTy, E->getExprLoc(), TInfo.Width);
1274       break;
1275     }
1276 
1277     if (E->isOpenCL()) {
1278       LibCallName = std::string("__opencl") +
1279           StringRef(LibCallName).drop_front(1).str();
1280 
1281     }
1282     // Optimized functions have the size in their name.
1283     if (UseOptimizedLibcall)
1284       LibCallName += "_" + llvm::utostr(Size);
1285     // By default, assume we return a value of the atomic type.
1286     if (!HaveRetTy) {
1287       if (UseOptimizedLibcall) {
1288         // Value is returned directly.
1289         // The function returns an appropriately sized integer type.
1290         RetTy = getContext().getIntTypeForBitwidth(
1291             getContext().toBits(TInfo.Width), /*Signed=*/false);
1292       } else {
1293         // Value is returned through parameter before the order.
1294         RetTy = getContext().VoidTy;
1295         Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1296                  getContext().VoidPtrTy);
1297       }
1298     }
1299     // order is always the last parameter
1300     Args.add(RValue::get(Order),
1301              getContext().IntTy);
1302     if (E->isOpenCL())
1303       Args.add(RValue::get(Scope), getContext().IntTy);
1304 
1305     // PostOp is only needed for the atomic_*_fetch operations, and
1306     // thus is only needed for and implemented in the
1307     // UseOptimizedLibcall codepath.
1308     assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1309 
1310     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1311     // The value is returned directly from the libcall.
1312     if (E->isCmpXChg())
1313       return Res;
1314 
1315     // The value is returned directly for optimized libcalls but the expr
1316     // provided an out-param.
1317     if (UseOptimizedLibcall && Res.getScalarVal()) {
1318       llvm::Value *ResVal = Res.getScalarVal();
1319       if (PostOpMinMax) {
1320         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1321         ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1322                                       E->getValueType()->isSignedIntegerType(),
1323                                       ResVal, LoadVal1);
1324       } else if (PostOp) {
1325         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1326         ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1327       }
1328       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1329         ResVal = Builder.CreateNot(ResVal);
1330 
1331       Builder.CreateStore(
1332           ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType()));
1333     }
1334 
1335     if (RValTy->isVoidType())
1336       return RValue::get(nullptr);
1337 
1338     return convertTempToRValue(
1339         Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1340         RValTy, E->getExprLoc());
1341   }
1342 
1343   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1344                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1345                  E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1346                  E->getOp() == AtomicExpr::AO__atomic_store ||
1347                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1348   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1349                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1350                 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1351                 E->getOp() == AtomicExpr::AO__atomic_load ||
1352                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1353 
1354   if (isa<llvm::ConstantInt>(Order)) {
1355     auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1356     // We should not ever get to a case where the ordering isn't a valid C ABI
1357     // value, but it's hard to enforce that in general.
1358     if (llvm::isValidAtomicOrderingCABI(ord))
1359       switch ((llvm::AtomicOrderingCABI)ord) {
1360       case llvm::AtomicOrderingCABI::relaxed:
1361         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1362                      llvm::AtomicOrdering::Monotonic, Scope);
1363         break;
1364       case llvm::AtomicOrderingCABI::consume:
1365       case llvm::AtomicOrderingCABI::acquire:
1366         if (IsStore)
1367           break; // Avoid crashing on code with undefined behavior
1368         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1369                      llvm::AtomicOrdering::Acquire, Scope);
1370         break;
1371       case llvm::AtomicOrderingCABI::release:
1372         if (IsLoad)
1373           break; // Avoid crashing on code with undefined behavior
1374         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1375                      llvm::AtomicOrdering::Release, Scope);
1376         break;
1377       case llvm::AtomicOrderingCABI::acq_rel:
1378         if (IsLoad || IsStore)
1379           break; // Avoid crashing on code with undefined behavior
1380         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1381                      llvm::AtomicOrdering::AcquireRelease, Scope);
1382         break;
1383       case llvm::AtomicOrderingCABI::seq_cst:
1384         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1385                      llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1386         break;
1387       }
1388     if (RValTy->isVoidType())
1389       return RValue::get(nullptr);
1390 
1391     return convertTempToRValue(
1392         Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1393         RValTy, E->getExprLoc());
1394   }
1395 
1396   // Long case, when Order isn't obviously constant.
1397 
1398   // Create all the relevant BB's
1399   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1400                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1401                    *SeqCstBB = nullptr;
1402   MonotonicBB = createBasicBlock("monotonic", CurFn);
1403   if (!IsStore)
1404     AcquireBB = createBasicBlock("acquire", CurFn);
1405   if (!IsLoad)
1406     ReleaseBB = createBasicBlock("release", CurFn);
1407   if (!IsLoad && !IsStore)
1408     AcqRelBB = createBasicBlock("acqrel", CurFn);
1409   SeqCstBB = createBasicBlock("seqcst", CurFn);
1410   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1411 
1412   // Create the switch for the split
1413   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1414   // doesn't matter unless someone is crazy enough to use something that
1415   // doesn't fold to a constant for the ordering.
1416   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1417   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1418 
1419   // Emit all the different atomics
1420   Builder.SetInsertPoint(MonotonicBB);
1421   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1422                llvm::AtomicOrdering::Monotonic, Scope);
1423   Builder.CreateBr(ContBB);
1424   if (!IsStore) {
1425     Builder.SetInsertPoint(AcquireBB);
1426     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1427                  llvm::AtomicOrdering::Acquire, Scope);
1428     Builder.CreateBr(ContBB);
1429     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1430                 AcquireBB);
1431     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1432                 AcquireBB);
1433   }
1434   if (!IsLoad) {
1435     Builder.SetInsertPoint(ReleaseBB);
1436     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1437                  llvm::AtomicOrdering::Release, Scope);
1438     Builder.CreateBr(ContBB);
1439     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1440                 ReleaseBB);
1441   }
1442   if (!IsLoad && !IsStore) {
1443     Builder.SetInsertPoint(AcqRelBB);
1444     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1445                  llvm::AtomicOrdering::AcquireRelease, Scope);
1446     Builder.CreateBr(ContBB);
1447     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1448                 AcqRelBB);
1449   }
1450   Builder.SetInsertPoint(SeqCstBB);
1451   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1452                llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1453   Builder.CreateBr(ContBB);
1454   SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1455               SeqCstBB);
1456 
1457   // Cleanup and return
1458   Builder.SetInsertPoint(ContBB);
1459   if (RValTy->isVoidType())
1460     return RValue::get(nullptr);
1461 
1462   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1463   return convertTempToRValue(
1464       Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1465       RValTy, E->getExprLoc());
1466 }
1467 
1468 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1469   llvm::IntegerType *ty =
1470     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1471   return CGF.Builder.CreateElementBitCast(addr, ty);
1472 }
1473 
1474 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1475   llvm::Type *Ty = Addr.getElementType();
1476   uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1477   if (SourceSizeInBits != AtomicSizeInBits) {
1478     Address Tmp = CreateTempAlloca();
1479     CGF.Builder.CreateMemCpy(Tmp, Addr,
1480                              std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1481     Addr = Tmp;
1482   }
1483 
1484   return emitCastToAtomicIntPointer(Addr);
1485 }
1486 
1487 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1488                                              AggValueSlot resultSlot,
1489                                              SourceLocation loc,
1490                                              bool asValue) const {
1491   if (LVal.isSimple()) {
1492     if (EvaluationKind == TEK_Aggregate)
1493       return resultSlot.asRValue();
1494 
1495     // Drill into the padding structure if we have one.
1496     if (hasPadding())
1497       addr = CGF.Builder.CreateStructGEP(addr, 0);
1498 
1499     // Otherwise, just convert the temporary to an r-value using the
1500     // normal conversion routine.
1501     return CGF.convertTempToRValue(addr, getValueType(), loc);
1502   }
1503   if (!asValue)
1504     // Get RValue from temp memory as atomic for non-simple lvalues
1505     return RValue::get(CGF.Builder.CreateLoad(addr));
1506   if (LVal.isBitField())
1507     return CGF.EmitLoadOfBitfieldLValue(
1508         LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1509                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1510   if (LVal.isVectorElt())
1511     return CGF.EmitLoadOfLValue(
1512         LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1513                               LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1514   assert(LVal.isExtVectorElt());
1515   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1516       addr, LVal.getExtVectorElts(), LVal.getType(),
1517       LVal.getBaseInfo(), TBAAAccessInfo()));
1518 }
1519 
1520 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1521                                              AggValueSlot ResultSlot,
1522                                              SourceLocation Loc,
1523                                              bool AsValue) const {
1524   // Try not to in some easy cases.
1525   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1526   if (getEvaluationKind() == TEK_Scalar &&
1527       (((!LVal.isBitField() ||
1528          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1529         !hasPadding()) ||
1530        !AsValue)) {
1531     auto *ValTy = AsValue
1532                       ? CGF.ConvertTypeForMem(ValueTy)
1533                       : getAtomicAddress().getElementType();
1534     if (ValTy->isIntegerTy()) {
1535       assert(IntVal->getType() == ValTy && "Different integer types.");
1536       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1537     } else if (ValTy->isPointerTy())
1538       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1539     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1540       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1541   }
1542 
1543   // Create a temporary.  This needs to be big enough to hold the
1544   // atomic integer.
1545   Address Temp = Address::invalid();
1546   bool TempIsVolatile = false;
1547   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1548     assert(!ResultSlot.isIgnored());
1549     Temp = ResultSlot.getAddress();
1550     TempIsVolatile = ResultSlot.isVolatile();
1551   } else {
1552     Temp = CreateTempAlloca();
1553   }
1554 
1555   // Slam the integer into the temporary.
1556   Address CastTemp = emitCastToAtomicIntPointer(Temp);
1557   CGF.Builder.CreateStore(IntVal, CastTemp)
1558       ->setVolatile(TempIsVolatile);
1559 
1560   return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1561 }
1562 
1563 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1564                                        llvm::AtomicOrdering AO, bool) {
1565   // void __atomic_load(size_t size, void *mem, void *return, int order);
1566   CallArgList Args;
1567   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1568   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1569            CGF.getContext().VoidPtrTy);
1570   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1571            CGF.getContext().VoidPtrTy);
1572   Args.add(
1573       RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1574       CGF.getContext().IntTy);
1575   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1576 }
1577 
1578 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1579                                           bool IsVolatile) {
1580   // Okay, we're doing this natively.
1581   Address Addr = getAtomicAddressAsAtomicIntPointer();
1582   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1583   Load->setAtomic(AO);
1584 
1585   // Other decoration.
1586   if (IsVolatile)
1587     Load->setVolatile(true);
1588   CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1589   return Load;
1590 }
1591 
1592 /// An LValue is a candidate for having its loads and stores be made atomic if
1593 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1594 /// performing such an operation can be performed without a libcall.
1595 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1596   if (!CGM.getCodeGenOpts().MSVolatile) return false;
1597   AtomicInfo AI(*this, LV);
1598   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1599   // An atomic is inline if we don't need to use a libcall.
1600   bool AtomicIsInline = !AI.shouldUseLibcall();
1601   // MSVC doesn't seem to do this for types wider than a pointer.
1602   if (getContext().getTypeSize(LV.getType()) >
1603       getContext().getTypeSize(getContext().getIntPtrType()))
1604     return false;
1605   return IsVolatile && AtomicIsInline;
1606 }
1607 
1608 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1609                                        AggValueSlot Slot) {
1610   llvm::AtomicOrdering AO;
1611   bool IsVolatile = LV.isVolatileQualified();
1612   if (LV.getType()->isAtomicType()) {
1613     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1614   } else {
1615     AO = llvm::AtomicOrdering::Acquire;
1616     IsVolatile = true;
1617   }
1618   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1619 }
1620 
1621 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1622                                   bool AsValue, llvm::AtomicOrdering AO,
1623                                   bool IsVolatile) {
1624   // Check whether we should use a library call.
1625   if (shouldUseLibcall()) {
1626     Address TempAddr = Address::invalid();
1627     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1628       assert(getEvaluationKind() == TEK_Aggregate);
1629       TempAddr = ResultSlot.getAddress();
1630     } else
1631       TempAddr = CreateTempAlloca();
1632 
1633     EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1634 
1635     // Okay, turn that back into the original value or whole atomic (for
1636     // non-simple lvalues) type.
1637     return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1638   }
1639 
1640   // Okay, we're doing this natively.
1641   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1642 
1643   // If we're ignoring an aggregate return, don't do anything.
1644   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1645     return RValue::getAggregate(Address::invalid(), false);
1646 
1647   // Okay, turn that back into the original value or atomic (for non-simple
1648   // lvalues) type.
1649   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1650 }
1651 
1652 /// Emit a load from an l-value of atomic type.  Note that the r-value
1653 /// we produce is an r-value of the atomic *value* type.
1654 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1655                                        llvm::AtomicOrdering AO, bool IsVolatile,
1656                                        AggValueSlot resultSlot) {
1657   AtomicInfo Atomics(*this, src);
1658   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1659                                 IsVolatile);
1660 }
1661 
1662 /// Copy an r-value into memory as part of storing to an atomic type.
1663 /// This needs to create a bit-pattern suitable for atomic operations.
1664 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1665   assert(LVal.isSimple());
1666   // If we have an r-value, the rvalue should be of the atomic type,
1667   // which means that the caller is responsible for having zeroed
1668   // any padding.  Just do an aggregate copy of that type.
1669   if (rvalue.isAggregate()) {
1670     LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1671     LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1672                                     getAtomicType());
1673     bool IsVolatile = rvalue.isVolatileQualified() ||
1674                       LVal.isVolatileQualified();
1675     CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1676                           AggValueSlot::DoesNotOverlap, IsVolatile);
1677     return;
1678   }
1679 
1680   // Okay, otherwise we're copying stuff.
1681 
1682   // Zero out the buffer if necessary.
1683   emitMemSetZeroIfNecessary();
1684 
1685   // Drill past the padding if present.
1686   LValue TempLVal = projectValue();
1687 
1688   // Okay, store the rvalue in.
1689   if (rvalue.isScalar()) {
1690     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1691   } else {
1692     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1693   }
1694 }
1695 
1696 
1697 /// Materialize an r-value into memory for the purposes of storing it
1698 /// to an atomic type.
1699 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1700   // Aggregate r-values are already in memory, and EmitAtomicStore
1701   // requires them to be values of the atomic type.
1702   if (rvalue.isAggregate())
1703     return rvalue.getAggregateAddress();
1704 
1705   // Otherwise, make a temporary and materialize into it.
1706   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1707   AtomicInfo Atomics(CGF, TempLV);
1708   Atomics.emitCopyIntoMemory(rvalue);
1709   return TempLV.getAddress(CGF);
1710 }
1711 
1712 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1713   // If we've got a scalar value of the right size, try to avoid going
1714   // through memory.
1715   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1716     llvm::Value *Value = RVal.getScalarVal();
1717     if (isa<llvm::IntegerType>(Value->getType()))
1718       return CGF.EmitToMemory(Value, ValueTy);
1719     else {
1720       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1721           CGF.getLLVMContext(),
1722           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1723       if (isa<llvm::PointerType>(Value->getType()))
1724         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1725       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1726         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1727     }
1728   }
1729   // Otherwise, we need to go through memory.
1730   // Put the r-value in memory.
1731   Address Addr = materializeRValue(RVal);
1732 
1733   // Cast the temporary to the atomic int type and pull a value out.
1734   Addr = emitCastToAtomicIntPointer(Addr);
1735   return CGF.Builder.CreateLoad(Addr);
1736 }
1737 
1738 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1739     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1740     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1741   // Do the atomic store.
1742   Address Addr = getAtomicAddressAsAtomicIntPointer();
1743   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1744                                                ExpectedVal, DesiredVal,
1745                                                Success, Failure);
1746   // Other decoration.
1747   Inst->setVolatile(LVal.isVolatileQualified());
1748   Inst->setWeak(IsWeak);
1749 
1750   // Okay, turn that back into the original value type.
1751   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1752   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1753   return std::make_pair(PreviousVal, SuccessFailureVal);
1754 }
1755 
1756 llvm::Value *
1757 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1758                                              llvm::Value *DesiredAddr,
1759                                              llvm::AtomicOrdering Success,
1760                                              llvm::AtomicOrdering Failure) {
1761   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1762   // void *desired, int success, int failure);
1763   CallArgList Args;
1764   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1765   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1766            CGF.getContext().VoidPtrTy);
1767   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1768            CGF.getContext().VoidPtrTy);
1769   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1770            CGF.getContext().VoidPtrTy);
1771   Args.add(RValue::get(
1772                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1773            CGF.getContext().IntTy);
1774   Args.add(RValue::get(
1775                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1776            CGF.getContext().IntTy);
1777   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1778                                               CGF.getContext().BoolTy, Args);
1779 
1780   return SuccessFailureRVal.getScalarVal();
1781 }
1782 
1783 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1784     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1785     llvm::AtomicOrdering Failure, bool IsWeak) {
1786   // Check whether we should use a library call.
1787   if (shouldUseLibcall()) {
1788     // Produce a source address.
1789     Address ExpectedAddr = materializeRValue(Expected);
1790     Address DesiredAddr = materializeRValue(Desired);
1791     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1792                                                  DesiredAddr.getPointer(),
1793                                                  Success, Failure);
1794     return std::make_pair(
1795         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1796                                   SourceLocation(), /*AsValue=*/false),
1797         Res);
1798   }
1799 
1800   // If we've got a scalar value of the right size, try to avoid going
1801   // through memory.
1802   auto *ExpectedVal = convertRValueToInt(Expected);
1803   auto *DesiredVal = convertRValueToInt(Desired);
1804   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1805                                          Failure, IsWeak);
1806   return std::make_pair(
1807       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1808                                 SourceLocation(), /*AsValue=*/false),
1809       Res.second);
1810 }
1811 
1812 static void
1813 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1814                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1815                       Address DesiredAddr) {
1816   RValue UpRVal;
1817   LValue AtomicLVal = Atomics.getAtomicLValue();
1818   LValue DesiredLVal;
1819   if (AtomicLVal.isSimple()) {
1820     UpRVal = OldRVal;
1821     DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1822   } else {
1823     // Build new lvalue for temp address.
1824     Address Ptr = Atomics.materializeRValue(OldRVal);
1825     LValue UpdateLVal;
1826     if (AtomicLVal.isBitField()) {
1827       UpdateLVal =
1828           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1829                                AtomicLVal.getType(),
1830                                AtomicLVal.getBaseInfo(),
1831                                AtomicLVal.getTBAAInfo());
1832       DesiredLVal =
1833           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1834                                AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1835                                AtomicLVal.getTBAAInfo());
1836     } else if (AtomicLVal.isVectorElt()) {
1837       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1838                                          AtomicLVal.getType(),
1839                                          AtomicLVal.getBaseInfo(),
1840                                          AtomicLVal.getTBAAInfo());
1841       DesiredLVal = LValue::MakeVectorElt(
1842           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1843           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1844     } else {
1845       assert(AtomicLVal.isExtVectorElt());
1846       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1847                                             AtomicLVal.getType(),
1848                                             AtomicLVal.getBaseInfo(),
1849                                             AtomicLVal.getTBAAInfo());
1850       DesiredLVal = LValue::MakeExtVectorElt(
1851           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1852           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1853     }
1854     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1855   }
1856   // Store new value in the corresponding memory area.
1857   RValue NewRVal = UpdateOp(UpRVal);
1858   if (NewRVal.isScalar()) {
1859     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1860   } else {
1861     assert(NewRVal.isComplex());
1862     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1863                            /*isInit=*/false);
1864   }
1865 }
1866 
1867 void AtomicInfo::EmitAtomicUpdateLibcall(
1868     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1869     bool IsVolatile) {
1870   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1871 
1872   Address ExpectedAddr = CreateTempAlloca();
1873 
1874   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1875   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1876   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1877   CGF.EmitBlock(ContBB);
1878   Address DesiredAddr = CreateTempAlloca();
1879   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1880       requiresMemSetZero(getAtomicAddress().getElementType())) {
1881     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1882     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1883   }
1884   auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1885                                            AggValueSlot::ignored(),
1886                                            SourceLocation(), /*AsValue=*/false);
1887   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1888   auto *Res =
1889       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1890                                        DesiredAddr.getPointer(),
1891                                        AO, Failure);
1892   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1893   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1894 }
1895 
1896 void AtomicInfo::EmitAtomicUpdateOp(
1897     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1898     bool IsVolatile) {
1899   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1900 
1901   // Do the atomic load.
1902   auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1903   // For non-simple lvalues perform compare-and-swap procedure.
1904   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1905   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1906   auto *CurBB = CGF.Builder.GetInsertBlock();
1907   CGF.EmitBlock(ContBB);
1908   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1909                                              /*NumReservedValues=*/2);
1910   PHI->addIncoming(OldVal, CurBB);
1911   Address NewAtomicAddr = CreateTempAlloca();
1912   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1913   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1914       requiresMemSetZero(getAtomicAddress().getElementType())) {
1915     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1916   }
1917   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1918                                            SourceLocation(), /*AsValue=*/false);
1919   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1920   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1921   // Try to write new value using cmpxchg operation.
1922   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1923   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1924   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1925   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1926 }
1927 
1928 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1929                                   RValue UpdateRVal, Address DesiredAddr) {
1930   LValue AtomicLVal = Atomics.getAtomicLValue();
1931   LValue DesiredLVal;
1932   // Build new lvalue for temp address.
1933   if (AtomicLVal.isBitField()) {
1934     DesiredLVal =
1935         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1936                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1937                              AtomicLVal.getTBAAInfo());
1938   } else if (AtomicLVal.isVectorElt()) {
1939     DesiredLVal =
1940         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1941                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1942                               AtomicLVal.getTBAAInfo());
1943   } else {
1944     assert(AtomicLVal.isExtVectorElt());
1945     DesiredLVal = LValue::MakeExtVectorElt(
1946         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1947         AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1948   }
1949   // Store new value in the corresponding memory area.
1950   assert(UpdateRVal.isScalar());
1951   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1952 }
1953 
1954 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1955                                          RValue UpdateRVal, bool IsVolatile) {
1956   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1957 
1958   Address ExpectedAddr = CreateTempAlloca();
1959 
1960   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1961   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1962   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1963   CGF.EmitBlock(ContBB);
1964   Address DesiredAddr = CreateTempAlloca();
1965   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1966       requiresMemSetZero(getAtomicAddress().getElementType())) {
1967     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1968     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1969   }
1970   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1971   auto *Res =
1972       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1973                                        DesiredAddr.getPointer(),
1974                                        AO, Failure);
1975   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1976   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1977 }
1978 
1979 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1980                                     bool IsVolatile) {
1981   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1982 
1983   // Do the atomic load.
1984   auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1985   // For non-simple lvalues perform compare-and-swap procedure.
1986   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1987   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1988   auto *CurBB = CGF.Builder.GetInsertBlock();
1989   CGF.EmitBlock(ContBB);
1990   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1991                                              /*NumReservedValues=*/2);
1992   PHI->addIncoming(OldVal, CurBB);
1993   Address NewAtomicAddr = CreateTempAlloca();
1994   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1995   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1996       requiresMemSetZero(getAtomicAddress().getElementType())) {
1997     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1998   }
1999   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
2000   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
2001   // Try to write new value using cmpxchg operation.
2002   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2003   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2004   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2005   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2006 }
2007 
2008 void AtomicInfo::EmitAtomicUpdate(
2009     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2010     bool IsVolatile) {
2011   if (shouldUseLibcall()) {
2012     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2013   } else {
2014     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2015   }
2016 }
2017 
2018 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2019                                   bool IsVolatile) {
2020   if (shouldUseLibcall()) {
2021     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2022   } else {
2023     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2024   }
2025 }
2026 
2027 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
2028                                       bool isInit) {
2029   bool IsVolatile = lvalue.isVolatileQualified();
2030   llvm::AtomicOrdering AO;
2031   if (lvalue.getType()->isAtomicType()) {
2032     AO = llvm::AtomicOrdering::SequentiallyConsistent;
2033   } else {
2034     AO = llvm::AtomicOrdering::Release;
2035     IsVolatile = true;
2036   }
2037   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2038 }
2039 
2040 /// Emit a store to an l-value of atomic type.
2041 ///
2042 /// Note that the r-value is expected to be an r-value *of the atomic
2043 /// type*; this means that for aggregate r-values, it should include
2044 /// storage for any padding that was necessary.
2045 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
2046                                       llvm::AtomicOrdering AO, bool IsVolatile,
2047                                       bool isInit) {
2048   // If this is an aggregate r-value, it should agree in type except
2049   // maybe for address-space qualification.
2050   assert(!rvalue.isAggregate() ||
2051          rvalue.getAggregateAddress().getElementType() ==
2052              dest.getAddress(*this).getElementType());
2053 
2054   AtomicInfo atomics(*this, dest);
2055   LValue LVal = atomics.getAtomicLValue();
2056 
2057   // If this is an initialization, just put the value there normally.
2058   if (LVal.isSimple()) {
2059     if (isInit) {
2060       atomics.emitCopyIntoMemory(rvalue);
2061       return;
2062     }
2063 
2064     // Check whether we should use a library call.
2065     if (atomics.shouldUseLibcall()) {
2066       // Produce a source address.
2067       Address srcAddr = atomics.materializeRValue(rvalue);
2068 
2069       // void __atomic_store(size_t size, void *mem, void *val, int order)
2070       CallArgList args;
2071       args.add(RValue::get(atomics.getAtomicSizeValue()),
2072                getContext().getSizeType());
2073       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2074                getContext().VoidPtrTy);
2075       args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2076                getContext().VoidPtrTy);
2077       args.add(
2078           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2079           getContext().IntTy);
2080       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2081       return;
2082     }
2083 
2084     // Okay, we're doing this natively.
2085     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2086 
2087     // Do the atomic store.
2088     Address addr =
2089         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2090     intValue = Builder.CreateIntCast(
2091         intValue, addr.getElementType(), /*isSigned=*/false);
2092     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2093 
2094     if (AO == llvm::AtomicOrdering::Acquire)
2095       AO = llvm::AtomicOrdering::Monotonic;
2096     else if (AO == llvm::AtomicOrdering::AcquireRelease)
2097       AO = llvm::AtomicOrdering::Release;
2098     // Initializations don't need to be atomic.
2099     if (!isInit)
2100       store->setAtomic(AO);
2101 
2102     // Other decoration.
2103     if (IsVolatile)
2104       store->setVolatile(true);
2105     CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2106     return;
2107   }
2108 
2109   // Emit simple atomic update operation.
2110   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2111 }
2112 
2113 /// Emit a compare-and-exchange op for atomic type.
2114 ///
2115 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2116     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2117     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2118     AggValueSlot Slot) {
2119   // If this is an aggregate r-value, it should agree in type except
2120   // maybe for address-space qualification.
2121   assert(!Expected.isAggregate() ||
2122          Expected.getAggregateAddress().getElementType() ==
2123              Obj.getAddress(*this).getElementType());
2124   assert(!Desired.isAggregate() ||
2125          Desired.getAggregateAddress().getElementType() ==
2126              Obj.getAddress(*this).getElementType());
2127   AtomicInfo Atomics(*this, Obj);
2128 
2129   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2130                                            IsWeak);
2131 }
2132 
2133 void CodeGenFunction::EmitAtomicUpdate(
2134     LValue LVal, llvm::AtomicOrdering AO,
2135     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2136   AtomicInfo Atomics(*this, LVal);
2137   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2138 }
2139 
2140 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2141   AtomicInfo atomics(*this, dest);
2142 
2143   switch (atomics.getEvaluationKind()) {
2144   case TEK_Scalar: {
2145     llvm::Value *value = EmitScalarExpr(init);
2146     atomics.emitCopyIntoMemory(RValue::get(value));
2147     return;
2148   }
2149 
2150   case TEK_Complex: {
2151     ComplexPairTy value = EmitComplexExpr(init);
2152     atomics.emitCopyIntoMemory(RValue::getComplex(value));
2153     return;
2154   }
2155 
2156   case TEK_Aggregate: {
2157     // Fix up the destination if the initializer isn't an expression
2158     // of atomic type.
2159     bool Zeroed = false;
2160     if (!init->getType()->isAtomicType()) {
2161       Zeroed = atomics.emitMemSetZeroIfNecessary();
2162       dest = atomics.projectValue();
2163     }
2164 
2165     // Evaluate the expression directly into the destination.
2166     AggValueSlot slot = AggValueSlot::forLValue(
2167         dest, *this, AggValueSlot::IsNotDestructed,
2168         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2169         AggValueSlot::DoesNotOverlap,
2170         Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2171 
2172     EmitAggExpr(init, slot);
2173     return;
2174   }
2175   }
2176   llvm_unreachable("bad evaluation kind");
2177 }
2178