1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the code for emitting atomic operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCall.h"
15 #include "CGRecordLayout.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/ASTContext.h"
20 #include "clang/CodeGen/CGFunctionInfo.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30   class AtomicInfo {
31     CodeGenFunction &CGF;
32     QualType AtomicTy;
33     QualType ValueTy;
34     uint64_t AtomicSizeInBits;
35     uint64_t ValueSizeInBits;
36     CharUnits AtomicAlign;
37     CharUnits ValueAlign;
38     CharUnits LValueAlign;
39     TypeEvaluationKind EvaluationKind;
40     bool UseLibcall;
41     LValue LVal;
42     CGBitFieldInfo BFI;
43   public:
44     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
45         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
46           EvaluationKind(TEK_Scalar), UseLibcall(true) {
47       assert(!lvalue.isGlobalReg());
48       ASTContext &C = CGF.getContext();
49       if (lvalue.isSimple()) {
50         AtomicTy = lvalue.getType();
51         if (auto *ATy = AtomicTy->getAs<AtomicType>())
52           ValueTy = ATy->getValueType();
53         else
54           ValueTy = AtomicTy;
55         EvaluationKind = CGF.getEvaluationKind(ValueTy);
56 
57         uint64_t ValueAlignInBits;
58         uint64_t AtomicAlignInBits;
59         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
60         ValueSizeInBits = ValueTI.Width;
61         ValueAlignInBits = ValueTI.Align;
62 
63         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
64         AtomicSizeInBits = AtomicTI.Width;
65         AtomicAlignInBits = AtomicTI.Align;
66 
67         assert(ValueSizeInBits <= AtomicSizeInBits);
68         assert(ValueAlignInBits <= AtomicAlignInBits);
69 
70         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
71         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
72         if (lvalue.getAlignment().isZero())
73           lvalue.setAlignment(AtomicAlign);
74 
75         LVal = lvalue;
76       } else if (lvalue.isBitField()) {
77         ValueTy = lvalue.getType();
78         ValueSizeInBits = C.getTypeSize(ValueTy);
79         auto &OrigBFI = lvalue.getBitFieldInfo();
80         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
81         AtomicSizeInBits = C.toBits(
82             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
83                 .alignTo(lvalue.getAlignment()));
84         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
85         auto OffsetInChars =
86             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
87             lvalue.getAlignment();
88         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
89             VoidPtrAddr, OffsetInChars.getQuantity());
90         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
91             VoidPtrAddr,
92             CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
93             "atomic_bitfield_base");
94         BFI = OrigBFI;
95         BFI.Offset = Offset;
96         BFI.StorageSize = AtomicSizeInBits;
97         BFI.StorageOffset += OffsetInChars;
98         LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
99                                     BFI, lvalue.getType(), lvalue.getBaseInfo(),
100                                     lvalue.getTBAAInfo());
101         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
102         if (AtomicTy.isNull()) {
103           llvm::APInt Size(
104               /*numBits=*/32,
105               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
106           AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
107                                             /*IndexTypeQuals=*/0);
108         }
109         AtomicAlign = ValueAlign = lvalue.getAlignment();
110       } else if (lvalue.isVectorElt()) {
111         ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
112         ValueSizeInBits = C.getTypeSize(ValueTy);
113         AtomicTy = lvalue.getType();
114         AtomicSizeInBits = C.getTypeSize(AtomicTy);
115         AtomicAlign = ValueAlign = lvalue.getAlignment();
116         LVal = lvalue;
117       } else {
118         assert(lvalue.isExtVectorElt());
119         ValueTy = lvalue.getType();
120         ValueSizeInBits = C.getTypeSize(ValueTy);
121         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122             lvalue.getType(), lvalue.getExtVectorAddress()
123                                   .getElementType()->getVectorNumElements());
124         AtomicSizeInBits = C.getTypeSize(AtomicTy);
125         AtomicAlign = ValueAlign = lvalue.getAlignment();
126         LVal = lvalue;
127       }
128       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130     }
131 
132     QualType getAtomicType() const { return AtomicTy; }
133     QualType getValueType() const { return ValueTy; }
134     CharUnits getAtomicAlignment() const { return AtomicAlign; }
135     CharUnits getValueAlignment() const { return ValueAlign; }
136     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139     bool shouldUseLibcall() const { return UseLibcall; }
140     const LValue &getAtomicLValue() const { return LVal; }
141     llvm::Value *getAtomicPointer() const {
142       if (LVal.isSimple())
143         return LVal.getPointer();
144       else if (LVal.isBitField())
145         return LVal.getBitFieldPointer();
146       else if (LVal.isVectorElt())
147         return LVal.getVectorPointer();
148       assert(LVal.isExtVectorElt());
149       return LVal.getExtVectorPointer();
150     }
151     Address getAtomicAddress() const {
152       return Address(getAtomicPointer(), getAtomicAlignment());
153     }
154 
155     Address getAtomicAddressAsAtomicIntPointer() const {
156       return emitCastToAtomicIntPointer(getAtomicAddress());
157     }
158 
159     /// Is the atomic size larger than the underlying value type?
160     ///
161     /// Note that the absence of padding does not mean that atomic
162     /// objects are completely interchangeable with non-atomic
163     /// objects: we might have promoted the alignment of a type
164     /// without making it bigger.
165     bool hasPadding() const {
166       return (ValueSizeInBits != AtomicSizeInBits);
167     }
168 
169     bool emitMemSetZeroIfNecessary() const;
170 
171     llvm::Value *getAtomicSizeValue() const {
172       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173       return CGF.CGM.getSize(size);
174     }
175 
176     /// Cast the given pointer to an integer pointer suitable for atomic
177     /// operations if the source.
178     Address emitCastToAtomicIntPointer(Address Addr) const;
179 
180     /// If Addr is compatible with the iN that will be used for an atomic
181     /// operation, bitcast it. Otherwise, create a temporary that is suitable
182     /// and copy the value across.
183     Address convertToAtomicIntPointer(Address Addr) const;
184 
185     /// Turn an atomic-layout object into an r-value.
186     RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187                                      SourceLocation loc, bool AsValue) const;
188 
189     /// \brief Converts a rvalue to integer value.
190     llvm::Value *convertRValueToInt(RValue RVal) const;
191 
192     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193                                      AggValueSlot ResultSlot,
194                                      SourceLocation Loc, bool AsValue) const;
195 
196     /// Copy an atomic r-value into atomic-layout memory.
197     void emitCopyIntoMemory(RValue rvalue) const;
198 
199     /// Project an l-value down to the value field.
200     LValue projectValue() const {
201       assert(LVal.isSimple());
202       Address addr = getAtomicAddress();
203       if (hasPadding())
204         addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
205 
206       return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207                               LVal.getBaseInfo(), LVal.getTBAAInfo());
208     }
209 
210     /// \brief Emits atomic load.
211     /// \returns Loaded value.
212     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213                           bool AsValue, llvm::AtomicOrdering AO,
214                           bool IsVolatile);
215 
216     /// \brief Emits atomic compare-and-exchange sequence.
217     /// \param Expected Expected value.
218     /// \param Desired Desired value.
219     /// \param Success Atomic ordering for success operation.
220     /// \param Failure Atomic ordering for failed operation.
221     /// \param IsWeak true if atomic operation is weak, false otherwise.
222     /// \returns Pair of values: previous value from storage (value type) and
223     /// boolean flag (i1 type) with true if success and false otherwise.
224     std::pair<RValue, llvm::Value *>
225     EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226                               llvm::AtomicOrdering Success =
227                                   llvm::AtomicOrdering::SequentiallyConsistent,
228                               llvm::AtomicOrdering Failure =
229                                   llvm::AtomicOrdering::SequentiallyConsistent,
230                               bool IsWeak = false);
231 
232     /// \brief Emits atomic update.
233     /// \param AO Atomic ordering.
234     /// \param UpdateOp Update operation for the current lvalue.
235     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
237                           bool IsVolatile);
238     /// \brief Emits atomic update.
239     /// \param AO Atomic ordering.
240     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241                           bool IsVolatile);
242 
243     /// Materialize an atomic r-value in atomic-layout memory.
244     Address materializeRValue(RValue rvalue) const;
245 
246     /// \brief Creates temp alloca for intermediate operations on atomic value.
247     Address CreateTempAlloca() const;
248   private:
249     bool requiresMemSetZero(llvm::Type *type) const;
250 
251 
252     /// \brief Emits atomic load as a libcall.
253     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254                                llvm::AtomicOrdering AO, bool IsVolatile);
255     /// \brief Emits atomic load as LLVM instruction.
256     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257     /// \brief Emits atomic compare-and-exchange op as a libcall.
258     llvm::Value *EmitAtomicCompareExchangeLibcall(
259         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260         llvm::AtomicOrdering Success =
261             llvm::AtomicOrdering::SequentiallyConsistent,
262         llvm::AtomicOrdering Failure =
263             llvm::AtomicOrdering::SequentiallyConsistent);
264     /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
265     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267         llvm::AtomicOrdering Success =
268             llvm::AtomicOrdering::SequentiallyConsistent,
269         llvm::AtomicOrdering Failure =
270             llvm::AtomicOrdering::SequentiallyConsistent,
271         bool IsWeak = false);
272     /// \brief Emit atomic update as libcalls.
273     void
274     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
276                             bool IsVolatile);
277     /// \brief Emit atomic update as LLVM instructions.
278     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
280                             bool IsVolatile);
281     /// \brief Emit atomic update as libcalls.
282     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283                                  bool IsVolatile);
284     /// \brief Emit atomic update as LLVM instructions.
285     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286                             bool IsVolatile);
287   };
288 }
289 
290 Address AtomicInfo::CreateTempAlloca() const {
291   Address TempAlloca = CGF.CreateMemTemp(
292       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293                                                                 : AtomicTy,
294       getAtomicAlignment(),
295       "atomic-temp");
296   // Cast to pointer to value type for bitfields.
297   if (LVal.isBitField())
298     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
299         TempAlloca, getAtomicAddress().getType());
300   return TempAlloca;
301 }
302 
303 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
304                                 StringRef fnName,
305                                 QualType resultType,
306                                 CallArgList &args) {
307   const CGFunctionInfo &fnInfo =
308     CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310   llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
311   auto callee = CGCallee::forDirect(fn);
312   return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
313 }
314 
315 /// Does a store of the given IR type modify the full expected width?
316 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
317                            uint64_t expectedSize) {
318   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
319 }
320 
321 /// Does the atomic type require memsetting to zero before initialization?
322 ///
323 /// The IR type is provided as a way of making certain queries faster.
324 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
325   // If the atomic type has size padding, we definitely need a memset.
326   if (hasPadding()) return true;
327 
328   // Otherwise, do some simple heuristics to try to avoid it:
329   switch (getEvaluationKind()) {
330   // For scalars and complexes, check whether the store size of the
331   // type uses the full size.
332   case TEK_Scalar:
333     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
334   case TEK_Complex:
335     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
336                            AtomicSizeInBits / 2);
337 
338   // Padding in structs has an undefined bit pattern.  User beware.
339   case TEK_Aggregate:
340     return false;
341   }
342   llvm_unreachable("bad evaluation kind");
343 }
344 
345 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
346   assert(LVal.isSimple());
347   llvm::Value *addr = LVal.getPointer();
348   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
349     return false;
350 
351   CGF.Builder.CreateMemSet(
352       addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
353       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
354       LVal.getAlignment().getQuantity());
355   return true;
356 }
357 
358 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
359                               Address Dest, Address Ptr,
360                               Address Val1, Address Val2,
361                               uint64_t Size,
362                               llvm::AtomicOrdering SuccessOrder,
363                               llvm::AtomicOrdering FailureOrder,
364                               llvm::SyncScope::ID Scope) {
365   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
366   llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
367   llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
368 
369   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
370       Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
371       Scope);
372   Pair->setVolatile(E->isVolatile());
373   Pair->setWeak(IsWeak);
374 
375   // Cmp holds the result of the compare-exchange operation: true on success,
376   // false on failure.
377   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
378   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
379 
380   // This basic block is used to hold the store instruction if the operation
381   // failed.
382   llvm::BasicBlock *StoreExpectedBB =
383       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
384 
385   // This basic block is the exit point of the operation, we should end up
386   // here regardless of whether or not the operation succeeded.
387   llvm::BasicBlock *ContinueBB =
388       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
389 
390   // Update Expected if Expected isn't equal to Old, otherwise branch to the
391   // exit point.
392   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
393 
394   CGF.Builder.SetInsertPoint(StoreExpectedBB);
395   // Update the memory at Expected with Old's value.
396   CGF.Builder.CreateStore(Old, Val1);
397   // Finally, branch to the exit point.
398   CGF.Builder.CreateBr(ContinueBB);
399 
400   CGF.Builder.SetInsertPoint(ContinueBB);
401   // Update the memory at Dest with Cmp's value.
402   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
403 }
404 
405 /// Given an ordering required on success, emit all possible cmpxchg
406 /// instructions to cope with the provided (but possibly only dynamically known)
407 /// FailureOrder.
408 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
409                                         bool IsWeak, Address Dest, Address Ptr,
410                                         Address Val1, Address Val2,
411                                         llvm::Value *FailureOrderVal,
412                                         uint64_t Size,
413                                         llvm::AtomicOrdering SuccessOrder,
414                                         llvm::SyncScope::ID Scope) {
415   llvm::AtomicOrdering FailureOrder;
416   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
417     auto FOS = FO->getSExtValue();
418     if (!llvm::isValidAtomicOrderingCABI(FOS))
419       FailureOrder = llvm::AtomicOrdering::Monotonic;
420     else
421       switch ((llvm::AtomicOrderingCABI)FOS) {
422       case llvm::AtomicOrderingCABI::relaxed:
423       case llvm::AtomicOrderingCABI::release:
424       case llvm::AtomicOrderingCABI::acq_rel:
425         FailureOrder = llvm::AtomicOrdering::Monotonic;
426         break;
427       case llvm::AtomicOrderingCABI::consume:
428       case llvm::AtomicOrderingCABI::acquire:
429         FailureOrder = llvm::AtomicOrdering::Acquire;
430         break;
431       case llvm::AtomicOrderingCABI::seq_cst:
432         FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
433         break;
434       }
435     if (isStrongerThan(FailureOrder, SuccessOrder)) {
436       // Don't assert on undefined behavior "failure argument shall be no
437       // stronger than the success argument".
438       FailureOrder =
439           llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
440     }
441     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
442                       FailureOrder, Scope);
443     return;
444   }
445 
446   // Create all the relevant BB's
447   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
448                    *SeqCstBB = nullptr;
449   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
450   if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
451       SuccessOrder != llvm::AtomicOrdering::Release)
452     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
453   if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
454     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
455 
456   llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
457 
458   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
459 
460   // Emit all the different atomics
461 
462   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
463   // doesn't matter unless someone is crazy enough to use something that
464   // doesn't fold to a constant for the ordering.
465   CGF.Builder.SetInsertPoint(MonotonicBB);
466   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
467                     Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
468   CGF.Builder.CreateBr(ContBB);
469 
470   if (AcquireBB) {
471     CGF.Builder.SetInsertPoint(AcquireBB);
472     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
473                       Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
474     CGF.Builder.CreateBr(ContBB);
475     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
476                 AcquireBB);
477     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
478                 AcquireBB);
479   }
480   if (SeqCstBB) {
481     CGF.Builder.SetInsertPoint(SeqCstBB);
482     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
483                       llvm::AtomicOrdering::SequentiallyConsistent, Scope);
484     CGF.Builder.CreateBr(ContBB);
485     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
486                 SeqCstBB);
487   }
488 
489   CGF.Builder.SetInsertPoint(ContBB);
490 }
491 
492 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
493                          Address Ptr, Address Val1, Address Val2,
494                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
495                          uint64_t Size, llvm::AtomicOrdering Order,
496                          llvm::SyncScope::ID Scope) {
497   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
498   llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
499 
500   switch (E->getOp()) {
501   case AtomicExpr::AO__c11_atomic_init:
502   case AtomicExpr::AO__opencl_atomic_init:
503     llvm_unreachable("Already handled!");
504 
505   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
506   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
507     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
508                                 FailureOrder, Size, Order, Scope);
509     return;
510   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
511   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
512     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
513                                 FailureOrder, Size, Order, Scope);
514     return;
515   case AtomicExpr::AO__atomic_compare_exchange:
516   case AtomicExpr::AO__atomic_compare_exchange_n: {
517     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
518       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
519                                   Val1, Val2, FailureOrder, Size, Order, Scope);
520     } else {
521       // Create all the relevant BB's
522       llvm::BasicBlock *StrongBB =
523           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
524       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
525       llvm::BasicBlock *ContBB =
526           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
527 
528       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
529       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
530 
531       CGF.Builder.SetInsertPoint(StrongBB);
532       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
533                                   FailureOrder, Size, Order, Scope);
534       CGF.Builder.CreateBr(ContBB);
535 
536       CGF.Builder.SetInsertPoint(WeakBB);
537       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
538                                   FailureOrder, Size, Order, Scope);
539       CGF.Builder.CreateBr(ContBB);
540 
541       CGF.Builder.SetInsertPoint(ContBB);
542     }
543     return;
544   }
545   case AtomicExpr::AO__c11_atomic_load:
546   case AtomicExpr::AO__opencl_atomic_load:
547   case AtomicExpr::AO__atomic_load_n:
548   case AtomicExpr::AO__atomic_load: {
549     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
550     Load->setAtomic(Order, Scope);
551     Load->setVolatile(E->isVolatile());
552     CGF.Builder.CreateStore(Load, Dest);
553     return;
554   }
555 
556   case AtomicExpr::AO__c11_atomic_store:
557   case AtomicExpr::AO__opencl_atomic_store:
558   case AtomicExpr::AO__atomic_store:
559   case AtomicExpr::AO__atomic_store_n: {
560     llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
561     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
562     Store->setAtomic(Order, Scope);
563     Store->setVolatile(E->isVolatile());
564     return;
565   }
566 
567   case AtomicExpr::AO__c11_atomic_exchange:
568   case AtomicExpr::AO__opencl_atomic_exchange:
569   case AtomicExpr::AO__atomic_exchange_n:
570   case AtomicExpr::AO__atomic_exchange:
571     Op = llvm::AtomicRMWInst::Xchg;
572     break;
573 
574   case AtomicExpr::AO__atomic_add_fetch:
575     PostOp = llvm::Instruction::Add;
576     LLVM_FALLTHROUGH;
577   case AtomicExpr::AO__c11_atomic_fetch_add:
578   case AtomicExpr::AO__opencl_atomic_fetch_add:
579   case AtomicExpr::AO__atomic_fetch_add:
580     Op = llvm::AtomicRMWInst::Add;
581     break;
582 
583   case AtomicExpr::AO__atomic_sub_fetch:
584     PostOp = llvm::Instruction::Sub;
585     LLVM_FALLTHROUGH;
586   case AtomicExpr::AO__c11_atomic_fetch_sub:
587   case AtomicExpr::AO__opencl_atomic_fetch_sub:
588   case AtomicExpr::AO__atomic_fetch_sub:
589     Op = llvm::AtomicRMWInst::Sub;
590     break;
591 
592   case AtomicExpr::AO__opencl_atomic_fetch_min:
593     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
594                                                   : llvm::AtomicRMWInst::UMin;
595     break;
596 
597   case AtomicExpr::AO__opencl_atomic_fetch_max:
598     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
599                                                   : llvm::AtomicRMWInst::UMax;
600     break;
601 
602   case AtomicExpr::AO__atomic_and_fetch:
603     PostOp = llvm::Instruction::And;
604     LLVM_FALLTHROUGH;
605   case AtomicExpr::AO__c11_atomic_fetch_and:
606   case AtomicExpr::AO__opencl_atomic_fetch_and:
607   case AtomicExpr::AO__atomic_fetch_and:
608     Op = llvm::AtomicRMWInst::And;
609     break;
610 
611   case AtomicExpr::AO__atomic_or_fetch:
612     PostOp = llvm::Instruction::Or;
613     LLVM_FALLTHROUGH;
614   case AtomicExpr::AO__c11_atomic_fetch_or:
615   case AtomicExpr::AO__opencl_atomic_fetch_or:
616   case AtomicExpr::AO__atomic_fetch_or:
617     Op = llvm::AtomicRMWInst::Or;
618     break;
619 
620   case AtomicExpr::AO__atomic_xor_fetch:
621     PostOp = llvm::Instruction::Xor;
622     LLVM_FALLTHROUGH;
623   case AtomicExpr::AO__c11_atomic_fetch_xor:
624   case AtomicExpr::AO__opencl_atomic_fetch_xor:
625   case AtomicExpr::AO__atomic_fetch_xor:
626     Op = llvm::AtomicRMWInst::Xor;
627     break;
628 
629   case AtomicExpr::AO__atomic_nand_fetch:
630     PostOp = llvm::Instruction::And; // the NOT is special cased below
631     LLVM_FALLTHROUGH;
632   case AtomicExpr::AO__atomic_fetch_nand:
633     Op = llvm::AtomicRMWInst::Nand;
634     break;
635   }
636 
637   llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
638   llvm::AtomicRMWInst *RMWI =
639       CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
640   RMWI->setVolatile(E->isVolatile());
641 
642   // For __atomic_*_fetch operations, perform the operation again to
643   // determine the value which was written.
644   llvm::Value *Result = RMWI;
645   if (PostOp)
646     Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
647   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
648     Result = CGF.Builder.CreateNot(Result);
649   CGF.Builder.CreateStore(Result, Dest);
650 }
651 
652 // This function emits any expression (scalar, complex, or aggregate)
653 // into a temporary alloca.
654 static Address
655 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
656   Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
657   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
658                        /*Init*/ true);
659   return DeclPtr;
660 }
661 
662 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
663                          Address Ptr, Address Val1, Address Val2,
664                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
665                          uint64_t Size, llvm::AtomicOrdering Order,
666                          llvm::Value *Scope) {
667   auto ScopeModel = Expr->getScopeModel();
668 
669   // LLVM atomic instructions always have synch scope. If clang atomic
670   // expression has no scope operand, use default LLVM synch scope.
671   if (!ScopeModel) {
672     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
673                  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
674     return;
675   }
676 
677   // Handle constant scope.
678   if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
679     auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
680         ScopeModel->map(SC->getZExtValue()), CGF.CGM.getLLVMContext());
681     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
682                  Order, SCID);
683     return;
684   }
685 
686   // Handle non-constant scope.
687   auto &Builder = CGF.Builder;
688   auto Scopes = ScopeModel->getRuntimeValues();
689   llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
690   for (auto S : Scopes)
691     BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
692 
693   llvm::BasicBlock *ContBB =
694       CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
695 
696   auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
697   // If unsupported synch scope is encountered at run time, assume a fallback
698   // synch scope value.
699   auto FallBack = ScopeModel->getFallBackValue();
700   llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
701   for (auto S : Scopes) {
702     auto *B = BB[S];
703     if (S != FallBack)
704       SI->addCase(Builder.getInt32(S), B);
705 
706     Builder.SetInsertPoint(B);
707     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
708                  Order,
709                  CGF.getTargetHooks().getLLVMSyncScopeID(ScopeModel->map(S),
710                                                          CGF.getLLVMContext()));
711     Builder.CreateBr(ContBB);
712   }
713 
714   Builder.SetInsertPoint(ContBB);
715 }
716 
717 static void
718 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
719                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
720                   SourceLocation Loc, CharUnits SizeInChars) {
721   if (UseOptimizedLibcall) {
722     // Load value and pass it to the function directly.
723     CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
724     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
725     ValTy =
726         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
727     llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
728                                                 SizeInBits)->getPointerTo();
729     Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
730     Val = CGF.EmitLoadOfScalar(Ptr, false,
731                                CGF.getContext().getPointerType(ValTy),
732                                Loc);
733     // Coerce the value into an appropriately sized integer type.
734     Args.add(RValue::get(Val), ValTy);
735   } else {
736     // Non-optimized functions always take a reference.
737     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
738                          CGF.getContext().VoidPtrTy);
739   }
740 }
741 
742 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
743   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
744   QualType MemTy = AtomicTy;
745   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
746     MemTy = AT->getValueType();
747   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
748 
749   Address Val1 = Address::invalid();
750   Address Val2 = Address::invalid();
751   Address Dest = Address::invalid();
752   Address Ptr = EmitPointerWithAlignment(E->getPtr());
753 
754   CharUnits sizeChars, alignChars;
755   std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
756   uint64_t Size = sizeChars.getQuantity();
757   unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
758   bool UseLibcall = ((Ptr.getAlignment() % sizeChars) != 0 ||
759                      getContext().toBits(sizeChars) > MaxInlineWidthInBits);
760 
761   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
762       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
763     LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
764     EmitAtomicInit(E->getVal1(), lvalue);
765     return RValue::get(nullptr);
766   }
767 
768   llvm::Value *Order = EmitScalarExpr(E->getOrder());
769   llvm::Value *Scope =
770       E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
771 
772   switch (E->getOp()) {
773   case AtomicExpr::AO__c11_atomic_init:
774   case AtomicExpr::AO__opencl_atomic_init:
775     llvm_unreachable("Already handled above with EmitAtomicInit!");
776 
777   case AtomicExpr::AO__c11_atomic_load:
778   case AtomicExpr::AO__opencl_atomic_load:
779   case AtomicExpr::AO__atomic_load_n:
780     break;
781 
782   case AtomicExpr::AO__atomic_load:
783     Dest = EmitPointerWithAlignment(E->getVal1());
784     break;
785 
786   case AtomicExpr::AO__atomic_store:
787     Val1 = EmitPointerWithAlignment(E->getVal1());
788     break;
789 
790   case AtomicExpr::AO__atomic_exchange:
791     Val1 = EmitPointerWithAlignment(E->getVal1());
792     Dest = EmitPointerWithAlignment(E->getVal2());
793     break;
794 
795   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
796   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
797   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
798   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
799   case AtomicExpr::AO__atomic_compare_exchange_n:
800   case AtomicExpr::AO__atomic_compare_exchange:
801     Val1 = EmitPointerWithAlignment(E->getVal1());
802     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
803       Val2 = EmitPointerWithAlignment(E->getVal2());
804     else
805       Val2 = EmitValToTemp(*this, E->getVal2());
806     OrderFail = EmitScalarExpr(E->getOrderFail());
807     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
808         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
809       IsWeak = EmitScalarExpr(E->getWeak());
810     break;
811 
812   case AtomicExpr::AO__c11_atomic_fetch_add:
813   case AtomicExpr::AO__c11_atomic_fetch_sub:
814   case AtomicExpr::AO__opencl_atomic_fetch_add:
815   case AtomicExpr::AO__opencl_atomic_fetch_sub:
816     if (MemTy->isPointerType()) {
817       // For pointer arithmetic, we're required to do a bit of math:
818       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
819       // ... but only for the C11 builtins. The GNU builtins expect the
820       // user to multiply by sizeof(T).
821       QualType Val1Ty = E->getVal1()->getType();
822       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
823       CharUnits PointeeIncAmt =
824           getContext().getTypeSizeInChars(MemTy->getPointeeType());
825       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
826       auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
827       Val1 = Temp;
828       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
829       break;
830     }
831       LLVM_FALLTHROUGH;
832   case AtomicExpr::AO__atomic_fetch_add:
833   case AtomicExpr::AO__atomic_fetch_sub:
834   case AtomicExpr::AO__atomic_add_fetch:
835   case AtomicExpr::AO__atomic_sub_fetch:
836   case AtomicExpr::AO__c11_atomic_store:
837   case AtomicExpr::AO__c11_atomic_exchange:
838   case AtomicExpr::AO__opencl_atomic_store:
839   case AtomicExpr::AO__opencl_atomic_exchange:
840   case AtomicExpr::AO__atomic_store_n:
841   case AtomicExpr::AO__atomic_exchange_n:
842   case AtomicExpr::AO__c11_atomic_fetch_and:
843   case AtomicExpr::AO__c11_atomic_fetch_or:
844   case AtomicExpr::AO__c11_atomic_fetch_xor:
845   case AtomicExpr::AO__opencl_atomic_fetch_and:
846   case AtomicExpr::AO__opencl_atomic_fetch_or:
847   case AtomicExpr::AO__opencl_atomic_fetch_xor:
848   case AtomicExpr::AO__opencl_atomic_fetch_min:
849   case AtomicExpr::AO__opencl_atomic_fetch_max:
850   case AtomicExpr::AO__atomic_fetch_and:
851   case AtomicExpr::AO__atomic_fetch_or:
852   case AtomicExpr::AO__atomic_fetch_xor:
853   case AtomicExpr::AO__atomic_fetch_nand:
854   case AtomicExpr::AO__atomic_and_fetch:
855   case AtomicExpr::AO__atomic_or_fetch:
856   case AtomicExpr::AO__atomic_xor_fetch:
857   case AtomicExpr::AO__atomic_nand_fetch:
858     Val1 = EmitValToTemp(*this, E->getVal1());
859     break;
860   }
861 
862   QualType RValTy = E->getType().getUnqualifiedType();
863 
864   // The inlined atomics only function on iN types, where N is a power of 2. We
865   // need to make sure (via temporaries if necessary) that all incoming values
866   // are compatible.
867   LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
868   AtomicInfo Atomics(*this, AtomicVal);
869 
870   Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
871   if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
872   if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
873   if (Dest.isValid())
874     Dest = Atomics.emitCastToAtomicIntPointer(Dest);
875   else if (E->isCmpXChg())
876     Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
877   else if (!RValTy->isVoidType())
878     Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
879 
880   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
881   if (UseLibcall) {
882     bool UseOptimizedLibcall = false;
883     switch (E->getOp()) {
884     case AtomicExpr::AO__c11_atomic_init:
885     case AtomicExpr::AO__opencl_atomic_init:
886       llvm_unreachable("Already handled above with EmitAtomicInit!");
887 
888     case AtomicExpr::AO__c11_atomic_fetch_add:
889     case AtomicExpr::AO__opencl_atomic_fetch_add:
890     case AtomicExpr::AO__atomic_fetch_add:
891     case AtomicExpr::AO__c11_atomic_fetch_and:
892     case AtomicExpr::AO__opencl_atomic_fetch_and:
893     case AtomicExpr::AO__atomic_fetch_and:
894     case AtomicExpr::AO__c11_atomic_fetch_or:
895     case AtomicExpr::AO__opencl_atomic_fetch_or:
896     case AtomicExpr::AO__atomic_fetch_or:
897     case AtomicExpr::AO__atomic_fetch_nand:
898     case AtomicExpr::AO__c11_atomic_fetch_sub:
899     case AtomicExpr::AO__opencl_atomic_fetch_sub:
900     case AtomicExpr::AO__atomic_fetch_sub:
901     case AtomicExpr::AO__c11_atomic_fetch_xor:
902     case AtomicExpr::AO__opencl_atomic_fetch_xor:
903     case AtomicExpr::AO__opencl_atomic_fetch_min:
904     case AtomicExpr::AO__opencl_atomic_fetch_max:
905     case AtomicExpr::AO__atomic_fetch_xor:
906     case AtomicExpr::AO__atomic_add_fetch:
907     case AtomicExpr::AO__atomic_and_fetch:
908     case AtomicExpr::AO__atomic_nand_fetch:
909     case AtomicExpr::AO__atomic_or_fetch:
910     case AtomicExpr::AO__atomic_sub_fetch:
911     case AtomicExpr::AO__atomic_xor_fetch:
912       // For these, only library calls for certain sizes exist.
913       UseOptimizedLibcall = true;
914       break;
915 
916     case AtomicExpr::AO__c11_atomic_load:
917     case AtomicExpr::AO__c11_atomic_store:
918     case AtomicExpr::AO__c11_atomic_exchange:
919     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
920     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
921     case AtomicExpr::AO__opencl_atomic_load:
922     case AtomicExpr::AO__opencl_atomic_store:
923     case AtomicExpr::AO__opencl_atomic_exchange:
924     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
925     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
926     case AtomicExpr::AO__atomic_load_n:
927     case AtomicExpr::AO__atomic_load:
928     case AtomicExpr::AO__atomic_store_n:
929     case AtomicExpr::AO__atomic_store:
930     case AtomicExpr::AO__atomic_exchange_n:
931     case AtomicExpr::AO__atomic_exchange:
932     case AtomicExpr::AO__atomic_compare_exchange_n:
933     case AtomicExpr::AO__atomic_compare_exchange:
934       // Only use optimized library calls for sizes for which they exist.
935       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
936         UseOptimizedLibcall = true;
937       break;
938     }
939 
940     CallArgList Args;
941     if (!UseOptimizedLibcall) {
942       // For non-optimized library calls, the size is the first parameter
943       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
944                getContext().getSizeType());
945     }
946     // Atomic address is the first or second parameter
947     // The OpenCL atomic library functions only accept pointer arguments to
948     // generic address space.
949     auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
950       if (!E->isOpenCL())
951         return V;
952       auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace();
953       if (AS == LangAS::opencl_generic)
954         return V;
955       auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
956       auto T = V->getType();
957       auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
958 
959       return getTargetHooks().performAddrSpaceCast(
960           *this, V, AS, LangAS::opencl_generic, DestType, false);
961     };
962 
963     Args.add(RValue::get(CastToGenericAddrSpace(
964                  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
965              getContext().VoidPtrTy);
966 
967     std::string LibCallName;
968     QualType LoweredMemTy =
969       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
970     QualType RetTy;
971     bool HaveRetTy = false;
972     llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
973     switch (E->getOp()) {
974     case AtomicExpr::AO__c11_atomic_init:
975     case AtomicExpr::AO__opencl_atomic_init:
976       llvm_unreachable("Already handled!");
977 
978     // There is only one libcall for compare an exchange, because there is no
979     // optimisation benefit possible from a libcall version of a weak compare
980     // and exchange.
981     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
982     //                                void *desired, int success, int failure)
983     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
984     //                                  int success, int failure)
985     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
986     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
987     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
988     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
989     case AtomicExpr::AO__atomic_compare_exchange:
990     case AtomicExpr::AO__atomic_compare_exchange_n:
991       LibCallName = "__atomic_compare_exchange";
992       RetTy = getContext().BoolTy;
993       HaveRetTy = true;
994       Args.add(
995           RValue::get(CastToGenericAddrSpace(
996               EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
997           getContext().VoidPtrTy);
998       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
999                         MemTy, E->getExprLoc(), sizeChars);
1000       Args.add(RValue::get(Order), getContext().IntTy);
1001       Order = OrderFail;
1002       break;
1003     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1004     //                        int order)
1005     // T __atomic_exchange_N(T *mem, T val, int order)
1006     case AtomicExpr::AO__c11_atomic_exchange:
1007     case AtomicExpr::AO__opencl_atomic_exchange:
1008     case AtomicExpr::AO__atomic_exchange_n:
1009     case AtomicExpr::AO__atomic_exchange:
1010       LibCallName = "__atomic_exchange";
1011       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1012                         MemTy, E->getExprLoc(), sizeChars);
1013       break;
1014     // void __atomic_store(size_t size, void *mem, void *val, int order)
1015     // void __atomic_store_N(T *mem, T val, int order)
1016     case AtomicExpr::AO__c11_atomic_store:
1017     case AtomicExpr::AO__opencl_atomic_store:
1018     case AtomicExpr::AO__atomic_store:
1019     case AtomicExpr::AO__atomic_store_n:
1020       LibCallName = "__atomic_store";
1021       RetTy = getContext().VoidTy;
1022       HaveRetTy = true;
1023       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1024                         MemTy, E->getExprLoc(), sizeChars);
1025       break;
1026     // void __atomic_load(size_t size, void *mem, void *return, int order)
1027     // T __atomic_load_N(T *mem, int order)
1028     case AtomicExpr::AO__c11_atomic_load:
1029     case AtomicExpr::AO__opencl_atomic_load:
1030     case AtomicExpr::AO__atomic_load:
1031     case AtomicExpr::AO__atomic_load_n:
1032       LibCallName = "__atomic_load";
1033       break;
1034     // T __atomic_add_fetch_N(T *mem, T val, int order)
1035     // T __atomic_fetch_add_N(T *mem, T val, int order)
1036     case AtomicExpr::AO__atomic_add_fetch:
1037       PostOp = llvm::Instruction::Add;
1038       LLVM_FALLTHROUGH;
1039     case AtomicExpr::AO__c11_atomic_fetch_add:
1040     case AtomicExpr::AO__opencl_atomic_fetch_add:
1041     case AtomicExpr::AO__atomic_fetch_add:
1042       LibCallName = "__atomic_fetch_add";
1043       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1044                         LoweredMemTy, E->getExprLoc(), sizeChars);
1045       break;
1046     // T __atomic_and_fetch_N(T *mem, T val, int order)
1047     // T __atomic_fetch_and_N(T *mem, T val, int order)
1048     case AtomicExpr::AO__atomic_and_fetch:
1049       PostOp = llvm::Instruction::And;
1050       LLVM_FALLTHROUGH;
1051     case AtomicExpr::AO__c11_atomic_fetch_and:
1052     case AtomicExpr::AO__opencl_atomic_fetch_and:
1053     case AtomicExpr::AO__atomic_fetch_and:
1054       LibCallName = "__atomic_fetch_and";
1055       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1056                         MemTy, E->getExprLoc(), sizeChars);
1057       break;
1058     // T __atomic_or_fetch_N(T *mem, T val, int order)
1059     // T __atomic_fetch_or_N(T *mem, T val, int order)
1060     case AtomicExpr::AO__atomic_or_fetch:
1061       PostOp = llvm::Instruction::Or;
1062       LLVM_FALLTHROUGH;
1063     case AtomicExpr::AO__c11_atomic_fetch_or:
1064     case AtomicExpr::AO__opencl_atomic_fetch_or:
1065     case AtomicExpr::AO__atomic_fetch_or:
1066       LibCallName = "__atomic_fetch_or";
1067       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1068                         MemTy, E->getExprLoc(), sizeChars);
1069       break;
1070     // T __atomic_sub_fetch_N(T *mem, T val, int order)
1071     // T __atomic_fetch_sub_N(T *mem, T val, int order)
1072     case AtomicExpr::AO__atomic_sub_fetch:
1073       PostOp = llvm::Instruction::Sub;
1074       LLVM_FALLTHROUGH;
1075     case AtomicExpr::AO__c11_atomic_fetch_sub:
1076     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1077     case AtomicExpr::AO__atomic_fetch_sub:
1078       LibCallName = "__atomic_fetch_sub";
1079       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1080                         LoweredMemTy, E->getExprLoc(), sizeChars);
1081       break;
1082     // T __atomic_xor_fetch_N(T *mem, T val, int order)
1083     // T __atomic_fetch_xor_N(T *mem, T val, int order)
1084     case AtomicExpr::AO__atomic_xor_fetch:
1085       PostOp = llvm::Instruction::Xor;
1086       LLVM_FALLTHROUGH;
1087     case AtomicExpr::AO__c11_atomic_fetch_xor:
1088     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1089     case AtomicExpr::AO__atomic_fetch_xor:
1090       LibCallName = "__atomic_fetch_xor";
1091       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1092                         MemTy, E->getExprLoc(), sizeChars);
1093       break;
1094     case AtomicExpr::AO__opencl_atomic_fetch_min:
1095       LibCallName = E->getValueType()->isSignedIntegerType()
1096                         ? "__atomic_fetch_min"
1097                         : "__atomic_fetch_umin";
1098       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1099                         LoweredMemTy, E->getExprLoc(), sizeChars);
1100       break;
1101     case AtomicExpr::AO__opencl_atomic_fetch_max:
1102       LibCallName = E->getValueType()->isSignedIntegerType()
1103                         ? "__atomic_fetch_max"
1104                         : "__atomic_fetch_umax";
1105       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1106                         LoweredMemTy, E->getExprLoc(), sizeChars);
1107       break;
1108     // T __atomic_nand_fetch_N(T *mem, T val, int order)
1109     // T __atomic_fetch_nand_N(T *mem, T val, int order)
1110     case AtomicExpr::AO__atomic_nand_fetch:
1111       PostOp = llvm::Instruction::And; // the NOT is special cased below
1112       LLVM_FALLTHROUGH;
1113     case AtomicExpr::AO__atomic_fetch_nand:
1114       LibCallName = "__atomic_fetch_nand";
1115       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1116                         MemTy, E->getExprLoc(), sizeChars);
1117       break;
1118     }
1119 
1120     if (E->isOpenCL()) {
1121       LibCallName = std::string("__opencl") +
1122           StringRef(LibCallName).drop_front(1).str();
1123 
1124     }
1125     // Optimized functions have the size in their name.
1126     if (UseOptimizedLibcall)
1127       LibCallName += "_" + llvm::utostr(Size);
1128     // By default, assume we return a value of the atomic type.
1129     if (!HaveRetTy) {
1130       if (UseOptimizedLibcall) {
1131         // Value is returned directly.
1132         // The function returns an appropriately sized integer type.
1133         RetTy = getContext().getIntTypeForBitwidth(
1134             getContext().toBits(sizeChars), /*Signed=*/false);
1135       } else {
1136         // Value is returned through parameter before the order.
1137         RetTy = getContext().VoidTy;
1138         Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1139                  getContext().VoidPtrTy);
1140       }
1141     }
1142     // order is always the last parameter
1143     Args.add(RValue::get(Order),
1144              getContext().IntTy);
1145     if (E->isOpenCL())
1146       Args.add(RValue::get(Scope), getContext().IntTy);
1147 
1148     // PostOp is only needed for the atomic_*_fetch operations, and
1149     // thus is only needed for and implemented in the
1150     // UseOptimizedLibcall codepath.
1151     assert(UseOptimizedLibcall || !PostOp);
1152 
1153     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1154     // The value is returned directly from the libcall.
1155     if (E->isCmpXChg())
1156       return Res;
1157 
1158     // The value is returned directly for optimized libcalls but the expr
1159     // provided an out-param.
1160     if (UseOptimizedLibcall && Res.getScalarVal()) {
1161       llvm::Value *ResVal = Res.getScalarVal();
1162       if (PostOp) {
1163         llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
1164         ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1165       }
1166       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1167         ResVal = Builder.CreateNot(ResVal);
1168 
1169       Builder.CreateStore(
1170           ResVal,
1171           Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1172     }
1173 
1174     if (RValTy->isVoidType())
1175       return RValue::get(nullptr);
1176 
1177     return convertTempToRValue(
1178         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1179         RValTy, E->getExprLoc());
1180   }
1181 
1182   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1183                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1184                  E->getOp() == AtomicExpr::AO__atomic_store ||
1185                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1186   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1187                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1188                 E->getOp() == AtomicExpr::AO__atomic_load ||
1189                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1190 
1191   if (isa<llvm::ConstantInt>(Order)) {
1192     auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1193     // We should not ever get to a case where the ordering isn't a valid C ABI
1194     // value, but it's hard to enforce that in general.
1195     if (llvm::isValidAtomicOrderingCABI(ord))
1196       switch ((llvm::AtomicOrderingCABI)ord) {
1197       case llvm::AtomicOrderingCABI::relaxed:
1198         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1199                      llvm::AtomicOrdering::Monotonic, Scope);
1200         break;
1201       case llvm::AtomicOrderingCABI::consume:
1202       case llvm::AtomicOrderingCABI::acquire:
1203         if (IsStore)
1204           break; // Avoid crashing on code with undefined behavior
1205         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1206                      llvm::AtomicOrdering::Acquire, Scope);
1207         break;
1208       case llvm::AtomicOrderingCABI::release:
1209         if (IsLoad)
1210           break; // Avoid crashing on code with undefined behavior
1211         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1212                      llvm::AtomicOrdering::Release, Scope);
1213         break;
1214       case llvm::AtomicOrderingCABI::acq_rel:
1215         if (IsLoad || IsStore)
1216           break; // Avoid crashing on code with undefined behavior
1217         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1218                      llvm::AtomicOrdering::AcquireRelease, Scope);
1219         break;
1220       case llvm::AtomicOrderingCABI::seq_cst:
1221         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1222                      llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1223         break;
1224       }
1225     if (RValTy->isVoidType())
1226       return RValue::get(nullptr);
1227 
1228     return convertTempToRValue(
1229         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1230                                         Dest.getAddressSpace())),
1231         RValTy, E->getExprLoc());
1232   }
1233 
1234   // Long case, when Order isn't obviously constant.
1235 
1236   // Create all the relevant BB's
1237   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1238                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1239                    *SeqCstBB = nullptr;
1240   MonotonicBB = createBasicBlock("monotonic", CurFn);
1241   if (!IsStore)
1242     AcquireBB = createBasicBlock("acquire", CurFn);
1243   if (!IsLoad)
1244     ReleaseBB = createBasicBlock("release", CurFn);
1245   if (!IsLoad && !IsStore)
1246     AcqRelBB = createBasicBlock("acqrel", CurFn);
1247   SeqCstBB = createBasicBlock("seqcst", CurFn);
1248   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1249 
1250   // Create the switch for the split
1251   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1252   // doesn't matter unless someone is crazy enough to use something that
1253   // doesn't fold to a constant for the ordering.
1254   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1255   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1256 
1257   // Emit all the different atomics
1258   Builder.SetInsertPoint(MonotonicBB);
1259   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1260                llvm::AtomicOrdering::Monotonic, Scope);
1261   Builder.CreateBr(ContBB);
1262   if (!IsStore) {
1263     Builder.SetInsertPoint(AcquireBB);
1264     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1265                  llvm::AtomicOrdering::Acquire, Scope);
1266     Builder.CreateBr(ContBB);
1267     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1268                 AcquireBB);
1269     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1270                 AcquireBB);
1271   }
1272   if (!IsLoad) {
1273     Builder.SetInsertPoint(ReleaseBB);
1274     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1275                  llvm::AtomicOrdering::Release, Scope);
1276     Builder.CreateBr(ContBB);
1277     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1278                 ReleaseBB);
1279   }
1280   if (!IsLoad && !IsStore) {
1281     Builder.SetInsertPoint(AcqRelBB);
1282     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1283                  llvm::AtomicOrdering::AcquireRelease, Scope);
1284     Builder.CreateBr(ContBB);
1285     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1286                 AcqRelBB);
1287   }
1288   Builder.SetInsertPoint(SeqCstBB);
1289   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1290                llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1291   Builder.CreateBr(ContBB);
1292   SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1293               SeqCstBB);
1294 
1295   // Cleanup and return
1296   Builder.SetInsertPoint(ContBB);
1297   if (RValTy->isVoidType())
1298     return RValue::get(nullptr);
1299 
1300   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1301   return convertTempToRValue(
1302       Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1303                                       Dest.getAddressSpace())),
1304       RValTy, E->getExprLoc());
1305 }
1306 
1307 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1308   unsigned addrspace =
1309     cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1310   llvm::IntegerType *ty =
1311     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1312   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1313 }
1314 
1315 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1316   llvm::Type *Ty = Addr.getElementType();
1317   uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1318   if (SourceSizeInBits != AtomicSizeInBits) {
1319     Address Tmp = CreateTempAlloca();
1320     CGF.Builder.CreateMemCpy(Tmp, Addr,
1321                              std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1322     Addr = Tmp;
1323   }
1324 
1325   return emitCastToAtomicIntPointer(Addr);
1326 }
1327 
1328 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1329                                              AggValueSlot resultSlot,
1330                                              SourceLocation loc,
1331                                              bool asValue) const {
1332   if (LVal.isSimple()) {
1333     if (EvaluationKind == TEK_Aggregate)
1334       return resultSlot.asRValue();
1335 
1336     // Drill into the padding structure if we have one.
1337     if (hasPadding())
1338       addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1339 
1340     // Otherwise, just convert the temporary to an r-value using the
1341     // normal conversion routine.
1342     return CGF.convertTempToRValue(addr, getValueType(), loc);
1343   }
1344   if (!asValue)
1345     // Get RValue from temp memory as atomic for non-simple lvalues
1346     return RValue::get(CGF.Builder.CreateLoad(addr));
1347   if (LVal.isBitField())
1348     return CGF.EmitLoadOfBitfieldLValue(
1349         LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1350                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1351   if (LVal.isVectorElt())
1352     return CGF.EmitLoadOfLValue(
1353         LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1354                               LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1355   assert(LVal.isExtVectorElt());
1356   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1357       addr, LVal.getExtVectorElts(), LVal.getType(),
1358       LVal.getBaseInfo(), TBAAAccessInfo()));
1359 }
1360 
1361 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1362                                              AggValueSlot ResultSlot,
1363                                              SourceLocation Loc,
1364                                              bool AsValue) const {
1365   // Try not to in some easy cases.
1366   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1367   if (getEvaluationKind() == TEK_Scalar &&
1368       (((!LVal.isBitField() ||
1369          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1370         !hasPadding()) ||
1371        !AsValue)) {
1372     auto *ValTy = AsValue
1373                       ? CGF.ConvertTypeForMem(ValueTy)
1374                       : getAtomicAddress().getType()->getPointerElementType();
1375     if (ValTy->isIntegerTy()) {
1376       assert(IntVal->getType() == ValTy && "Different integer types.");
1377       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1378     } else if (ValTy->isPointerTy())
1379       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1380     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1381       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1382   }
1383 
1384   // Create a temporary.  This needs to be big enough to hold the
1385   // atomic integer.
1386   Address Temp = Address::invalid();
1387   bool TempIsVolatile = false;
1388   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1389     assert(!ResultSlot.isIgnored());
1390     Temp = ResultSlot.getAddress();
1391     TempIsVolatile = ResultSlot.isVolatile();
1392   } else {
1393     Temp = CreateTempAlloca();
1394   }
1395 
1396   // Slam the integer into the temporary.
1397   Address CastTemp = emitCastToAtomicIntPointer(Temp);
1398   CGF.Builder.CreateStore(IntVal, CastTemp)
1399       ->setVolatile(TempIsVolatile);
1400 
1401   return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1402 }
1403 
1404 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1405                                        llvm::AtomicOrdering AO, bool) {
1406   // void __atomic_load(size_t size, void *mem, void *return, int order);
1407   CallArgList Args;
1408   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1409   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1410            CGF.getContext().VoidPtrTy);
1411   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1412            CGF.getContext().VoidPtrTy);
1413   Args.add(
1414       RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1415       CGF.getContext().IntTy);
1416   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1417 }
1418 
1419 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1420                                           bool IsVolatile) {
1421   // Okay, we're doing this natively.
1422   Address Addr = getAtomicAddressAsAtomicIntPointer();
1423   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1424   Load->setAtomic(AO);
1425 
1426   // Other decoration.
1427   if (IsVolatile)
1428     Load->setVolatile(true);
1429   CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1430   return Load;
1431 }
1432 
1433 /// An LValue is a candidate for having its loads and stores be made atomic if
1434 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1435 /// performing such an operation can be performed without a libcall.
1436 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1437   if (!CGM.getCodeGenOpts().MSVolatile) return false;
1438   AtomicInfo AI(*this, LV);
1439   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1440   // An atomic is inline if we don't need to use a libcall.
1441   bool AtomicIsInline = !AI.shouldUseLibcall();
1442   // MSVC doesn't seem to do this for types wider than a pointer.
1443   if (getContext().getTypeSize(LV.getType()) >
1444       getContext().getTypeSize(getContext().getIntPtrType()))
1445     return false;
1446   return IsVolatile && AtomicIsInline;
1447 }
1448 
1449 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1450                                        AggValueSlot Slot) {
1451   llvm::AtomicOrdering AO;
1452   bool IsVolatile = LV.isVolatileQualified();
1453   if (LV.getType()->isAtomicType()) {
1454     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1455   } else {
1456     AO = llvm::AtomicOrdering::Acquire;
1457     IsVolatile = true;
1458   }
1459   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1460 }
1461 
1462 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1463                                   bool AsValue, llvm::AtomicOrdering AO,
1464                                   bool IsVolatile) {
1465   // Check whether we should use a library call.
1466   if (shouldUseLibcall()) {
1467     Address TempAddr = Address::invalid();
1468     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1469       assert(getEvaluationKind() == TEK_Aggregate);
1470       TempAddr = ResultSlot.getAddress();
1471     } else
1472       TempAddr = CreateTempAlloca();
1473 
1474     EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1475 
1476     // Okay, turn that back into the original value or whole atomic (for
1477     // non-simple lvalues) type.
1478     return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1479   }
1480 
1481   // Okay, we're doing this natively.
1482   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1483 
1484   // If we're ignoring an aggregate return, don't do anything.
1485   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1486     return RValue::getAggregate(Address::invalid(), false);
1487 
1488   // Okay, turn that back into the original value or atomic (for non-simple
1489   // lvalues) type.
1490   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1491 }
1492 
1493 /// Emit a load from an l-value of atomic type.  Note that the r-value
1494 /// we produce is an r-value of the atomic *value* type.
1495 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1496                                        llvm::AtomicOrdering AO, bool IsVolatile,
1497                                        AggValueSlot resultSlot) {
1498   AtomicInfo Atomics(*this, src);
1499   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1500                                 IsVolatile);
1501 }
1502 
1503 /// Copy an r-value into memory as part of storing to an atomic type.
1504 /// This needs to create a bit-pattern suitable for atomic operations.
1505 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1506   assert(LVal.isSimple());
1507   // If we have an r-value, the rvalue should be of the atomic type,
1508   // which means that the caller is responsible for having zeroed
1509   // any padding.  Just do an aggregate copy of that type.
1510   if (rvalue.isAggregate()) {
1511     CGF.EmitAggregateCopy(getAtomicAddress(),
1512                           rvalue.getAggregateAddress(),
1513                           getAtomicType(),
1514                           (rvalue.isVolatileQualified()
1515                            || LVal.isVolatileQualified()));
1516     return;
1517   }
1518 
1519   // Okay, otherwise we're copying stuff.
1520 
1521   // Zero out the buffer if necessary.
1522   emitMemSetZeroIfNecessary();
1523 
1524   // Drill past the padding if present.
1525   LValue TempLVal = projectValue();
1526 
1527   // Okay, store the rvalue in.
1528   if (rvalue.isScalar()) {
1529     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1530   } else {
1531     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1532   }
1533 }
1534 
1535 
1536 /// Materialize an r-value into memory for the purposes of storing it
1537 /// to an atomic type.
1538 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1539   // Aggregate r-values are already in memory, and EmitAtomicStore
1540   // requires them to be values of the atomic type.
1541   if (rvalue.isAggregate())
1542     return rvalue.getAggregateAddress();
1543 
1544   // Otherwise, make a temporary and materialize into it.
1545   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1546   AtomicInfo Atomics(CGF, TempLV);
1547   Atomics.emitCopyIntoMemory(rvalue);
1548   return TempLV.getAddress();
1549 }
1550 
1551 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1552   // If we've got a scalar value of the right size, try to avoid going
1553   // through memory.
1554   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1555     llvm::Value *Value = RVal.getScalarVal();
1556     if (isa<llvm::IntegerType>(Value->getType()))
1557       return CGF.EmitToMemory(Value, ValueTy);
1558     else {
1559       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1560           CGF.getLLVMContext(),
1561           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1562       if (isa<llvm::PointerType>(Value->getType()))
1563         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1564       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1565         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1566     }
1567   }
1568   // Otherwise, we need to go through memory.
1569   // Put the r-value in memory.
1570   Address Addr = materializeRValue(RVal);
1571 
1572   // Cast the temporary to the atomic int type and pull a value out.
1573   Addr = emitCastToAtomicIntPointer(Addr);
1574   return CGF.Builder.CreateLoad(Addr);
1575 }
1576 
1577 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1578     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1579     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1580   // Do the atomic store.
1581   Address Addr = getAtomicAddressAsAtomicIntPointer();
1582   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1583                                                ExpectedVal, DesiredVal,
1584                                                Success, Failure);
1585   // Other decoration.
1586   Inst->setVolatile(LVal.isVolatileQualified());
1587   Inst->setWeak(IsWeak);
1588 
1589   // Okay, turn that back into the original value type.
1590   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1591   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1592   return std::make_pair(PreviousVal, SuccessFailureVal);
1593 }
1594 
1595 llvm::Value *
1596 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1597                                              llvm::Value *DesiredAddr,
1598                                              llvm::AtomicOrdering Success,
1599                                              llvm::AtomicOrdering Failure) {
1600   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1601   // void *desired, int success, int failure);
1602   CallArgList Args;
1603   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1604   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1605            CGF.getContext().VoidPtrTy);
1606   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1607            CGF.getContext().VoidPtrTy);
1608   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1609            CGF.getContext().VoidPtrTy);
1610   Args.add(RValue::get(
1611                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1612            CGF.getContext().IntTy);
1613   Args.add(RValue::get(
1614                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1615            CGF.getContext().IntTy);
1616   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1617                                               CGF.getContext().BoolTy, Args);
1618 
1619   return SuccessFailureRVal.getScalarVal();
1620 }
1621 
1622 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1623     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1624     llvm::AtomicOrdering Failure, bool IsWeak) {
1625   if (isStrongerThan(Failure, Success))
1626     // Don't assert on undefined behavior "failure argument shall be no stronger
1627     // than the success argument".
1628     Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1629 
1630   // Check whether we should use a library call.
1631   if (shouldUseLibcall()) {
1632     // Produce a source address.
1633     Address ExpectedAddr = materializeRValue(Expected);
1634     Address DesiredAddr = materializeRValue(Desired);
1635     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1636                                                  DesiredAddr.getPointer(),
1637                                                  Success, Failure);
1638     return std::make_pair(
1639         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1640                                   SourceLocation(), /*AsValue=*/false),
1641         Res);
1642   }
1643 
1644   // If we've got a scalar value of the right size, try to avoid going
1645   // through memory.
1646   auto *ExpectedVal = convertRValueToInt(Expected);
1647   auto *DesiredVal = convertRValueToInt(Desired);
1648   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1649                                          Failure, IsWeak);
1650   return std::make_pair(
1651       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1652                                 SourceLocation(), /*AsValue=*/false),
1653       Res.second);
1654 }
1655 
1656 static void
1657 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1658                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1659                       Address DesiredAddr) {
1660   RValue UpRVal;
1661   LValue AtomicLVal = Atomics.getAtomicLValue();
1662   LValue DesiredLVal;
1663   if (AtomicLVal.isSimple()) {
1664     UpRVal = OldRVal;
1665     DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1666   } else {
1667     // Build new lvalue for temp address
1668     Address Ptr = Atomics.materializeRValue(OldRVal);
1669     LValue UpdateLVal;
1670     if (AtomicLVal.isBitField()) {
1671       UpdateLVal =
1672           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1673                                AtomicLVal.getType(),
1674                                AtomicLVal.getBaseInfo(),
1675                                AtomicLVal.getTBAAInfo());
1676       DesiredLVal =
1677           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1678                                AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1679                                AtomicLVal.getTBAAInfo());
1680     } else if (AtomicLVal.isVectorElt()) {
1681       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1682                                          AtomicLVal.getType(),
1683                                          AtomicLVal.getBaseInfo(),
1684                                          AtomicLVal.getTBAAInfo());
1685       DesiredLVal = LValue::MakeVectorElt(
1686           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1687           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1688     } else {
1689       assert(AtomicLVal.isExtVectorElt());
1690       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1691                                             AtomicLVal.getType(),
1692                                             AtomicLVal.getBaseInfo(),
1693                                             AtomicLVal.getTBAAInfo());
1694       DesiredLVal = LValue::MakeExtVectorElt(
1695           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1696           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1697     }
1698     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1699   }
1700   // Store new value in the corresponding memory area
1701   RValue NewRVal = UpdateOp(UpRVal);
1702   if (NewRVal.isScalar()) {
1703     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1704   } else {
1705     assert(NewRVal.isComplex());
1706     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1707                            /*isInit=*/false);
1708   }
1709 }
1710 
1711 void AtomicInfo::EmitAtomicUpdateLibcall(
1712     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1713     bool IsVolatile) {
1714   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1715 
1716   Address ExpectedAddr = CreateTempAlloca();
1717 
1718   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1719   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1720   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1721   CGF.EmitBlock(ContBB);
1722   Address DesiredAddr = CreateTempAlloca();
1723   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1724       requiresMemSetZero(getAtomicAddress().getElementType())) {
1725     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1726     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1727   }
1728   auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1729                                            AggValueSlot::ignored(),
1730                                            SourceLocation(), /*AsValue=*/false);
1731   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1732   auto *Res =
1733       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1734                                        DesiredAddr.getPointer(),
1735                                        AO, Failure);
1736   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1737   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1738 }
1739 
1740 void AtomicInfo::EmitAtomicUpdateOp(
1741     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1742     bool IsVolatile) {
1743   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1744 
1745   // Do the atomic load.
1746   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1747   // For non-simple lvalues perform compare-and-swap procedure.
1748   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1749   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1750   auto *CurBB = CGF.Builder.GetInsertBlock();
1751   CGF.EmitBlock(ContBB);
1752   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1753                                              /*NumReservedValues=*/2);
1754   PHI->addIncoming(OldVal, CurBB);
1755   Address NewAtomicAddr = CreateTempAlloca();
1756   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1757   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1758       requiresMemSetZero(getAtomicAddress().getElementType())) {
1759     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1760   }
1761   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1762                                            SourceLocation(), /*AsValue=*/false);
1763   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1764   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1765   // Try to write new value using cmpxchg operation
1766   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1767   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1768   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1769   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1770 }
1771 
1772 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1773                                   RValue UpdateRVal, Address DesiredAddr) {
1774   LValue AtomicLVal = Atomics.getAtomicLValue();
1775   LValue DesiredLVal;
1776   // Build new lvalue for temp address
1777   if (AtomicLVal.isBitField()) {
1778     DesiredLVal =
1779         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1780                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1781                              AtomicLVal.getTBAAInfo());
1782   } else if (AtomicLVal.isVectorElt()) {
1783     DesiredLVal =
1784         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1785                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1786                               AtomicLVal.getTBAAInfo());
1787   } else {
1788     assert(AtomicLVal.isExtVectorElt());
1789     DesiredLVal = LValue::MakeExtVectorElt(
1790         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1791         AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1792   }
1793   // Store new value in the corresponding memory area
1794   assert(UpdateRVal.isScalar());
1795   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1796 }
1797 
1798 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1799                                          RValue UpdateRVal, bool IsVolatile) {
1800   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1801 
1802   Address ExpectedAddr = CreateTempAlloca();
1803 
1804   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1805   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1806   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1807   CGF.EmitBlock(ContBB);
1808   Address DesiredAddr = CreateTempAlloca();
1809   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1810       requiresMemSetZero(getAtomicAddress().getElementType())) {
1811     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1812     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1813   }
1814   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1815   auto *Res =
1816       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1817                                        DesiredAddr.getPointer(),
1818                                        AO, Failure);
1819   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1820   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1821 }
1822 
1823 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1824                                     bool IsVolatile) {
1825   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1826 
1827   // Do the atomic load.
1828   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1829   // For non-simple lvalues perform compare-and-swap procedure.
1830   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1831   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1832   auto *CurBB = CGF.Builder.GetInsertBlock();
1833   CGF.EmitBlock(ContBB);
1834   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1835                                              /*NumReservedValues=*/2);
1836   PHI->addIncoming(OldVal, CurBB);
1837   Address NewAtomicAddr = CreateTempAlloca();
1838   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1839   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1840       requiresMemSetZero(getAtomicAddress().getElementType())) {
1841     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1842   }
1843   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1844   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1845   // Try to write new value using cmpxchg operation
1846   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1847   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1848   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1849   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1850 }
1851 
1852 void AtomicInfo::EmitAtomicUpdate(
1853     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1854     bool IsVolatile) {
1855   if (shouldUseLibcall()) {
1856     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1857   } else {
1858     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1859   }
1860 }
1861 
1862 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1863                                   bool IsVolatile) {
1864   if (shouldUseLibcall()) {
1865     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1866   } else {
1867     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1868   }
1869 }
1870 
1871 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1872                                       bool isInit) {
1873   bool IsVolatile = lvalue.isVolatileQualified();
1874   llvm::AtomicOrdering AO;
1875   if (lvalue.getType()->isAtomicType()) {
1876     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1877   } else {
1878     AO = llvm::AtomicOrdering::Release;
1879     IsVolatile = true;
1880   }
1881   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1882 }
1883 
1884 /// Emit a store to an l-value of atomic type.
1885 ///
1886 /// Note that the r-value is expected to be an r-value *of the atomic
1887 /// type*; this means that for aggregate r-values, it should include
1888 /// storage for any padding that was necessary.
1889 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1890                                       llvm::AtomicOrdering AO, bool IsVolatile,
1891                                       bool isInit) {
1892   // If this is an aggregate r-value, it should agree in type except
1893   // maybe for address-space qualification.
1894   assert(!rvalue.isAggregate() ||
1895          rvalue.getAggregateAddress().getElementType()
1896            == dest.getAddress().getElementType());
1897 
1898   AtomicInfo atomics(*this, dest);
1899   LValue LVal = atomics.getAtomicLValue();
1900 
1901   // If this is an initialization, just put the value there normally.
1902   if (LVal.isSimple()) {
1903     if (isInit) {
1904       atomics.emitCopyIntoMemory(rvalue);
1905       return;
1906     }
1907 
1908     // Check whether we should use a library call.
1909     if (atomics.shouldUseLibcall()) {
1910       // Produce a source address.
1911       Address srcAddr = atomics.materializeRValue(rvalue);
1912 
1913       // void __atomic_store(size_t size, void *mem, void *val, int order)
1914       CallArgList args;
1915       args.add(RValue::get(atomics.getAtomicSizeValue()),
1916                getContext().getSizeType());
1917       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1918                getContext().VoidPtrTy);
1919       args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1920                getContext().VoidPtrTy);
1921       args.add(
1922           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1923           getContext().IntTy);
1924       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1925       return;
1926     }
1927 
1928     // Okay, we're doing this natively.
1929     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1930 
1931     // Do the atomic store.
1932     Address addr =
1933         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1934     intValue = Builder.CreateIntCast(
1935         intValue, addr.getElementType(), /*isSigned=*/false);
1936     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1937 
1938     // Initializations don't need to be atomic.
1939     if (!isInit)
1940       store->setAtomic(AO);
1941 
1942     // Other decoration.
1943     if (IsVolatile)
1944       store->setVolatile(true);
1945     CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1946     return;
1947   }
1948 
1949   // Emit simple atomic update operation.
1950   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1951 }
1952 
1953 /// Emit a compare-and-exchange op for atomic type.
1954 ///
1955 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1956     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1957     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1958     AggValueSlot Slot) {
1959   // If this is an aggregate r-value, it should agree in type except
1960   // maybe for address-space qualification.
1961   assert(!Expected.isAggregate() ||
1962          Expected.getAggregateAddress().getElementType() ==
1963              Obj.getAddress().getElementType());
1964   assert(!Desired.isAggregate() ||
1965          Desired.getAggregateAddress().getElementType() ==
1966              Obj.getAddress().getElementType());
1967   AtomicInfo Atomics(*this, Obj);
1968 
1969   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1970                                            IsWeak);
1971 }
1972 
1973 void CodeGenFunction::EmitAtomicUpdate(
1974     LValue LVal, llvm::AtomicOrdering AO,
1975     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1976   AtomicInfo Atomics(*this, LVal);
1977   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1978 }
1979 
1980 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1981   AtomicInfo atomics(*this, dest);
1982 
1983   switch (atomics.getEvaluationKind()) {
1984   case TEK_Scalar: {
1985     llvm::Value *value = EmitScalarExpr(init);
1986     atomics.emitCopyIntoMemory(RValue::get(value));
1987     return;
1988   }
1989 
1990   case TEK_Complex: {
1991     ComplexPairTy value = EmitComplexExpr(init);
1992     atomics.emitCopyIntoMemory(RValue::getComplex(value));
1993     return;
1994   }
1995 
1996   case TEK_Aggregate: {
1997     // Fix up the destination if the initializer isn't an expression
1998     // of atomic type.
1999     bool Zeroed = false;
2000     if (!init->getType()->isAtomicType()) {
2001       Zeroed = atomics.emitMemSetZeroIfNecessary();
2002       dest = atomics.projectValue();
2003     }
2004 
2005     // Evaluate the expression directly into the destination.
2006     AggValueSlot slot = AggValueSlot::forLValue(dest,
2007                                         AggValueSlot::IsNotDestructed,
2008                                         AggValueSlot::DoesNotNeedGCBarriers,
2009                                         AggValueSlot::IsNotAliased,
2010                                         Zeroed ? AggValueSlot::IsZeroed :
2011                                                  AggValueSlot::IsNotZeroed);
2012 
2013     EmitAggExpr(init, slot);
2014     return;
2015   }
2016   }
2017   llvm_unreachable("bad evaluation kind");
2018 }
2019