1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the code for emitting atomic operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CodeGenFunction.h"
15 #include "CGCall.h"
16 #include "CodeGenModule.h"
17 #include "clang/AST/ASTContext.h"
18 #include "llvm/IR/DataLayout.h"
19 #include "llvm/IR/Intrinsics.h"
20 #include "llvm/IR/Operator.h"
21 
22 using namespace clang;
23 using namespace CodeGen;
24 
25 // The ABI values for various atomic memory orderings.
26 enum AtomicOrderingKind {
27   AO_ABI_memory_order_relaxed = 0,
28   AO_ABI_memory_order_consume = 1,
29   AO_ABI_memory_order_acquire = 2,
30   AO_ABI_memory_order_release = 3,
31   AO_ABI_memory_order_acq_rel = 4,
32   AO_ABI_memory_order_seq_cst = 5
33 };
34 
35 namespace {
36   class AtomicInfo {
37     CodeGenFunction &CGF;
38     QualType AtomicTy;
39     QualType ValueTy;
40     uint64_t AtomicSizeInBits;
41     uint64_t ValueSizeInBits;
42     CharUnits AtomicAlign;
43     CharUnits ValueAlign;
44     CharUnits LValueAlign;
45     TypeEvaluationKind EvaluationKind;
46     bool UseLibcall;
47   public:
48     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) {
49       assert(lvalue.isSimple());
50 
51       AtomicTy = lvalue.getType();
52       ValueTy = AtomicTy->castAs<AtomicType>()->getValueType();
53       EvaluationKind = CGF.getEvaluationKind(ValueTy);
54 
55       ASTContext &C = CGF.getContext();
56 
57       uint64_t valueAlignInBits;
58       llvm::tie(ValueSizeInBits, valueAlignInBits) = C.getTypeInfo(ValueTy);
59 
60       uint64_t atomicAlignInBits;
61       llvm::tie(AtomicSizeInBits, atomicAlignInBits) = C.getTypeInfo(AtomicTy);
62 
63       assert(ValueSizeInBits <= AtomicSizeInBits);
64       assert(valueAlignInBits <= atomicAlignInBits);
65 
66       AtomicAlign = C.toCharUnitsFromBits(atomicAlignInBits);
67       ValueAlign = C.toCharUnitsFromBits(valueAlignInBits);
68       if (lvalue.getAlignment().isZero())
69         lvalue.setAlignment(AtomicAlign);
70 
71       UseLibcall =
72         (AtomicSizeInBits > uint64_t(C.toBits(lvalue.getAlignment())) ||
73          AtomicSizeInBits > C.getTargetInfo().getMaxAtomicInlineWidth());
74     }
75 
76     QualType getAtomicType() const { return AtomicTy; }
77     QualType getValueType() const { return ValueTy; }
78     CharUnits getAtomicAlignment() const { return AtomicAlign; }
79     CharUnits getValueAlignment() const { return ValueAlign; }
80     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
81     uint64_t getValueSizeInBits() const { return AtomicSizeInBits; }
82     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
83     bool shouldUseLibcall() const { return UseLibcall; }
84 
85     /// Is the atomic size larger than the underlying value type?
86     ///
87     /// Note that the absence of padding does not mean that atomic
88     /// objects are completely interchangeable with non-atomic
89     /// objects: we might have promoted the alignment of a type
90     /// without making it bigger.
91     bool hasPadding() const {
92       return (ValueSizeInBits != AtomicSizeInBits);
93     }
94 
95     void emitMemSetZeroIfNecessary(LValue dest) const;
96 
97     llvm::Value *getAtomicSizeValue() const {
98       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
99       return CGF.CGM.getSize(size);
100     }
101 
102     /// Cast the given pointer to an integer pointer suitable for
103     /// atomic operations.
104     llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
105 
106     /// Turn an atomic-layout object into an r-value.
107     RValue convertTempToRValue(llvm::Value *addr,
108                                AggValueSlot resultSlot) const;
109 
110     /// Copy an atomic r-value into atomic-layout memory.
111     void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const;
112 
113     /// Project an l-value down to the value field.
114     LValue projectValue(LValue lvalue) const {
115       llvm::Value *addr = lvalue.getAddress();
116       if (hasPadding())
117         addr = CGF.Builder.CreateStructGEP(addr, 0);
118 
119       return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(),
120                               CGF.getContext(), lvalue.getTBAAInfo());
121     }
122 
123     /// Materialize an atomic r-value in atomic-layout memory.
124     llvm::Value *materializeRValue(RValue rvalue) const;
125 
126   private:
127     bool requiresMemSetZero(llvm::Type *type) const;
128   };
129 }
130 
131 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
132                                 StringRef fnName,
133                                 QualType resultType,
134                                 CallArgList &args) {
135   const CGFunctionInfo &fnInfo =
136     CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
137             FunctionType::ExtInfo(), RequiredArgs::All);
138   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
139   llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
140   return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
141 }
142 
143 /// Does a store of the given IR type modify the full expected width?
144 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
145                            uint64_t expectedSize) {
146   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
147 }
148 
149 /// Does the atomic type require memsetting to zero before initialization?
150 ///
151 /// The IR type is provided as a way of making certain queries faster.
152 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
153   // If the atomic type has size padding, we definitely need a memset.
154   if (hasPadding()) return true;
155 
156   // Otherwise, do some simple heuristics to try to avoid it:
157   switch (getEvaluationKind()) {
158   // For scalars and complexes, check whether the store size of the
159   // type uses the full size.
160   case TEK_Scalar:
161     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
162   case TEK_Complex:
163     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
164                            AtomicSizeInBits / 2);
165 
166   // Just be pessimistic about aggregates.
167   case TEK_Aggregate:
168     return true;
169   }
170   llvm_unreachable("bad evaluation kind");
171 }
172 
173 void AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const {
174   llvm::Value *addr = dest.getAddress();
175   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
176     return;
177 
178   CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
179                            AtomicSizeInBits / 8,
180                            dest.getAlignment().getQuantity());
181 }
182 
183 static void
184 EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
185              llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
186              uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) {
187   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
188   llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
189 
190   switch (E->getOp()) {
191   case AtomicExpr::AO__c11_atomic_init:
192     llvm_unreachable("Already handled!");
193 
194   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
195   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
196   case AtomicExpr::AO__atomic_compare_exchange:
197   case AtomicExpr::AO__atomic_compare_exchange_n: {
198     // Note that cmpxchg only supports specifying one ordering and
199     // doesn't support weak cmpxchg, at least at the moment.
200     llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
201     LoadVal1->setAlignment(Align);
202     llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2);
203     LoadVal2->setAlignment(Align);
204     llvm::AtomicCmpXchgInst *CXI =
205         CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order);
206     CXI->setVolatile(E->isVolatile());
207     llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1);
208     StoreVal1->setAlignment(Align);
209     llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1);
210     CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
211     return;
212   }
213 
214   case AtomicExpr::AO__c11_atomic_load:
215   case AtomicExpr::AO__atomic_load_n:
216   case AtomicExpr::AO__atomic_load: {
217     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
218     Load->setAtomic(Order);
219     Load->setAlignment(Size);
220     Load->setVolatile(E->isVolatile());
221     llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
222     StoreDest->setAlignment(Align);
223     return;
224   }
225 
226   case AtomicExpr::AO__c11_atomic_store:
227   case AtomicExpr::AO__atomic_store:
228   case AtomicExpr::AO__atomic_store_n: {
229     assert(!Dest && "Store does not return a value");
230     llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
231     LoadVal1->setAlignment(Align);
232     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
233     Store->setAtomic(Order);
234     Store->setAlignment(Size);
235     Store->setVolatile(E->isVolatile());
236     return;
237   }
238 
239   case AtomicExpr::AO__c11_atomic_exchange:
240   case AtomicExpr::AO__atomic_exchange_n:
241   case AtomicExpr::AO__atomic_exchange:
242     Op = llvm::AtomicRMWInst::Xchg;
243     break;
244 
245   case AtomicExpr::AO__atomic_add_fetch:
246     PostOp = llvm::Instruction::Add;
247     // Fall through.
248   case AtomicExpr::AO__c11_atomic_fetch_add:
249   case AtomicExpr::AO__atomic_fetch_add:
250     Op = llvm::AtomicRMWInst::Add;
251     break;
252 
253   case AtomicExpr::AO__atomic_sub_fetch:
254     PostOp = llvm::Instruction::Sub;
255     // Fall through.
256   case AtomicExpr::AO__c11_atomic_fetch_sub:
257   case AtomicExpr::AO__atomic_fetch_sub:
258     Op = llvm::AtomicRMWInst::Sub;
259     break;
260 
261   case AtomicExpr::AO__atomic_and_fetch:
262     PostOp = llvm::Instruction::And;
263     // Fall through.
264   case AtomicExpr::AO__c11_atomic_fetch_and:
265   case AtomicExpr::AO__atomic_fetch_and:
266     Op = llvm::AtomicRMWInst::And;
267     break;
268 
269   case AtomicExpr::AO__atomic_or_fetch:
270     PostOp = llvm::Instruction::Or;
271     // Fall through.
272   case AtomicExpr::AO__c11_atomic_fetch_or:
273   case AtomicExpr::AO__atomic_fetch_or:
274     Op = llvm::AtomicRMWInst::Or;
275     break;
276 
277   case AtomicExpr::AO__atomic_xor_fetch:
278     PostOp = llvm::Instruction::Xor;
279     // Fall through.
280   case AtomicExpr::AO__c11_atomic_fetch_xor:
281   case AtomicExpr::AO__atomic_fetch_xor:
282     Op = llvm::AtomicRMWInst::Xor;
283     break;
284 
285   case AtomicExpr::AO__atomic_nand_fetch:
286     PostOp = llvm::Instruction::And;
287     // Fall through.
288   case AtomicExpr::AO__atomic_fetch_nand:
289     Op = llvm::AtomicRMWInst::Nand;
290     break;
291   }
292 
293   llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
294   LoadVal1->setAlignment(Align);
295   llvm::AtomicRMWInst *RMWI =
296       CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
297   RMWI->setVolatile(E->isVolatile());
298 
299   // For __atomic_*_fetch operations, perform the operation again to
300   // determine the value which was written.
301   llvm::Value *Result = RMWI;
302   if (PostOp)
303     Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
304   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
305     Result = CGF.Builder.CreateNot(Result);
306   llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
307   StoreDest->setAlignment(Align);
308 }
309 
310 // This function emits any expression (scalar, complex, or aggregate)
311 // into a temporary alloca.
312 static llvm::Value *
313 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
314   llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
315   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
316                        /*Init*/ true);
317   return DeclPtr;
318 }
319 
320 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
321   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
322   QualType MemTy = AtomicTy;
323   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
324     MemTy = AT->getValueType();
325   CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
326   uint64_t Size = sizeChars.getQuantity();
327   CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
328   unsigned Align = alignChars.getQuantity();
329   unsigned MaxInlineWidthInBits =
330     getTarget().getMaxAtomicInlineWidth();
331   bool UseLibcall = (Size != Align ||
332                      getContext().toBits(sizeChars) > MaxInlineWidthInBits);
333 
334   llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
335   Ptr = EmitScalarExpr(E->getPtr());
336 
337   if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
338     assert(!Dest && "Init does not return a value");
339     LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
340     EmitAtomicInit(E->getVal1(), lvalue);
341     return RValue::get(0);
342   }
343 
344   Order = EmitScalarExpr(E->getOrder());
345 
346   switch (E->getOp()) {
347   case AtomicExpr::AO__c11_atomic_init:
348     llvm_unreachable("Already handled!");
349 
350   case AtomicExpr::AO__c11_atomic_load:
351   case AtomicExpr::AO__atomic_load_n:
352     break;
353 
354   case AtomicExpr::AO__atomic_load:
355     Dest = EmitScalarExpr(E->getVal1());
356     break;
357 
358   case AtomicExpr::AO__atomic_store:
359     Val1 = EmitScalarExpr(E->getVal1());
360     break;
361 
362   case AtomicExpr::AO__atomic_exchange:
363     Val1 = EmitScalarExpr(E->getVal1());
364     Dest = EmitScalarExpr(E->getVal2());
365     break;
366 
367   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
368   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
369   case AtomicExpr::AO__atomic_compare_exchange_n:
370   case AtomicExpr::AO__atomic_compare_exchange:
371     Val1 = EmitScalarExpr(E->getVal1());
372     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
373       Val2 = EmitScalarExpr(E->getVal2());
374     else
375       Val2 = EmitValToTemp(*this, E->getVal2());
376     OrderFail = EmitScalarExpr(E->getOrderFail());
377     // Evaluate and discard the 'weak' argument.
378     if (E->getNumSubExprs() == 6)
379       EmitScalarExpr(E->getWeak());
380     break;
381 
382   case AtomicExpr::AO__c11_atomic_fetch_add:
383   case AtomicExpr::AO__c11_atomic_fetch_sub:
384     if (MemTy->isPointerType()) {
385       // For pointer arithmetic, we're required to do a bit of math:
386       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
387       // ... but only for the C11 builtins. The GNU builtins expect the
388       // user to multiply by sizeof(T).
389       QualType Val1Ty = E->getVal1()->getType();
390       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
391       CharUnits PointeeIncAmt =
392           getContext().getTypeSizeInChars(MemTy->getPointeeType());
393       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
394       Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
395       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
396       break;
397     }
398     // Fall through.
399   case AtomicExpr::AO__atomic_fetch_add:
400   case AtomicExpr::AO__atomic_fetch_sub:
401   case AtomicExpr::AO__atomic_add_fetch:
402   case AtomicExpr::AO__atomic_sub_fetch:
403   case AtomicExpr::AO__c11_atomic_store:
404   case AtomicExpr::AO__c11_atomic_exchange:
405   case AtomicExpr::AO__atomic_store_n:
406   case AtomicExpr::AO__atomic_exchange_n:
407   case AtomicExpr::AO__c11_atomic_fetch_and:
408   case AtomicExpr::AO__c11_atomic_fetch_or:
409   case AtomicExpr::AO__c11_atomic_fetch_xor:
410   case AtomicExpr::AO__atomic_fetch_and:
411   case AtomicExpr::AO__atomic_fetch_or:
412   case AtomicExpr::AO__atomic_fetch_xor:
413   case AtomicExpr::AO__atomic_fetch_nand:
414   case AtomicExpr::AO__atomic_and_fetch:
415   case AtomicExpr::AO__atomic_or_fetch:
416   case AtomicExpr::AO__atomic_xor_fetch:
417   case AtomicExpr::AO__atomic_nand_fetch:
418     Val1 = EmitValToTemp(*this, E->getVal1());
419     break;
420   }
421 
422   if (!E->getType()->isVoidType() && !Dest)
423     Dest = CreateMemTemp(E->getType(), ".atomicdst");
424 
425   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
426   if (UseLibcall) {
427 
428     SmallVector<QualType, 5> Params;
429     CallArgList Args;
430     // Size is always the first parameter
431     Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
432              getContext().getSizeType());
433     // Atomic address is always the second parameter
434     Args.add(RValue::get(EmitCastToVoidPtr(Ptr)),
435              getContext().VoidPtrTy);
436 
437     const char* LibCallName;
438     QualType RetTy = getContext().VoidTy;
439     switch (E->getOp()) {
440     // There is only one libcall for compare an exchange, because there is no
441     // optimisation benefit possible from a libcall version of a weak compare
442     // and exchange.
443     // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
444     //                                void *desired, int success, int failure)
445     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
446     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
447     case AtomicExpr::AO__atomic_compare_exchange:
448     case AtomicExpr::AO__atomic_compare_exchange_n:
449       LibCallName = "__atomic_compare_exchange";
450       RetTy = getContext().BoolTy;
451       Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
452                getContext().VoidPtrTy);
453       Args.add(RValue::get(EmitCastToVoidPtr(Val2)),
454                getContext().VoidPtrTy);
455       Args.add(RValue::get(Order),
456                getContext().IntTy);
457       Order = OrderFail;
458       break;
459     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
460     //                        int order)
461     case AtomicExpr::AO__c11_atomic_exchange:
462     case AtomicExpr::AO__atomic_exchange_n:
463     case AtomicExpr::AO__atomic_exchange:
464       LibCallName = "__atomic_exchange";
465       Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
466                getContext().VoidPtrTy);
467       Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
468                getContext().VoidPtrTy);
469       break;
470     // void __atomic_store(size_t size, void *mem, void *val, int order)
471     case AtomicExpr::AO__c11_atomic_store:
472     case AtomicExpr::AO__atomic_store:
473     case AtomicExpr::AO__atomic_store_n:
474       LibCallName = "__atomic_store";
475       Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
476                getContext().VoidPtrTy);
477       break;
478     // void __atomic_load(size_t size, void *mem, void *return, int order)
479     case AtomicExpr::AO__c11_atomic_load:
480     case AtomicExpr::AO__atomic_load:
481     case AtomicExpr::AO__atomic_load_n:
482       LibCallName = "__atomic_load";
483       Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
484                getContext().VoidPtrTy);
485       break;
486     default: return EmitUnsupportedRValue(E, "atomic library call");
487     }
488     // order is always the last parameter
489     Args.add(RValue::get(Order),
490              getContext().IntTy);
491 
492     const CGFunctionInfo &FuncInfo =
493         CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args,
494             FunctionType::ExtInfo(), RequiredArgs::All);
495     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
496     llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
497     RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
498     if (E->isCmpXChg())
499       return Res;
500     if (E->getType()->isVoidType())
501       return RValue::get(0);
502     return convertTempToRValue(Dest, E->getType());
503   }
504 
505   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
506                  E->getOp() == AtomicExpr::AO__atomic_store ||
507                  E->getOp() == AtomicExpr::AO__atomic_store_n;
508   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
509                 E->getOp() == AtomicExpr::AO__atomic_load ||
510                 E->getOp() == AtomicExpr::AO__atomic_load_n;
511 
512   llvm::Type *IPtrTy =
513       llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
514   llvm::Value *OrigDest = Dest;
515   Ptr = Builder.CreateBitCast(Ptr, IPtrTy);
516   if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy);
517   if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy);
518   if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
519 
520   if (isa<llvm::ConstantInt>(Order)) {
521     int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
522     switch (ord) {
523     case AO_ABI_memory_order_relaxed:
524       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
525                    llvm::Monotonic);
526       break;
527     case AO_ABI_memory_order_consume:
528     case AO_ABI_memory_order_acquire:
529       if (IsStore)
530         break; // Avoid crashing on code with undefined behavior
531       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
532                    llvm::Acquire);
533       break;
534     case AO_ABI_memory_order_release:
535       if (IsLoad)
536         break; // Avoid crashing on code with undefined behavior
537       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
538                    llvm::Release);
539       break;
540     case AO_ABI_memory_order_acq_rel:
541       if (IsLoad || IsStore)
542         break; // Avoid crashing on code with undefined behavior
543       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
544                    llvm::AcquireRelease);
545       break;
546     case AO_ABI_memory_order_seq_cst:
547       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
548                    llvm::SequentiallyConsistent);
549       break;
550     default: // invalid order
551       // We should not ever get here normally, but it's hard to
552       // enforce that in general.
553       break;
554     }
555     if (E->getType()->isVoidType())
556       return RValue::get(0);
557     return convertTempToRValue(OrigDest, E->getType());
558   }
559 
560   // Long case, when Order isn't obviously constant.
561 
562   // Create all the relevant BB's
563   llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0,
564                    *AcqRelBB = 0, *SeqCstBB = 0;
565   MonotonicBB = createBasicBlock("monotonic", CurFn);
566   if (!IsStore)
567     AcquireBB = createBasicBlock("acquire", CurFn);
568   if (!IsLoad)
569     ReleaseBB = createBasicBlock("release", CurFn);
570   if (!IsLoad && !IsStore)
571     AcqRelBB = createBasicBlock("acqrel", CurFn);
572   SeqCstBB = createBasicBlock("seqcst", CurFn);
573   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
574 
575   // Create the switch for the split
576   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
577   // doesn't matter unless someone is crazy enough to use something that
578   // doesn't fold to a constant for the ordering.
579   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
580   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
581 
582   // Emit all the different atomics
583   Builder.SetInsertPoint(MonotonicBB);
584   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
585                llvm::Monotonic);
586   Builder.CreateBr(ContBB);
587   if (!IsStore) {
588     Builder.SetInsertPoint(AcquireBB);
589     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
590                  llvm::Acquire);
591     Builder.CreateBr(ContBB);
592     SI->addCase(Builder.getInt32(1), AcquireBB);
593     SI->addCase(Builder.getInt32(2), AcquireBB);
594   }
595   if (!IsLoad) {
596     Builder.SetInsertPoint(ReleaseBB);
597     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
598                  llvm::Release);
599     Builder.CreateBr(ContBB);
600     SI->addCase(Builder.getInt32(3), ReleaseBB);
601   }
602   if (!IsLoad && !IsStore) {
603     Builder.SetInsertPoint(AcqRelBB);
604     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
605                  llvm::AcquireRelease);
606     Builder.CreateBr(ContBB);
607     SI->addCase(Builder.getInt32(4), AcqRelBB);
608   }
609   Builder.SetInsertPoint(SeqCstBB);
610   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
611                llvm::SequentiallyConsistent);
612   Builder.CreateBr(ContBB);
613   SI->addCase(Builder.getInt32(5), SeqCstBB);
614 
615   // Cleanup and return
616   Builder.SetInsertPoint(ContBB);
617   if (E->getType()->isVoidType())
618     return RValue::get(0);
619   return convertTempToRValue(OrigDest, E->getType());
620 }
621 
622 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
623   unsigned addrspace =
624     cast<llvm::PointerType>(addr->getType())->getAddressSpace();
625   llvm::IntegerType *ty =
626     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
627   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
628 }
629 
630 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
631                                        AggValueSlot resultSlot) const {
632   if (EvaluationKind == TEK_Aggregate) {
633     // Nothing to do if the result is ignored.
634     if (resultSlot.isIgnored()) return resultSlot.asRValue();
635 
636     assert(resultSlot.getAddr() == addr || hasPadding());
637 
638     // In these cases, we should have emitted directly into the result slot.
639     if (!hasPadding() || resultSlot.isValueOfAtomic())
640       return resultSlot.asRValue();
641 
642     // Otherwise, fall into the common path.
643   }
644 
645   // Drill into the padding structure if we have one.
646   if (hasPadding())
647     addr = CGF.Builder.CreateStructGEP(addr, 0);
648 
649   // If we're emitting to an aggregate, copy into the result slot.
650   if (EvaluationKind == TEK_Aggregate) {
651     CGF.EmitAggregateCopy(resultSlot.getAddr(), addr, getValueType(),
652                           resultSlot.isVolatile());
653     return resultSlot.asRValue();
654   }
655 
656   // Otherwise, just convert the temporary to an r-value using the
657   // normal conversion routine.
658   return CGF.convertTempToRValue(addr, getValueType());
659 }
660 
661 /// Emit a load from an l-value of atomic type.  Note that the r-value
662 /// we produce is an r-value of the atomic *value* type.
663 RValue CodeGenFunction::EmitAtomicLoad(LValue src, AggValueSlot resultSlot) {
664   AtomicInfo atomics(*this, src);
665 
666   // Check whether we should use a library call.
667   if (atomics.shouldUseLibcall()) {
668     llvm::Value *tempAddr;
669     if (resultSlot.isValueOfAtomic()) {
670       assert(atomics.getEvaluationKind() == TEK_Aggregate);
671       tempAddr = resultSlot.getPaddedAtomicAddr();
672     } else if (!resultSlot.isIgnored() && !atomics.hasPadding()) {
673       assert(atomics.getEvaluationKind() == TEK_Aggregate);
674       tempAddr = resultSlot.getAddr();
675     } else {
676       tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
677     }
678 
679     // void __atomic_load(size_t size, void *mem, void *return, int order);
680     CallArgList args;
681     args.add(RValue::get(atomics.getAtomicSizeValue()),
682              getContext().getSizeType());
683     args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
684              getContext().VoidPtrTy);
685     args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
686              getContext().VoidPtrTy);
687     args.add(RValue::get(llvm::ConstantInt::get(IntTy,
688                                                 AO_ABI_memory_order_seq_cst)),
689              getContext().IntTy);
690     emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
691 
692     // Produce the r-value.
693     return atomics.convertTempToRValue(tempAddr, resultSlot);
694   }
695 
696   // Okay, we're doing this natively.
697   llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
698   llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
699   load->setAtomic(llvm::SequentiallyConsistent);
700 
701   // Other decoration.
702   load->setAlignment(src.getAlignment().getQuantity());
703   if (src.isVolatileQualified())
704     load->setVolatile(true);
705   if (src.getTBAAInfo())
706     CGM.DecorateInstruction(load, src.getTBAAInfo());
707 
708   // Okay, turn that back into the original value type.
709   QualType valueType = atomics.getValueType();
710   llvm::Value *result = load;
711 
712   // If we're ignoring an aggregate return, don't do anything.
713   if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
714     return RValue::getAggregate(0, false);
715 
716   // The easiest way to do this this is to go through memory, but we
717   // try not to in some easy cases.
718   if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) {
719     llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType);
720     if (isa<llvm::IntegerType>(resultTy)) {
721       assert(result->getType() == resultTy);
722       result = EmitFromMemory(result, valueType);
723     } else if (isa<llvm::PointerType>(resultTy)) {
724       result = Builder.CreateIntToPtr(result, resultTy);
725     } else {
726       result = Builder.CreateBitCast(result, resultTy);
727     }
728     return RValue::get(result);
729   }
730 
731   // Create a temporary.  This needs to be big enough to hold the
732   // atomic integer.
733   llvm::Value *temp;
734   bool tempIsVolatile = false;
735   CharUnits tempAlignment;
736   if (atomics.getEvaluationKind() == TEK_Aggregate &&
737       (!atomics.hasPadding() || resultSlot.isValueOfAtomic())) {
738     assert(!resultSlot.isIgnored());
739     if (resultSlot.isValueOfAtomic()) {
740       temp = resultSlot.getPaddedAtomicAddr();
741       tempAlignment = atomics.getAtomicAlignment();
742     } else {
743       temp = resultSlot.getAddr();
744       tempAlignment = atomics.getValueAlignment();
745     }
746     tempIsVolatile = resultSlot.isVolatile();
747   } else {
748     temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
749     tempAlignment = atomics.getAtomicAlignment();
750   }
751 
752   // Slam the integer into the temporary.
753   llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp);
754   Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity())
755     ->setVolatile(tempIsVolatile);
756 
757   return atomics.convertTempToRValue(temp, resultSlot);
758 }
759 
760 
761 
762 /// Copy an r-value into memory as part of storing to an atomic type.
763 /// This needs to create a bit-pattern suitable for atomic operations.
764 void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const {
765   // If we have an r-value, the rvalue should be of the atomic type,
766   // which means that the caller is responsible for having zeroed
767   // any padding.  Just do an aggregate copy of that type.
768   if (rvalue.isAggregate()) {
769     CGF.EmitAggregateCopy(dest.getAddress(),
770                           rvalue.getAggregateAddr(),
771                           getAtomicType(),
772                           (rvalue.isVolatileQualified()
773                            || dest.isVolatileQualified()),
774                           dest.getAlignment());
775     return;
776   }
777 
778   // Okay, otherwise we're copying stuff.
779 
780   // Zero out the buffer if necessary.
781   emitMemSetZeroIfNecessary(dest);
782 
783   // Drill past the padding if present.
784   dest = projectValue(dest);
785 
786   // Okay, store the rvalue in.
787   if (rvalue.isScalar()) {
788     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true);
789   } else {
790     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true);
791   }
792 }
793 
794 
795 /// Materialize an r-value into memory for the purposes of storing it
796 /// to an atomic type.
797 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
798   // Aggregate r-values are already in memory, and EmitAtomicStore
799   // requires them to be values of the atomic type.
800   if (rvalue.isAggregate())
801     return rvalue.getAggregateAddr();
802 
803   // Otherwise, make a temporary and materialize into it.
804   llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp");
805   LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment());
806   emitCopyIntoMemory(rvalue, tempLV);
807   return temp;
808 }
809 
810 /// Emit a store to an l-value of atomic type.
811 ///
812 /// Note that the r-value is expected to be an r-value *of the atomic
813 /// type*; this means that for aggregate r-values, it should include
814 /// storage for any padding that was necessary.
815 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
816                                       bool isInit) {
817   // If this is an aggregate r-value, it should agree in type except
818   // maybe for address-space qualification.
819   assert(!rvalue.isAggregate() ||
820          rvalue.getAggregateAddr()->getType()->getPointerElementType()
821            == dest.getAddress()->getType()->getPointerElementType());
822 
823   AtomicInfo atomics(*this, dest);
824 
825   // If this is an initialization, just put the value there normally.
826   if (isInit) {
827     atomics.emitCopyIntoMemory(rvalue, dest);
828     return;
829   }
830 
831   // Check whether we should use a library call.
832   if (atomics.shouldUseLibcall()) {
833     // Produce a source address.
834     llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
835 
836     // void __atomic_store(size_t size, void *mem, void *val, int order)
837     CallArgList args;
838     args.add(RValue::get(atomics.getAtomicSizeValue()),
839              getContext().getSizeType());
840     args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())),
841              getContext().VoidPtrTy);
842     args.add(RValue::get(EmitCastToVoidPtr(srcAddr)),
843              getContext().VoidPtrTy);
844     args.add(RValue::get(llvm::ConstantInt::get(IntTy,
845                                                 AO_ABI_memory_order_seq_cst)),
846              getContext().IntTy);
847     emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
848     return;
849   }
850 
851   // Okay, we're doing this natively.
852   llvm::Value *intValue;
853 
854   // If we've got a scalar value of the right size, try to avoid going
855   // through memory.
856   if (rvalue.isScalar() && !atomics.hasPadding()) {
857     llvm::Value *value = rvalue.getScalarVal();
858     if (isa<llvm::IntegerType>(value->getType())) {
859       intValue = value;
860     } else {
861       llvm::IntegerType *inputIntTy =
862         llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits());
863       if (isa<llvm::PointerType>(value->getType())) {
864         intValue = Builder.CreatePtrToInt(value, inputIntTy);
865       } else {
866         intValue = Builder.CreateBitCast(value, inputIntTy);
867       }
868     }
869 
870   // Otherwise, we need to go through memory.
871   } else {
872     // Put the r-value in memory.
873     llvm::Value *addr = atomics.materializeRValue(rvalue);
874 
875     // Cast the temporary to the atomic int type and pull a value out.
876     addr = atomics.emitCastToAtomicIntPointer(addr);
877     intValue = Builder.CreateAlignedLoad(addr,
878                                  atomics.getAtomicAlignment().getQuantity());
879   }
880 
881   // Do the atomic store.
882   llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
883   llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
884 
885   // Initializations don't need to be atomic.
886   if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
887 
888   // Other decoration.
889   store->setAlignment(dest.getAlignment().getQuantity());
890   if (dest.isVolatileQualified())
891     store->setVolatile(true);
892   if (dest.getTBAAInfo())
893     CGM.DecorateInstruction(store, dest.getTBAAInfo());
894 }
895 
896 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
897   AtomicInfo atomics(*this, dest);
898 
899   switch (atomics.getEvaluationKind()) {
900   case TEK_Scalar: {
901     llvm::Value *value = EmitScalarExpr(init);
902     atomics.emitCopyIntoMemory(RValue::get(value), dest);
903     return;
904   }
905 
906   case TEK_Complex: {
907     ComplexPairTy value = EmitComplexExpr(init);
908     atomics.emitCopyIntoMemory(RValue::getComplex(value), dest);
909     return;
910   }
911 
912   case TEK_Aggregate: {
913     // Memset the buffer first if there's any possibility of
914     // uninitialized internal bits.
915     atomics.emitMemSetZeroIfNecessary(dest);
916 
917     // HACK: whether the initializer actually has an atomic type
918     // doesn't really seem reliable right now.
919     if (!init->getType()->isAtomicType()) {
920       dest = atomics.projectValue(dest);
921     }
922 
923     // Evaluate the expression directly into the destination.
924     AggValueSlot slot = AggValueSlot::forLValue(dest,
925                                         AggValueSlot::IsNotDestructed,
926                                         AggValueSlot::DoesNotNeedGCBarriers,
927                                         AggValueSlot::IsNotAliased);
928     EmitAggExpr(init, slot);
929     return;
930   }
931   }
932   llvm_unreachable("bad evaluation kind");
933 }
934