1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the code for emitting atomic operations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CGCall.h" 15 #include "CGRecordLayout.h" 16 #include "CodeGenFunction.h" 17 #include "CodeGenModule.h" 18 #include "TargetInfo.h" 19 #include "clang/AST/ASTContext.h" 20 #include "clang/CodeGen/CGFunctionInfo.h" 21 #include "clang/Sema/SemaDiagnostic.h" 22 #include "llvm/ADT/DenseMap.h" 23 #include "llvm/IR/DataLayout.h" 24 #include "llvm/IR/Intrinsics.h" 25 #include "llvm/IR/Operator.h" 26 27 using namespace clang; 28 using namespace CodeGen; 29 30 namespace { 31 class AtomicInfo { 32 CodeGenFunction &CGF; 33 QualType AtomicTy; 34 QualType ValueTy; 35 uint64_t AtomicSizeInBits; 36 uint64_t ValueSizeInBits; 37 CharUnits AtomicAlign; 38 CharUnits ValueAlign; 39 CharUnits LValueAlign; 40 TypeEvaluationKind EvaluationKind; 41 bool UseLibcall; 42 LValue LVal; 43 CGBitFieldInfo BFI; 44 public: 45 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 46 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 47 EvaluationKind(TEK_Scalar), UseLibcall(true) { 48 assert(!lvalue.isGlobalReg()); 49 ASTContext &C = CGF.getContext(); 50 if (lvalue.isSimple()) { 51 AtomicTy = lvalue.getType(); 52 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 53 ValueTy = ATy->getValueType(); 54 else 55 ValueTy = AtomicTy; 56 EvaluationKind = CGF.getEvaluationKind(ValueTy); 57 58 uint64_t ValueAlignInBits; 59 uint64_t AtomicAlignInBits; 60 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 61 ValueSizeInBits = ValueTI.Width; 62 ValueAlignInBits = ValueTI.Align; 63 64 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 65 AtomicSizeInBits = AtomicTI.Width; 66 AtomicAlignInBits = AtomicTI.Align; 67 68 assert(ValueSizeInBits <= AtomicSizeInBits); 69 assert(ValueAlignInBits <= AtomicAlignInBits); 70 71 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 72 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 73 if (lvalue.getAlignment().isZero()) 74 lvalue.setAlignment(AtomicAlign); 75 76 LVal = lvalue; 77 } else if (lvalue.isBitField()) { 78 ValueTy = lvalue.getType(); 79 ValueSizeInBits = C.getTypeSize(ValueTy); 80 auto &OrigBFI = lvalue.getBitFieldInfo(); 81 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 82 AtomicSizeInBits = C.toBits( 83 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 84 .alignTo(lvalue.getAlignment())); 85 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 86 auto OffsetInChars = 87 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 88 lvalue.getAlignment(); 89 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 90 VoidPtrAddr, OffsetInChars.getQuantity()); 91 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 92 VoidPtrAddr, 93 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 94 "atomic_bitfield_base"); 95 BFI = OrigBFI; 96 BFI.Offset = Offset; 97 BFI.StorageSize = AtomicSizeInBits; 98 BFI.StorageOffset += OffsetInChars; 99 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 100 BFI, lvalue.getType(), lvalue.getBaseInfo(), 101 lvalue.getTBAAInfo()); 102 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 103 if (AtomicTy.isNull()) { 104 llvm::APInt Size( 105 /*numBits=*/32, 106 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 107 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, 108 /*IndexTypeQuals=*/0); 109 } 110 AtomicAlign = ValueAlign = lvalue.getAlignment(); 111 } else if (lvalue.isVectorElt()) { 112 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); 113 ValueSizeInBits = C.getTypeSize(ValueTy); 114 AtomicTy = lvalue.getType(); 115 AtomicSizeInBits = C.getTypeSize(AtomicTy); 116 AtomicAlign = ValueAlign = lvalue.getAlignment(); 117 LVal = lvalue; 118 } else { 119 assert(lvalue.isExtVectorElt()); 120 ValueTy = lvalue.getType(); 121 ValueSizeInBits = C.getTypeSize(ValueTy); 122 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 123 lvalue.getType(), lvalue.getExtVectorAddress() 124 .getElementType()->getVectorNumElements()); 125 AtomicSizeInBits = C.getTypeSize(AtomicTy); 126 AtomicAlign = ValueAlign = lvalue.getAlignment(); 127 LVal = lvalue; 128 } 129 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 130 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 131 } 132 133 QualType getAtomicType() const { return AtomicTy; } 134 QualType getValueType() const { return ValueTy; } 135 CharUnits getAtomicAlignment() const { return AtomicAlign; } 136 CharUnits getValueAlignment() const { return ValueAlign; } 137 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 138 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 139 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 140 bool shouldUseLibcall() const { return UseLibcall; } 141 const LValue &getAtomicLValue() const { return LVal; } 142 llvm::Value *getAtomicPointer() const { 143 if (LVal.isSimple()) 144 return LVal.getPointer(); 145 else if (LVal.isBitField()) 146 return LVal.getBitFieldPointer(); 147 else if (LVal.isVectorElt()) 148 return LVal.getVectorPointer(); 149 assert(LVal.isExtVectorElt()); 150 return LVal.getExtVectorPointer(); 151 } 152 Address getAtomicAddress() const { 153 return Address(getAtomicPointer(), getAtomicAlignment()); 154 } 155 156 Address getAtomicAddressAsAtomicIntPointer() const { 157 return emitCastToAtomicIntPointer(getAtomicAddress()); 158 } 159 160 /// Is the atomic size larger than the underlying value type? 161 /// 162 /// Note that the absence of padding does not mean that atomic 163 /// objects are completely interchangeable with non-atomic 164 /// objects: we might have promoted the alignment of a type 165 /// without making it bigger. 166 bool hasPadding() const { 167 return (ValueSizeInBits != AtomicSizeInBits); 168 } 169 170 bool emitMemSetZeroIfNecessary() const; 171 172 llvm::Value *getAtomicSizeValue() const { 173 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 174 return CGF.CGM.getSize(size); 175 } 176 177 /// Cast the given pointer to an integer pointer suitable for atomic 178 /// operations if the source. 179 Address emitCastToAtomicIntPointer(Address Addr) const; 180 181 /// If Addr is compatible with the iN that will be used for an atomic 182 /// operation, bitcast it. Otherwise, create a temporary that is suitable 183 /// and copy the value across. 184 Address convertToAtomicIntPointer(Address Addr) const; 185 186 /// Turn an atomic-layout object into an r-value. 187 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 188 SourceLocation loc, bool AsValue) const; 189 190 /// Converts a rvalue to integer value. 191 llvm::Value *convertRValueToInt(RValue RVal) const; 192 193 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 194 AggValueSlot ResultSlot, 195 SourceLocation Loc, bool AsValue) const; 196 197 /// Copy an atomic r-value into atomic-layout memory. 198 void emitCopyIntoMemory(RValue rvalue) const; 199 200 /// Project an l-value down to the value field. 201 LValue projectValue() const { 202 assert(LVal.isSimple()); 203 Address addr = getAtomicAddress(); 204 if (hasPadding()) 205 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 206 207 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 208 LVal.getBaseInfo(), LVal.getTBAAInfo()); 209 } 210 211 /// Emits atomic load. 212 /// \returns Loaded value. 213 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 214 bool AsValue, llvm::AtomicOrdering AO, 215 bool IsVolatile); 216 217 /// Emits atomic compare-and-exchange sequence. 218 /// \param Expected Expected value. 219 /// \param Desired Desired value. 220 /// \param Success Atomic ordering for success operation. 221 /// \param Failure Atomic ordering for failed operation. 222 /// \param IsWeak true if atomic operation is weak, false otherwise. 223 /// \returns Pair of values: previous value from storage (value type) and 224 /// boolean flag (i1 type) with true if success and false otherwise. 225 std::pair<RValue, llvm::Value *> 226 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 227 llvm::AtomicOrdering Success = 228 llvm::AtomicOrdering::SequentiallyConsistent, 229 llvm::AtomicOrdering Failure = 230 llvm::AtomicOrdering::SequentiallyConsistent, 231 bool IsWeak = false); 232 233 /// Emits atomic update. 234 /// \param AO Atomic ordering. 235 /// \param UpdateOp Update operation for the current lvalue. 236 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 237 const llvm::function_ref<RValue(RValue)> &UpdateOp, 238 bool IsVolatile); 239 /// Emits atomic update. 240 /// \param AO Atomic ordering. 241 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 242 bool IsVolatile); 243 244 /// Materialize an atomic r-value in atomic-layout memory. 245 Address materializeRValue(RValue rvalue) const; 246 247 /// Creates temp alloca for intermediate operations on atomic value. 248 Address CreateTempAlloca() const; 249 private: 250 bool requiresMemSetZero(llvm::Type *type) const; 251 252 253 /// Emits atomic load as a libcall. 254 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 255 llvm::AtomicOrdering AO, bool IsVolatile); 256 /// Emits atomic load as LLVM instruction. 257 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 258 /// Emits atomic compare-and-exchange op as a libcall. 259 llvm::Value *EmitAtomicCompareExchangeLibcall( 260 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 261 llvm::AtomicOrdering Success = 262 llvm::AtomicOrdering::SequentiallyConsistent, 263 llvm::AtomicOrdering Failure = 264 llvm::AtomicOrdering::SequentiallyConsistent); 265 /// Emits atomic compare-and-exchange op as LLVM instruction. 266 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 267 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 268 llvm::AtomicOrdering Success = 269 llvm::AtomicOrdering::SequentiallyConsistent, 270 llvm::AtomicOrdering Failure = 271 llvm::AtomicOrdering::SequentiallyConsistent, 272 bool IsWeak = false); 273 /// Emit atomic update as libcalls. 274 void 275 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 276 const llvm::function_ref<RValue(RValue)> &UpdateOp, 277 bool IsVolatile); 278 /// Emit atomic update as LLVM instructions. 279 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 280 const llvm::function_ref<RValue(RValue)> &UpdateOp, 281 bool IsVolatile); 282 /// Emit atomic update as libcalls. 283 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 284 bool IsVolatile); 285 /// Emit atomic update as LLVM instructions. 286 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 287 bool IsVolatile); 288 }; 289 } 290 291 Address AtomicInfo::CreateTempAlloca() const { 292 Address TempAlloca = CGF.CreateMemTemp( 293 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 294 : AtomicTy, 295 getAtomicAlignment(), 296 "atomic-temp"); 297 // Cast to pointer to value type for bitfields. 298 if (LVal.isBitField()) 299 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 300 TempAlloca, getAtomicAddress().getType()); 301 return TempAlloca; 302 } 303 304 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 305 StringRef fnName, 306 QualType resultType, 307 CallArgList &args) { 308 const CGFunctionInfo &fnInfo = 309 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 310 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 311 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 312 auto callee = CGCallee::forDirect(fn); 313 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 314 } 315 316 /// Does a store of the given IR type modify the full expected width? 317 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 318 uint64_t expectedSize) { 319 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 320 } 321 322 /// Does the atomic type require memsetting to zero before initialization? 323 /// 324 /// The IR type is provided as a way of making certain queries faster. 325 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 326 // If the atomic type has size padding, we definitely need a memset. 327 if (hasPadding()) return true; 328 329 // Otherwise, do some simple heuristics to try to avoid it: 330 switch (getEvaluationKind()) { 331 // For scalars and complexes, check whether the store size of the 332 // type uses the full size. 333 case TEK_Scalar: 334 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 335 case TEK_Complex: 336 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 337 AtomicSizeInBits / 2); 338 339 // Padding in structs has an undefined bit pattern. User beware. 340 case TEK_Aggregate: 341 return false; 342 } 343 llvm_unreachable("bad evaluation kind"); 344 } 345 346 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 347 assert(LVal.isSimple()); 348 llvm::Value *addr = LVal.getPointer(); 349 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 350 return false; 351 352 CGF.Builder.CreateMemSet( 353 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 354 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 355 LVal.getAlignment().getQuantity()); 356 return true; 357 } 358 359 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 360 Address Dest, Address Ptr, 361 Address Val1, Address Val2, 362 uint64_t Size, 363 llvm::AtomicOrdering SuccessOrder, 364 llvm::AtomicOrdering FailureOrder, 365 llvm::SyncScope::ID Scope) { 366 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 367 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 368 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 369 370 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 371 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 372 Scope); 373 Pair->setVolatile(E->isVolatile()); 374 Pair->setWeak(IsWeak); 375 376 // Cmp holds the result of the compare-exchange operation: true on success, 377 // false on failure. 378 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 379 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 380 381 // This basic block is used to hold the store instruction if the operation 382 // failed. 383 llvm::BasicBlock *StoreExpectedBB = 384 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 385 386 // This basic block is the exit point of the operation, we should end up 387 // here regardless of whether or not the operation succeeded. 388 llvm::BasicBlock *ContinueBB = 389 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 390 391 // Update Expected if Expected isn't equal to Old, otherwise branch to the 392 // exit point. 393 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 394 395 CGF.Builder.SetInsertPoint(StoreExpectedBB); 396 // Update the memory at Expected with Old's value. 397 CGF.Builder.CreateStore(Old, Val1); 398 // Finally, branch to the exit point. 399 CGF.Builder.CreateBr(ContinueBB); 400 401 CGF.Builder.SetInsertPoint(ContinueBB); 402 // Update the memory at Dest with Cmp's value. 403 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 404 } 405 406 /// Given an ordering required on success, emit all possible cmpxchg 407 /// instructions to cope with the provided (but possibly only dynamically known) 408 /// FailureOrder. 409 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 410 bool IsWeak, Address Dest, Address Ptr, 411 Address Val1, Address Val2, 412 llvm::Value *FailureOrderVal, 413 uint64_t Size, 414 llvm::AtomicOrdering SuccessOrder, 415 llvm::SyncScope::ID Scope) { 416 llvm::AtomicOrdering FailureOrder; 417 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 418 auto FOS = FO->getSExtValue(); 419 if (!llvm::isValidAtomicOrderingCABI(FOS)) 420 FailureOrder = llvm::AtomicOrdering::Monotonic; 421 else 422 switch ((llvm::AtomicOrderingCABI)FOS) { 423 case llvm::AtomicOrderingCABI::relaxed: 424 case llvm::AtomicOrderingCABI::release: 425 case llvm::AtomicOrderingCABI::acq_rel: 426 FailureOrder = llvm::AtomicOrdering::Monotonic; 427 break; 428 case llvm::AtomicOrderingCABI::consume: 429 case llvm::AtomicOrderingCABI::acquire: 430 FailureOrder = llvm::AtomicOrdering::Acquire; 431 break; 432 case llvm::AtomicOrderingCABI::seq_cst: 433 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 434 break; 435 } 436 if (isStrongerThan(FailureOrder, SuccessOrder)) { 437 // Don't assert on undefined behavior "failure argument shall be no 438 // stronger than the success argument". 439 FailureOrder = 440 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 441 } 442 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 443 FailureOrder, Scope); 444 return; 445 } 446 447 // Create all the relevant BB's 448 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 449 *SeqCstBB = nullptr; 450 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 451 if (SuccessOrder != llvm::AtomicOrdering::Monotonic && 452 SuccessOrder != llvm::AtomicOrdering::Release) 453 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 454 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 455 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 456 457 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 458 459 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 460 461 // Emit all the different atomics 462 463 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 464 // doesn't matter unless someone is crazy enough to use something that 465 // doesn't fold to a constant for the ordering. 466 CGF.Builder.SetInsertPoint(MonotonicBB); 467 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 468 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 469 CGF.Builder.CreateBr(ContBB); 470 471 if (AcquireBB) { 472 CGF.Builder.SetInsertPoint(AcquireBB); 473 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 474 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope); 475 CGF.Builder.CreateBr(ContBB); 476 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 477 AcquireBB); 478 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 479 AcquireBB); 480 } 481 if (SeqCstBB) { 482 CGF.Builder.SetInsertPoint(SeqCstBB); 483 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 484 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 485 CGF.Builder.CreateBr(ContBB); 486 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 487 SeqCstBB); 488 } 489 490 CGF.Builder.SetInsertPoint(ContBB); 491 } 492 493 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 494 Address Ptr, Address Val1, Address Val2, 495 llvm::Value *IsWeak, llvm::Value *FailureOrder, 496 uint64_t Size, llvm::AtomicOrdering Order, 497 llvm::SyncScope::ID Scope) { 498 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 499 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 500 501 switch (E->getOp()) { 502 case AtomicExpr::AO__c11_atomic_init: 503 case AtomicExpr::AO__opencl_atomic_init: 504 llvm_unreachable("Already handled!"); 505 506 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 507 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 508 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 509 FailureOrder, Size, Order, Scope); 510 return; 511 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 512 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 513 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 514 FailureOrder, Size, Order, Scope); 515 return; 516 case AtomicExpr::AO__atomic_compare_exchange: 517 case AtomicExpr::AO__atomic_compare_exchange_n: { 518 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 519 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 520 Val1, Val2, FailureOrder, Size, Order, Scope); 521 } else { 522 // Create all the relevant BB's 523 llvm::BasicBlock *StrongBB = 524 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 525 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 526 llvm::BasicBlock *ContBB = 527 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 528 529 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 530 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 531 532 CGF.Builder.SetInsertPoint(StrongBB); 533 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 534 FailureOrder, Size, Order, Scope); 535 CGF.Builder.CreateBr(ContBB); 536 537 CGF.Builder.SetInsertPoint(WeakBB); 538 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 539 FailureOrder, Size, Order, Scope); 540 CGF.Builder.CreateBr(ContBB); 541 542 CGF.Builder.SetInsertPoint(ContBB); 543 } 544 return; 545 } 546 case AtomicExpr::AO__c11_atomic_load: 547 case AtomicExpr::AO__opencl_atomic_load: 548 case AtomicExpr::AO__atomic_load_n: 549 case AtomicExpr::AO__atomic_load: { 550 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 551 Load->setAtomic(Order, Scope); 552 Load->setVolatile(E->isVolatile()); 553 CGF.Builder.CreateStore(Load, Dest); 554 return; 555 } 556 557 case AtomicExpr::AO__c11_atomic_store: 558 case AtomicExpr::AO__opencl_atomic_store: 559 case AtomicExpr::AO__atomic_store: 560 case AtomicExpr::AO__atomic_store_n: { 561 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 562 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 563 Store->setAtomic(Order, Scope); 564 Store->setVolatile(E->isVolatile()); 565 return; 566 } 567 568 case AtomicExpr::AO__c11_atomic_exchange: 569 case AtomicExpr::AO__opencl_atomic_exchange: 570 case AtomicExpr::AO__atomic_exchange_n: 571 case AtomicExpr::AO__atomic_exchange: 572 Op = llvm::AtomicRMWInst::Xchg; 573 break; 574 575 case AtomicExpr::AO__atomic_add_fetch: 576 PostOp = llvm::Instruction::Add; 577 LLVM_FALLTHROUGH; 578 case AtomicExpr::AO__c11_atomic_fetch_add: 579 case AtomicExpr::AO__opencl_atomic_fetch_add: 580 case AtomicExpr::AO__atomic_fetch_add: 581 Op = llvm::AtomicRMWInst::Add; 582 break; 583 584 case AtomicExpr::AO__atomic_sub_fetch: 585 PostOp = llvm::Instruction::Sub; 586 LLVM_FALLTHROUGH; 587 case AtomicExpr::AO__c11_atomic_fetch_sub: 588 case AtomicExpr::AO__opencl_atomic_fetch_sub: 589 case AtomicExpr::AO__atomic_fetch_sub: 590 Op = llvm::AtomicRMWInst::Sub; 591 break; 592 593 case AtomicExpr::AO__opencl_atomic_fetch_min: 594 case AtomicExpr::AO__atomic_fetch_min: 595 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min 596 : llvm::AtomicRMWInst::UMin; 597 break; 598 599 case AtomicExpr::AO__opencl_atomic_fetch_max: 600 case AtomicExpr::AO__atomic_fetch_max: 601 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max 602 : llvm::AtomicRMWInst::UMax; 603 break; 604 605 case AtomicExpr::AO__atomic_and_fetch: 606 PostOp = llvm::Instruction::And; 607 LLVM_FALLTHROUGH; 608 case AtomicExpr::AO__c11_atomic_fetch_and: 609 case AtomicExpr::AO__opencl_atomic_fetch_and: 610 case AtomicExpr::AO__atomic_fetch_and: 611 Op = llvm::AtomicRMWInst::And; 612 break; 613 614 case AtomicExpr::AO__atomic_or_fetch: 615 PostOp = llvm::Instruction::Or; 616 LLVM_FALLTHROUGH; 617 case AtomicExpr::AO__c11_atomic_fetch_or: 618 case AtomicExpr::AO__opencl_atomic_fetch_or: 619 case AtomicExpr::AO__atomic_fetch_or: 620 Op = llvm::AtomicRMWInst::Or; 621 break; 622 623 case AtomicExpr::AO__atomic_xor_fetch: 624 PostOp = llvm::Instruction::Xor; 625 LLVM_FALLTHROUGH; 626 case AtomicExpr::AO__c11_atomic_fetch_xor: 627 case AtomicExpr::AO__opencl_atomic_fetch_xor: 628 case AtomicExpr::AO__atomic_fetch_xor: 629 Op = llvm::AtomicRMWInst::Xor; 630 break; 631 632 case AtomicExpr::AO__atomic_nand_fetch: 633 PostOp = llvm::Instruction::And; // the NOT is special cased below 634 LLVM_FALLTHROUGH; 635 case AtomicExpr::AO__atomic_fetch_nand: 636 Op = llvm::AtomicRMWInst::Nand; 637 break; 638 } 639 640 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 641 llvm::AtomicRMWInst *RMWI = 642 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 643 RMWI->setVolatile(E->isVolatile()); 644 645 // For __atomic_*_fetch operations, perform the operation again to 646 // determine the value which was written. 647 llvm::Value *Result = RMWI; 648 if (PostOp) 649 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 650 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 651 Result = CGF.Builder.CreateNot(Result); 652 CGF.Builder.CreateStore(Result, Dest); 653 } 654 655 // This function emits any expression (scalar, complex, or aggregate) 656 // into a temporary alloca. 657 static Address 658 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 659 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 660 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 661 /*Init*/ true); 662 return DeclPtr; 663 } 664 665 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 666 Address Ptr, Address Val1, Address Val2, 667 llvm::Value *IsWeak, llvm::Value *FailureOrder, 668 uint64_t Size, llvm::AtomicOrdering Order, 669 llvm::Value *Scope) { 670 auto ScopeModel = Expr->getScopeModel(); 671 672 // LLVM atomic instructions always have synch scope. If clang atomic 673 // expression has no scope operand, use default LLVM synch scope. 674 if (!ScopeModel) { 675 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 676 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 677 return; 678 } 679 680 // Handle constant scope. 681 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 682 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 683 ScopeModel->map(SC->getZExtValue()), CGF.CGM.getLLVMContext()); 684 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 685 Order, SCID); 686 return; 687 } 688 689 // Handle non-constant scope. 690 auto &Builder = CGF.Builder; 691 auto Scopes = ScopeModel->getRuntimeValues(); 692 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 693 for (auto S : Scopes) 694 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 695 696 llvm::BasicBlock *ContBB = 697 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 698 699 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 700 // If unsupported synch scope is encountered at run time, assume a fallback 701 // synch scope value. 702 auto FallBack = ScopeModel->getFallBackValue(); 703 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 704 for (auto S : Scopes) { 705 auto *B = BB[S]; 706 if (S != FallBack) 707 SI->addCase(Builder.getInt32(S), B); 708 709 Builder.SetInsertPoint(B); 710 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 711 Order, 712 CGF.getTargetHooks().getLLVMSyncScopeID(ScopeModel->map(S), 713 CGF.getLLVMContext())); 714 Builder.CreateBr(ContBB); 715 } 716 717 Builder.SetInsertPoint(ContBB); 718 } 719 720 static void 721 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 722 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 723 SourceLocation Loc, CharUnits SizeInChars) { 724 if (UseOptimizedLibcall) { 725 // Load value and pass it to the function directly. 726 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 727 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 728 ValTy = 729 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 730 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 731 SizeInBits)->getPointerTo(); 732 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 733 Val = CGF.EmitLoadOfScalar(Ptr, false, 734 CGF.getContext().getPointerType(ValTy), 735 Loc); 736 // Coerce the value into an appropriately sized integer type. 737 Args.add(RValue::get(Val), ValTy); 738 } else { 739 // Non-optimized functions always take a reference. 740 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 741 CGF.getContext().VoidPtrTy); 742 } 743 } 744 745 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 746 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 747 QualType MemTy = AtomicTy; 748 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 749 MemTy = AT->getValueType(); 750 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 751 752 Address Val1 = Address::invalid(); 753 Address Val2 = Address::invalid(); 754 Address Dest = Address::invalid(); 755 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 756 757 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 758 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 759 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 760 EmitAtomicInit(E->getVal1(), lvalue); 761 return RValue::get(nullptr); 762 } 763 764 CharUnits sizeChars, alignChars; 765 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); 766 uint64_t Size = sizeChars.getQuantity(); 767 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 768 bool UseLibcall = ((Ptr.getAlignment() % sizeChars) != 0 || 769 getContext().toBits(sizeChars) > MaxInlineWidthInBits); 770 771 if (UseLibcall) 772 CGM.getDiags().Report(E->getLocStart(), diag::warn_atomic_op_misaligned); 773 774 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 775 llvm::Value *Scope = 776 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 777 778 switch (E->getOp()) { 779 case AtomicExpr::AO__c11_atomic_init: 780 case AtomicExpr::AO__opencl_atomic_init: 781 llvm_unreachable("Already handled above with EmitAtomicInit!"); 782 783 case AtomicExpr::AO__c11_atomic_load: 784 case AtomicExpr::AO__opencl_atomic_load: 785 case AtomicExpr::AO__atomic_load_n: 786 break; 787 788 case AtomicExpr::AO__atomic_load: 789 Dest = EmitPointerWithAlignment(E->getVal1()); 790 break; 791 792 case AtomicExpr::AO__atomic_store: 793 Val1 = EmitPointerWithAlignment(E->getVal1()); 794 break; 795 796 case AtomicExpr::AO__atomic_exchange: 797 Val1 = EmitPointerWithAlignment(E->getVal1()); 798 Dest = EmitPointerWithAlignment(E->getVal2()); 799 break; 800 801 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 802 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 803 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 804 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 805 case AtomicExpr::AO__atomic_compare_exchange_n: 806 case AtomicExpr::AO__atomic_compare_exchange: 807 Val1 = EmitPointerWithAlignment(E->getVal1()); 808 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 809 Val2 = EmitPointerWithAlignment(E->getVal2()); 810 else 811 Val2 = EmitValToTemp(*this, E->getVal2()); 812 OrderFail = EmitScalarExpr(E->getOrderFail()); 813 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 814 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 815 IsWeak = EmitScalarExpr(E->getWeak()); 816 break; 817 818 case AtomicExpr::AO__c11_atomic_fetch_add: 819 case AtomicExpr::AO__c11_atomic_fetch_sub: 820 case AtomicExpr::AO__opencl_atomic_fetch_add: 821 case AtomicExpr::AO__opencl_atomic_fetch_sub: 822 if (MemTy->isPointerType()) { 823 // For pointer arithmetic, we're required to do a bit of math: 824 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 825 // ... but only for the C11 builtins. The GNU builtins expect the 826 // user to multiply by sizeof(T). 827 QualType Val1Ty = E->getVal1()->getType(); 828 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 829 CharUnits PointeeIncAmt = 830 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 831 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 832 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 833 Val1 = Temp; 834 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 835 break; 836 } 837 LLVM_FALLTHROUGH; 838 case AtomicExpr::AO__atomic_fetch_add: 839 case AtomicExpr::AO__atomic_fetch_sub: 840 case AtomicExpr::AO__atomic_add_fetch: 841 case AtomicExpr::AO__atomic_sub_fetch: 842 case AtomicExpr::AO__c11_atomic_store: 843 case AtomicExpr::AO__c11_atomic_exchange: 844 case AtomicExpr::AO__opencl_atomic_store: 845 case AtomicExpr::AO__opencl_atomic_exchange: 846 case AtomicExpr::AO__atomic_store_n: 847 case AtomicExpr::AO__atomic_exchange_n: 848 case AtomicExpr::AO__c11_atomic_fetch_and: 849 case AtomicExpr::AO__c11_atomic_fetch_or: 850 case AtomicExpr::AO__c11_atomic_fetch_xor: 851 case AtomicExpr::AO__opencl_atomic_fetch_and: 852 case AtomicExpr::AO__opencl_atomic_fetch_or: 853 case AtomicExpr::AO__opencl_atomic_fetch_xor: 854 case AtomicExpr::AO__opencl_atomic_fetch_min: 855 case AtomicExpr::AO__opencl_atomic_fetch_max: 856 case AtomicExpr::AO__atomic_fetch_and: 857 case AtomicExpr::AO__atomic_fetch_or: 858 case AtomicExpr::AO__atomic_fetch_xor: 859 case AtomicExpr::AO__atomic_fetch_nand: 860 case AtomicExpr::AO__atomic_and_fetch: 861 case AtomicExpr::AO__atomic_or_fetch: 862 case AtomicExpr::AO__atomic_xor_fetch: 863 case AtomicExpr::AO__atomic_nand_fetch: 864 case AtomicExpr::AO__atomic_fetch_min: 865 case AtomicExpr::AO__atomic_fetch_max: 866 Val1 = EmitValToTemp(*this, E->getVal1()); 867 break; 868 } 869 870 QualType RValTy = E->getType().getUnqualifiedType(); 871 872 // The inlined atomics only function on iN types, where N is a power of 2. We 873 // need to make sure (via temporaries if necessary) that all incoming values 874 // are compatible. 875 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 876 AtomicInfo Atomics(*this, AtomicVal); 877 878 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 879 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); 880 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); 881 if (Dest.isValid()) 882 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 883 else if (E->isCmpXChg()) 884 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 885 else if (!RValTy->isVoidType()) 886 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); 887 888 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 889 if (UseLibcall) { 890 bool UseOptimizedLibcall = false; 891 switch (E->getOp()) { 892 case AtomicExpr::AO__c11_atomic_init: 893 case AtomicExpr::AO__opencl_atomic_init: 894 llvm_unreachable("Already handled above with EmitAtomicInit!"); 895 896 case AtomicExpr::AO__c11_atomic_fetch_add: 897 case AtomicExpr::AO__opencl_atomic_fetch_add: 898 case AtomicExpr::AO__atomic_fetch_add: 899 case AtomicExpr::AO__c11_atomic_fetch_and: 900 case AtomicExpr::AO__opencl_atomic_fetch_and: 901 case AtomicExpr::AO__atomic_fetch_and: 902 case AtomicExpr::AO__c11_atomic_fetch_or: 903 case AtomicExpr::AO__opencl_atomic_fetch_or: 904 case AtomicExpr::AO__atomic_fetch_or: 905 case AtomicExpr::AO__atomic_fetch_nand: 906 case AtomicExpr::AO__c11_atomic_fetch_sub: 907 case AtomicExpr::AO__opencl_atomic_fetch_sub: 908 case AtomicExpr::AO__atomic_fetch_sub: 909 case AtomicExpr::AO__c11_atomic_fetch_xor: 910 case AtomicExpr::AO__opencl_atomic_fetch_xor: 911 case AtomicExpr::AO__opencl_atomic_fetch_min: 912 case AtomicExpr::AO__opencl_atomic_fetch_max: 913 case AtomicExpr::AO__atomic_fetch_xor: 914 case AtomicExpr::AO__atomic_add_fetch: 915 case AtomicExpr::AO__atomic_and_fetch: 916 case AtomicExpr::AO__atomic_nand_fetch: 917 case AtomicExpr::AO__atomic_or_fetch: 918 case AtomicExpr::AO__atomic_sub_fetch: 919 case AtomicExpr::AO__atomic_xor_fetch: 920 case AtomicExpr::AO__atomic_fetch_min: 921 case AtomicExpr::AO__atomic_fetch_max: 922 // For these, only library calls for certain sizes exist. 923 UseOptimizedLibcall = true; 924 break; 925 926 case AtomicExpr::AO__c11_atomic_load: 927 case AtomicExpr::AO__c11_atomic_store: 928 case AtomicExpr::AO__c11_atomic_exchange: 929 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 930 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 931 case AtomicExpr::AO__opencl_atomic_load: 932 case AtomicExpr::AO__opencl_atomic_store: 933 case AtomicExpr::AO__opencl_atomic_exchange: 934 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 935 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 936 case AtomicExpr::AO__atomic_load_n: 937 case AtomicExpr::AO__atomic_load: 938 case AtomicExpr::AO__atomic_store_n: 939 case AtomicExpr::AO__atomic_store: 940 case AtomicExpr::AO__atomic_exchange_n: 941 case AtomicExpr::AO__atomic_exchange: 942 case AtomicExpr::AO__atomic_compare_exchange_n: 943 case AtomicExpr::AO__atomic_compare_exchange: 944 // Only use optimized library calls for sizes for which they exist. 945 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 946 UseOptimizedLibcall = true; 947 break; 948 } 949 950 CallArgList Args; 951 if (!UseOptimizedLibcall) { 952 // For non-optimized library calls, the size is the first parameter 953 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 954 getContext().getSizeType()); 955 } 956 // Atomic address is the first or second parameter 957 // The OpenCL atomic library functions only accept pointer arguments to 958 // generic address space. 959 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 960 if (!E->isOpenCL()) 961 return V; 962 auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace(); 963 if (AS == LangAS::opencl_generic) 964 return V; 965 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 966 auto T = V->getType(); 967 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS); 968 969 return getTargetHooks().performAddrSpaceCast( 970 *this, V, AS, LangAS::opencl_generic, DestType, false); 971 }; 972 973 Args.add(RValue::get(CastToGenericAddrSpace( 974 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), 975 getContext().VoidPtrTy); 976 977 std::string LibCallName; 978 QualType LoweredMemTy = 979 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 980 QualType RetTy; 981 bool HaveRetTy = false; 982 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 983 switch (E->getOp()) { 984 case AtomicExpr::AO__c11_atomic_init: 985 case AtomicExpr::AO__opencl_atomic_init: 986 llvm_unreachable("Already handled!"); 987 988 // There is only one libcall for compare an exchange, because there is no 989 // optimisation benefit possible from a libcall version of a weak compare 990 // and exchange. 991 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 992 // void *desired, int success, int failure) 993 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 994 // int success, int failure) 995 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 996 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 997 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 998 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 999 case AtomicExpr::AO__atomic_compare_exchange: 1000 case AtomicExpr::AO__atomic_compare_exchange_n: 1001 LibCallName = "__atomic_compare_exchange"; 1002 RetTy = getContext().BoolTy; 1003 HaveRetTy = true; 1004 Args.add( 1005 RValue::get(CastToGenericAddrSpace( 1006 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), 1007 getContext().VoidPtrTy); 1008 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 1009 MemTy, E->getExprLoc(), sizeChars); 1010 Args.add(RValue::get(Order), getContext().IntTy); 1011 Order = OrderFail; 1012 break; 1013 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1014 // int order) 1015 // T __atomic_exchange_N(T *mem, T val, int order) 1016 case AtomicExpr::AO__c11_atomic_exchange: 1017 case AtomicExpr::AO__opencl_atomic_exchange: 1018 case AtomicExpr::AO__atomic_exchange_n: 1019 case AtomicExpr::AO__atomic_exchange: 1020 LibCallName = "__atomic_exchange"; 1021 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1022 MemTy, E->getExprLoc(), sizeChars); 1023 break; 1024 // void __atomic_store(size_t size, void *mem, void *val, int order) 1025 // void __atomic_store_N(T *mem, T val, int order) 1026 case AtomicExpr::AO__c11_atomic_store: 1027 case AtomicExpr::AO__opencl_atomic_store: 1028 case AtomicExpr::AO__atomic_store: 1029 case AtomicExpr::AO__atomic_store_n: 1030 LibCallName = "__atomic_store"; 1031 RetTy = getContext().VoidTy; 1032 HaveRetTy = true; 1033 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1034 MemTy, E->getExprLoc(), sizeChars); 1035 break; 1036 // void __atomic_load(size_t size, void *mem, void *return, int order) 1037 // T __atomic_load_N(T *mem, int order) 1038 case AtomicExpr::AO__c11_atomic_load: 1039 case AtomicExpr::AO__opencl_atomic_load: 1040 case AtomicExpr::AO__atomic_load: 1041 case AtomicExpr::AO__atomic_load_n: 1042 LibCallName = "__atomic_load"; 1043 break; 1044 // T __atomic_add_fetch_N(T *mem, T val, int order) 1045 // T __atomic_fetch_add_N(T *mem, T val, int order) 1046 case AtomicExpr::AO__atomic_add_fetch: 1047 PostOp = llvm::Instruction::Add; 1048 LLVM_FALLTHROUGH; 1049 case AtomicExpr::AO__c11_atomic_fetch_add: 1050 case AtomicExpr::AO__opencl_atomic_fetch_add: 1051 case AtomicExpr::AO__atomic_fetch_add: 1052 LibCallName = "__atomic_fetch_add"; 1053 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1054 LoweredMemTy, E->getExprLoc(), sizeChars); 1055 break; 1056 // T __atomic_and_fetch_N(T *mem, T val, int order) 1057 // T __atomic_fetch_and_N(T *mem, T val, int order) 1058 case AtomicExpr::AO__atomic_and_fetch: 1059 PostOp = llvm::Instruction::And; 1060 LLVM_FALLTHROUGH; 1061 case AtomicExpr::AO__c11_atomic_fetch_and: 1062 case AtomicExpr::AO__opencl_atomic_fetch_and: 1063 case AtomicExpr::AO__atomic_fetch_and: 1064 LibCallName = "__atomic_fetch_and"; 1065 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1066 MemTy, E->getExprLoc(), sizeChars); 1067 break; 1068 // T __atomic_or_fetch_N(T *mem, T val, int order) 1069 // T __atomic_fetch_or_N(T *mem, T val, int order) 1070 case AtomicExpr::AO__atomic_or_fetch: 1071 PostOp = llvm::Instruction::Or; 1072 LLVM_FALLTHROUGH; 1073 case AtomicExpr::AO__c11_atomic_fetch_or: 1074 case AtomicExpr::AO__opencl_atomic_fetch_or: 1075 case AtomicExpr::AO__atomic_fetch_or: 1076 LibCallName = "__atomic_fetch_or"; 1077 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1078 MemTy, E->getExprLoc(), sizeChars); 1079 break; 1080 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1081 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1082 case AtomicExpr::AO__atomic_sub_fetch: 1083 PostOp = llvm::Instruction::Sub; 1084 LLVM_FALLTHROUGH; 1085 case AtomicExpr::AO__c11_atomic_fetch_sub: 1086 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1087 case AtomicExpr::AO__atomic_fetch_sub: 1088 LibCallName = "__atomic_fetch_sub"; 1089 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1090 LoweredMemTy, E->getExprLoc(), sizeChars); 1091 break; 1092 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1093 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1094 case AtomicExpr::AO__atomic_xor_fetch: 1095 PostOp = llvm::Instruction::Xor; 1096 LLVM_FALLTHROUGH; 1097 case AtomicExpr::AO__c11_atomic_fetch_xor: 1098 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1099 case AtomicExpr::AO__atomic_fetch_xor: 1100 LibCallName = "__atomic_fetch_xor"; 1101 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1102 MemTy, E->getExprLoc(), sizeChars); 1103 break; 1104 case AtomicExpr::AO__atomic_fetch_min: 1105 case AtomicExpr::AO__opencl_atomic_fetch_min: 1106 LibCallName = E->getValueType()->isSignedIntegerType() 1107 ? "__atomic_fetch_min" 1108 : "__atomic_fetch_umin"; 1109 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1110 LoweredMemTy, E->getExprLoc(), sizeChars); 1111 break; 1112 case AtomicExpr::AO__atomic_fetch_max: 1113 case AtomicExpr::AO__opencl_atomic_fetch_max: 1114 LibCallName = E->getValueType()->isSignedIntegerType() 1115 ? "__atomic_fetch_max" 1116 : "__atomic_fetch_umax"; 1117 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1118 LoweredMemTy, E->getExprLoc(), sizeChars); 1119 break; 1120 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1121 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1122 case AtomicExpr::AO__atomic_nand_fetch: 1123 PostOp = llvm::Instruction::And; // the NOT is special cased below 1124 LLVM_FALLTHROUGH; 1125 case AtomicExpr::AO__atomic_fetch_nand: 1126 LibCallName = "__atomic_fetch_nand"; 1127 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1128 MemTy, E->getExprLoc(), sizeChars); 1129 break; 1130 } 1131 1132 if (E->isOpenCL()) { 1133 LibCallName = std::string("__opencl") + 1134 StringRef(LibCallName).drop_front(1).str(); 1135 1136 } 1137 // Optimized functions have the size in their name. 1138 if (UseOptimizedLibcall) 1139 LibCallName += "_" + llvm::utostr(Size); 1140 // By default, assume we return a value of the atomic type. 1141 if (!HaveRetTy) { 1142 if (UseOptimizedLibcall) { 1143 // Value is returned directly. 1144 // The function returns an appropriately sized integer type. 1145 RetTy = getContext().getIntTypeForBitwidth( 1146 getContext().toBits(sizeChars), /*Signed=*/false); 1147 } else { 1148 // Value is returned through parameter before the order. 1149 RetTy = getContext().VoidTy; 1150 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 1151 getContext().VoidPtrTy); 1152 } 1153 } 1154 // order is always the last parameter 1155 Args.add(RValue::get(Order), 1156 getContext().IntTy); 1157 if (E->isOpenCL()) 1158 Args.add(RValue::get(Scope), getContext().IntTy); 1159 1160 // PostOp is only needed for the atomic_*_fetch operations, and 1161 // thus is only needed for and implemented in the 1162 // UseOptimizedLibcall codepath. 1163 assert(UseOptimizedLibcall || !PostOp); 1164 1165 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1166 // The value is returned directly from the libcall. 1167 if (E->isCmpXChg()) 1168 return Res; 1169 1170 // The value is returned directly for optimized libcalls but the expr 1171 // provided an out-param. 1172 if (UseOptimizedLibcall && Res.getScalarVal()) { 1173 llvm::Value *ResVal = Res.getScalarVal(); 1174 if (PostOp) { 1175 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1176 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1177 } 1178 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1179 ResVal = Builder.CreateNot(ResVal); 1180 1181 Builder.CreateStore( 1182 ResVal, 1183 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 1184 } 1185 1186 if (RValTy->isVoidType()) 1187 return RValue::get(nullptr); 1188 1189 return convertTempToRValue( 1190 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1191 RValTy, E->getExprLoc()); 1192 } 1193 1194 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1195 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1196 E->getOp() == AtomicExpr::AO__atomic_store || 1197 E->getOp() == AtomicExpr::AO__atomic_store_n; 1198 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1199 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1200 E->getOp() == AtomicExpr::AO__atomic_load || 1201 E->getOp() == AtomicExpr::AO__atomic_load_n; 1202 1203 if (isa<llvm::ConstantInt>(Order)) { 1204 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1205 // We should not ever get to a case where the ordering isn't a valid C ABI 1206 // value, but it's hard to enforce that in general. 1207 if (llvm::isValidAtomicOrderingCABI(ord)) 1208 switch ((llvm::AtomicOrderingCABI)ord) { 1209 case llvm::AtomicOrderingCABI::relaxed: 1210 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1211 llvm::AtomicOrdering::Monotonic, Scope); 1212 break; 1213 case llvm::AtomicOrderingCABI::consume: 1214 case llvm::AtomicOrderingCABI::acquire: 1215 if (IsStore) 1216 break; // Avoid crashing on code with undefined behavior 1217 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1218 llvm::AtomicOrdering::Acquire, Scope); 1219 break; 1220 case llvm::AtomicOrderingCABI::release: 1221 if (IsLoad) 1222 break; // Avoid crashing on code with undefined behavior 1223 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1224 llvm::AtomicOrdering::Release, Scope); 1225 break; 1226 case llvm::AtomicOrderingCABI::acq_rel: 1227 if (IsLoad || IsStore) 1228 break; // Avoid crashing on code with undefined behavior 1229 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1230 llvm::AtomicOrdering::AcquireRelease, Scope); 1231 break; 1232 case llvm::AtomicOrderingCABI::seq_cst: 1233 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1234 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1235 break; 1236 } 1237 if (RValTy->isVoidType()) 1238 return RValue::get(nullptr); 1239 1240 return convertTempToRValue( 1241 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1242 Dest.getAddressSpace())), 1243 RValTy, E->getExprLoc()); 1244 } 1245 1246 // Long case, when Order isn't obviously constant. 1247 1248 // Create all the relevant BB's 1249 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1250 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1251 *SeqCstBB = nullptr; 1252 MonotonicBB = createBasicBlock("monotonic", CurFn); 1253 if (!IsStore) 1254 AcquireBB = createBasicBlock("acquire", CurFn); 1255 if (!IsLoad) 1256 ReleaseBB = createBasicBlock("release", CurFn); 1257 if (!IsLoad && !IsStore) 1258 AcqRelBB = createBasicBlock("acqrel", CurFn); 1259 SeqCstBB = createBasicBlock("seqcst", CurFn); 1260 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1261 1262 // Create the switch for the split 1263 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1264 // doesn't matter unless someone is crazy enough to use something that 1265 // doesn't fold to a constant for the ordering. 1266 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1267 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1268 1269 // Emit all the different atomics 1270 Builder.SetInsertPoint(MonotonicBB); 1271 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1272 llvm::AtomicOrdering::Monotonic, Scope); 1273 Builder.CreateBr(ContBB); 1274 if (!IsStore) { 1275 Builder.SetInsertPoint(AcquireBB); 1276 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1277 llvm::AtomicOrdering::Acquire, Scope); 1278 Builder.CreateBr(ContBB); 1279 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1280 AcquireBB); 1281 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1282 AcquireBB); 1283 } 1284 if (!IsLoad) { 1285 Builder.SetInsertPoint(ReleaseBB); 1286 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1287 llvm::AtomicOrdering::Release, Scope); 1288 Builder.CreateBr(ContBB); 1289 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1290 ReleaseBB); 1291 } 1292 if (!IsLoad && !IsStore) { 1293 Builder.SetInsertPoint(AcqRelBB); 1294 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1295 llvm::AtomicOrdering::AcquireRelease, Scope); 1296 Builder.CreateBr(ContBB); 1297 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1298 AcqRelBB); 1299 } 1300 Builder.SetInsertPoint(SeqCstBB); 1301 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1302 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1303 Builder.CreateBr(ContBB); 1304 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1305 SeqCstBB); 1306 1307 // Cleanup and return 1308 Builder.SetInsertPoint(ContBB); 1309 if (RValTy->isVoidType()) 1310 return RValue::get(nullptr); 1311 1312 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1313 return convertTempToRValue( 1314 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1315 Dest.getAddressSpace())), 1316 RValTy, E->getExprLoc()); 1317 } 1318 1319 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1320 unsigned addrspace = 1321 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 1322 llvm::IntegerType *ty = 1323 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1324 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 1325 } 1326 1327 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1328 llvm::Type *Ty = Addr.getElementType(); 1329 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1330 if (SourceSizeInBits != AtomicSizeInBits) { 1331 Address Tmp = CreateTempAlloca(); 1332 CGF.Builder.CreateMemCpy(Tmp, Addr, 1333 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1334 Addr = Tmp; 1335 } 1336 1337 return emitCastToAtomicIntPointer(Addr); 1338 } 1339 1340 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1341 AggValueSlot resultSlot, 1342 SourceLocation loc, 1343 bool asValue) const { 1344 if (LVal.isSimple()) { 1345 if (EvaluationKind == TEK_Aggregate) 1346 return resultSlot.asRValue(); 1347 1348 // Drill into the padding structure if we have one. 1349 if (hasPadding()) 1350 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 1351 1352 // Otherwise, just convert the temporary to an r-value using the 1353 // normal conversion routine. 1354 return CGF.convertTempToRValue(addr, getValueType(), loc); 1355 } 1356 if (!asValue) 1357 // Get RValue from temp memory as atomic for non-simple lvalues 1358 return RValue::get(CGF.Builder.CreateLoad(addr)); 1359 if (LVal.isBitField()) 1360 return CGF.EmitLoadOfBitfieldLValue( 1361 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1362 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1363 if (LVal.isVectorElt()) 1364 return CGF.EmitLoadOfLValue( 1365 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1366 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1367 assert(LVal.isExtVectorElt()); 1368 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1369 addr, LVal.getExtVectorElts(), LVal.getType(), 1370 LVal.getBaseInfo(), TBAAAccessInfo())); 1371 } 1372 1373 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1374 AggValueSlot ResultSlot, 1375 SourceLocation Loc, 1376 bool AsValue) const { 1377 // Try not to in some easy cases. 1378 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1379 if (getEvaluationKind() == TEK_Scalar && 1380 (((!LVal.isBitField() || 1381 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1382 !hasPadding()) || 1383 !AsValue)) { 1384 auto *ValTy = AsValue 1385 ? CGF.ConvertTypeForMem(ValueTy) 1386 : getAtomicAddress().getType()->getPointerElementType(); 1387 if (ValTy->isIntegerTy()) { 1388 assert(IntVal->getType() == ValTy && "Different integer types."); 1389 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1390 } else if (ValTy->isPointerTy()) 1391 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1392 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1393 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1394 } 1395 1396 // Create a temporary. This needs to be big enough to hold the 1397 // atomic integer. 1398 Address Temp = Address::invalid(); 1399 bool TempIsVolatile = false; 1400 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1401 assert(!ResultSlot.isIgnored()); 1402 Temp = ResultSlot.getAddress(); 1403 TempIsVolatile = ResultSlot.isVolatile(); 1404 } else { 1405 Temp = CreateTempAlloca(); 1406 } 1407 1408 // Slam the integer into the temporary. 1409 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1410 CGF.Builder.CreateStore(IntVal, CastTemp) 1411 ->setVolatile(TempIsVolatile); 1412 1413 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1414 } 1415 1416 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1417 llvm::AtomicOrdering AO, bool) { 1418 // void __atomic_load(size_t size, void *mem, void *return, int order); 1419 CallArgList Args; 1420 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1421 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1422 CGF.getContext().VoidPtrTy); 1423 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1424 CGF.getContext().VoidPtrTy); 1425 Args.add( 1426 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1427 CGF.getContext().IntTy); 1428 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1429 } 1430 1431 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1432 bool IsVolatile) { 1433 // Okay, we're doing this natively. 1434 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1435 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1436 Load->setAtomic(AO); 1437 1438 // Other decoration. 1439 if (IsVolatile) 1440 Load->setVolatile(true); 1441 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1442 return Load; 1443 } 1444 1445 /// An LValue is a candidate for having its loads and stores be made atomic if 1446 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1447 /// performing such an operation can be performed without a libcall. 1448 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1449 if (!CGM.getCodeGenOpts().MSVolatile) return false; 1450 AtomicInfo AI(*this, LV); 1451 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1452 // An atomic is inline if we don't need to use a libcall. 1453 bool AtomicIsInline = !AI.shouldUseLibcall(); 1454 // MSVC doesn't seem to do this for types wider than a pointer. 1455 if (getContext().getTypeSize(LV.getType()) > 1456 getContext().getTypeSize(getContext().getIntPtrType())) 1457 return false; 1458 return IsVolatile && AtomicIsInline; 1459 } 1460 1461 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1462 AggValueSlot Slot) { 1463 llvm::AtomicOrdering AO; 1464 bool IsVolatile = LV.isVolatileQualified(); 1465 if (LV.getType()->isAtomicType()) { 1466 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1467 } else { 1468 AO = llvm::AtomicOrdering::Acquire; 1469 IsVolatile = true; 1470 } 1471 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1472 } 1473 1474 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1475 bool AsValue, llvm::AtomicOrdering AO, 1476 bool IsVolatile) { 1477 // Check whether we should use a library call. 1478 if (shouldUseLibcall()) { 1479 Address TempAddr = Address::invalid(); 1480 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1481 assert(getEvaluationKind() == TEK_Aggregate); 1482 TempAddr = ResultSlot.getAddress(); 1483 } else 1484 TempAddr = CreateTempAlloca(); 1485 1486 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1487 1488 // Okay, turn that back into the original value or whole atomic (for 1489 // non-simple lvalues) type. 1490 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1491 } 1492 1493 // Okay, we're doing this natively. 1494 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1495 1496 // If we're ignoring an aggregate return, don't do anything. 1497 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1498 return RValue::getAggregate(Address::invalid(), false); 1499 1500 // Okay, turn that back into the original value or atomic (for non-simple 1501 // lvalues) type. 1502 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1503 } 1504 1505 /// Emit a load from an l-value of atomic type. Note that the r-value 1506 /// we produce is an r-value of the atomic *value* type. 1507 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1508 llvm::AtomicOrdering AO, bool IsVolatile, 1509 AggValueSlot resultSlot) { 1510 AtomicInfo Atomics(*this, src); 1511 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1512 IsVolatile); 1513 } 1514 1515 /// Copy an r-value into memory as part of storing to an atomic type. 1516 /// This needs to create a bit-pattern suitable for atomic operations. 1517 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1518 assert(LVal.isSimple()); 1519 // If we have an r-value, the rvalue should be of the atomic type, 1520 // which means that the caller is responsible for having zeroed 1521 // any padding. Just do an aggregate copy of that type. 1522 if (rvalue.isAggregate()) { 1523 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1524 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1525 getAtomicType()); 1526 bool IsVolatile = rvalue.isVolatileQualified() || 1527 LVal.isVolatileQualified(); 1528 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1529 AggValueSlot::DoesNotOverlap, IsVolatile); 1530 return; 1531 } 1532 1533 // Okay, otherwise we're copying stuff. 1534 1535 // Zero out the buffer if necessary. 1536 emitMemSetZeroIfNecessary(); 1537 1538 // Drill past the padding if present. 1539 LValue TempLVal = projectValue(); 1540 1541 // Okay, store the rvalue in. 1542 if (rvalue.isScalar()) { 1543 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1544 } else { 1545 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1546 } 1547 } 1548 1549 1550 /// Materialize an r-value into memory for the purposes of storing it 1551 /// to an atomic type. 1552 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1553 // Aggregate r-values are already in memory, and EmitAtomicStore 1554 // requires them to be values of the atomic type. 1555 if (rvalue.isAggregate()) 1556 return rvalue.getAggregateAddress(); 1557 1558 // Otherwise, make a temporary and materialize into it. 1559 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1560 AtomicInfo Atomics(CGF, TempLV); 1561 Atomics.emitCopyIntoMemory(rvalue); 1562 return TempLV.getAddress(); 1563 } 1564 1565 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1566 // If we've got a scalar value of the right size, try to avoid going 1567 // through memory. 1568 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1569 llvm::Value *Value = RVal.getScalarVal(); 1570 if (isa<llvm::IntegerType>(Value->getType())) 1571 return CGF.EmitToMemory(Value, ValueTy); 1572 else { 1573 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1574 CGF.getLLVMContext(), 1575 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1576 if (isa<llvm::PointerType>(Value->getType())) 1577 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1578 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1579 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1580 } 1581 } 1582 // Otherwise, we need to go through memory. 1583 // Put the r-value in memory. 1584 Address Addr = materializeRValue(RVal); 1585 1586 // Cast the temporary to the atomic int type and pull a value out. 1587 Addr = emitCastToAtomicIntPointer(Addr); 1588 return CGF.Builder.CreateLoad(Addr); 1589 } 1590 1591 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1592 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1593 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1594 // Do the atomic store. 1595 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1596 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1597 ExpectedVal, DesiredVal, 1598 Success, Failure); 1599 // Other decoration. 1600 Inst->setVolatile(LVal.isVolatileQualified()); 1601 Inst->setWeak(IsWeak); 1602 1603 // Okay, turn that back into the original value type. 1604 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1605 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1606 return std::make_pair(PreviousVal, SuccessFailureVal); 1607 } 1608 1609 llvm::Value * 1610 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1611 llvm::Value *DesiredAddr, 1612 llvm::AtomicOrdering Success, 1613 llvm::AtomicOrdering Failure) { 1614 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1615 // void *desired, int success, int failure); 1616 CallArgList Args; 1617 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1618 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1619 CGF.getContext().VoidPtrTy); 1620 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1621 CGF.getContext().VoidPtrTy); 1622 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1623 CGF.getContext().VoidPtrTy); 1624 Args.add(RValue::get( 1625 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1626 CGF.getContext().IntTy); 1627 Args.add(RValue::get( 1628 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1629 CGF.getContext().IntTy); 1630 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1631 CGF.getContext().BoolTy, Args); 1632 1633 return SuccessFailureRVal.getScalarVal(); 1634 } 1635 1636 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1637 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1638 llvm::AtomicOrdering Failure, bool IsWeak) { 1639 if (isStrongerThan(Failure, Success)) 1640 // Don't assert on undefined behavior "failure argument shall be no stronger 1641 // than the success argument". 1642 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 1643 1644 // Check whether we should use a library call. 1645 if (shouldUseLibcall()) { 1646 // Produce a source address. 1647 Address ExpectedAddr = materializeRValue(Expected); 1648 Address DesiredAddr = materializeRValue(Desired); 1649 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1650 DesiredAddr.getPointer(), 1651 Success, Failure); 1652 return std::make_pair( 1653 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1654 SourceLocation(), /*AsValue=*/false), 1655 Res); 1656 } 1657 1658 // If we've got a scalar value of the right size, try to avoid going 1659 // through memory. 1660 auto *ExpectedVal = convertRValueToInt(Expected); 1661 auto *DesiredVal = convertRValueToInt(Desired); 1662 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1663 Failure, IsWeak); 1664 return std::make_pair( 1665 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1666 SourceLocation(), /*AsValue=*/false), 1667 Res.second); 1668 } 1669 1670 static void 1671 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1672 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1673 Address DesiredAddr) { 1674 RValue UpRVal; 1675 LValue AtomicLVal = Atomics.getAtomicLValue(); 1676 LValue DesiredLVal; 1677 if (AtomicLVal.isSimple()) { 1678 UpRVal = OldRVal; 1679 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1680 } else { 1681 // Build new lvalue for temp address 1682 Address Ptr = Atomics.materializeRValue(OldRVal); 1683 LValue UpdateLVal; 1684 if (AtomicLVal.isBitField()) { 1685 UpdateLVal = 1686 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1687 AtomicLVal.getType(), 1688 AtomicLVal.getBaseInfo(), 1689 AtomicLVal.getTBAAInfo()); 1690 DesiredLVal = 1691 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1692 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1693 AtomicLVal.getTBAAInfo()); 1694 } else if (AtomicLVal.isVectorElt()) { 1695 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1696 AtomicLVal.getType(), 1697 AtomicLVal.getBaseInfo(), 1698 AtomicLVal.getTBAAInfo()); 1699 DesiredLVal = LValue::MakeVectorElt( 1700 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1701 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1702 } else { 1703 assert(AtomicLVal.isExtVectorElt()); 1704 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1705 AtomicLVal.getType(), 1706 AtomicLVal.getBaseInfo(), 1707 AtomicLVal.getTBAAInfo()); 1708 DesiredLVal = LValue::MakeExtVectorElt( 1709 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1710 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1711 } 1712 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1713 } 1714 // Store new value in the corresponding memory area 1715 RValue NewRVal = UpdateOp(UpRVal); 1716 if (NewRVal.isScalar()) { 1717 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1718 } else { 1719 assert(NewRVal.isComplex()); 1720 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1721 /*isInit=*/false); 1722 } 1723 } 1724 1725 void AtomicInfo::EmitAtomicUpdateLibcall( 1726 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1727 bool IsVolatile) { 1728 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1729 1730 Address ExpectedAddr = CreateTempAlloca(); 1731 1732 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1733 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1734 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1735 CGF.EmitBlock(ContBB); 1736 Address DesiredAddr = CreateTempAlloca(); 1737 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1738 requiresMemSetZero(getAtomicAddress().getElementType())) { 1739 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1740 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1741 } 1742 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1743 AggValueSlot::ignored(), 1744 SourceLocation(), /*AsValue=*/false); 1745 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1746 auto *Res = 1747 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1748 DesiredAddr.getPointer(), 1749 AO, Failure); 1750 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1751 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1752 } 1753 1754 void AtomicInfo::EmitAtomicUpdateOp( 1755 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1756 bool IsVolatile) { 1757 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1758 1759 // Do the atomic load. 1760 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1761 // For non-simple lvalues perform compare-and-swap procedure. 1762 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1763 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1764 auto *CurBB = CGF.Builder.GetInsertBlock(); 1765 CGF.EmitBlock(ContBB); 1766 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1767 /*NumReservedValues=*/2); 1768 PHI->addIncoming(OldVal, CurBB); 1769 Address NewAtomicAddr = CreateTempAlloca(); 1770 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1771 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1772 requiresMemSetZero(getAtomicAddress().getElementType())) { 1773 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1774 } 1775 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1776 SourceLocation(), /*AsValue=*/false); 1777 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1778 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1779 // Try to write new value using cmpxchg operation 1780 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1781 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1782 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1783 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1784 } 1785 1786 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1787 RValue UpdateRVal, Address DesiredAddr) { 1788 LValue AtomicLVal = Atomics.getAtomicLValue(); 1789 LValue DesiredLVal; 1790 // Build new lvalue for temp address 1791 if (AtomicLVal.isBitField()) { 1792 DesiredLVal = 1793 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1794 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1795 AtomicLVal.getTBAAInfo()); 1796 } else if (AtomicLVal.isVectorElt()) { 1797 DesiredLVal = 1798 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1799 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1800 AtomicLVal.getTBAAInfo()); 1801 } else { 1802 assert(AtomicLVal.isExtVectorElt()); 1803 DesiredLVal = LValue::MakeExtVectorElt( 1804 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1805 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1806 } 1807 // Store new value in the corresponding memory area 1808 assert(UpdateRVal.isScalar()); 1809 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1810 } 1811 1812 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1813 RValue UpdateRVal, bool IsVolatile) { 1814 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1815 1816 Address ExpectedAddr = CreateTempAlloca(); 1817 1818 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1819 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1820 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1821 CGF.EmitBlock(ContBB); 1822 Address DesiredAddr = CreateTempAlloca(); 1823 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1824 requiresMemSetZero(getAtomicAddress().getElementType())) { 1825 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1826 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1827 } 1828 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1829 auto *Res = 1830 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1831 DesiredAddr.getPointer(), 1832 AO, Failure); 1833 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1834 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1835 } 1836 1837 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1838 bool IsVolatile) { 1839 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1840 1841 // Do the atomic load. 1842 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1843 // For non-simple lvalues perform compare-and-swap procedure. 1844 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1845 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1846 auto *CurBB = CGF.Builder.GetInsertBlock(); 1847 CGF.EmitBlock(ContBB); 1848 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1849 /*NumReservedValues=*/2); 1850 PHI->addIncoming(OldVal, CurBB); 1851 Address NewAtomicAddr = CreateTempAlloca(); 1852 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1853 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1854 requiresMemSetZero(getAtomicAddress().getElementType())) { 1855 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1856 } 1857 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1858 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1859 // Try to write new value using cmpxchg operation 1860 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1861 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1862 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1863 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1864 } 1865 1866 void AtomicInfo::EmitAtomicUpdate( 1867 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1868 bool IsVolatile) { 1869 if (shouldUseLibcall()) { 1870 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1871 } else { 1872 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1873 } 1874 } 1875 1876 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1877 bool IsVolatile) { 1878 if (shouldUseLibcall()) { 1879 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1880 } else { 1881 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1882 } 1883 } 1884 1885 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1886 bool isInit) { 1887 bool IsVolatile = lvalue.isVolatileQualified(); 1888 llvm::AtomicOrdering AO; 1889 if (lvalue.getType()->isAtomicType()) { 1890 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1891 } else { 1892 AO = llvm::AtomicOrdering::Release; 1893 IsVolatile = true; 1894 } 1895 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1896 } 1897 1898 /// Emit a store to an l-value of atomic type. 1899 /// 1900 /// Note that the r-value is expected to be an r-value *of the atomic 1901 /// type*; this means that for aggregate r-values, it should include 1902 /// storage for any padding that was necessary. 1903 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1904 llvm::AtomicOrdering AO, bool IsVolatile, 1905 bool isInit) { 1906 // If this is an aggregate r-value, it should agree in type except 1907 // maybe for address-space qualification. 1908 assert(!rvalue.isAggregate() || 1909 rvalue.getAggregateAddress().getElementType() 1910 == dest.getAddress().getElementType()); 1911 1912 AtomicInfo atomics(*this, dest); 1913 LValue LVal = atomics.getAtomicLValue(); 1914 1915 // If this is an initialization, just put the value there normally. 1916 if (LVal.isSimple()) { 1917 if (isInit) { 1918 atomics.emitCopyIntoMemory(rvalue); 1919 return; 1920 } 1921 1922 // Check whether we should use a library call. 1923 if (atomics.shouldUseLibcall()) { 1924 // Produce a source address. 1925 Address srcAddr = atomics.materializeRValue(rvalue); 1926 1927 // void __atomic_store(size_t size, void *mem, void *val, int order) 1928 CallArgList args; 1929 args.add(RValue::get(atomics.getAtomicSizeValue()), 1930 getContext().getSizeType()); 1931 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 1932 getContext().VoidPtrTy); 1933 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 1934 getContext().VoidPtrTy); 1935 args.add( 1936 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 1937 getContext().IntTy); 1938 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 1939 return; 1940 } 1941 1942 // Okay, we're doing this natively. 1943 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 1944 1945 // Do the atomic store. 1946 Address addr = 1947 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 1948 intValue = Builder.CreateIntCast( 1949 intValue, addr.getElementType(), /*isSigned=*/false); 1950 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 1951 1952 // Initializations don't need to be atomic. 1953 if (!isInit) 1954 store->setAtomic(AO); 1955 1956 // Other decoration. 1957 if (IsVolatile) 1958 store->setVolatile(true); 1959 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 1960 return; 1961 } 1962 1963 // Emit simple atomic update operation. 1964 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 1965 } 1966 1967 /// Emit a compare-and-exchange op for atomic type. 1968 /// 1969 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 1970 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 1971 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 1972 AggValueSlot Slot) { 1973 // If this is an aggregate r-value, it should agree in type except 1974 // maybe for address-space qualification. 1975 assert(!Expected.isAggregate() || 1976 Expected.getAggregateAddress().getElementType() == 1977 Obj.getAddress().getElementType()); 1978 assert(!Desired.isAggregate() || 1979 Desired.getAggregateAddress().getElementType() == 1980 Obj.getAddress().getElementType()); 1981 AtomicInfo Atomics(*this, Obj); 1982 1983 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 1984 IsWeak); 1985 } 1986 1987 void CodeGenFunction::EmitAtomicUpdate( 1988 LValue LVal, llvm::AtomicOrdering AO, 1989 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 1990 AtomicInfo Atomics(*this, LVal); 1991 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 1992 } 1993 1994 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 1995 AtomicInfo atomics(*this, dest); 1996 1997 switch (atomics.getEvaluationKind()) { 1998 case TEK_Scalar: { 1999 llvm::Value *value = EmitScalarExpr(init); 2000 atomics.emitCopyIntoMemory(RValue::get(value)); 2001 return; 2002 } 2003 2004 case TEK_Complex: { 2005 ComplexPairTy value = EmitComplexExpr(init); 2006 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2007 return; 2008 } 2009 2010 case TEK_Aggregate: { 2011 // Fix up the destination if the initializer isn't an expression 2012 // of atomic type. 2013 bool Zeroed = false; 2014 if (!init->getType()->isAtomicType()) { 2015 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2016 dest = atomics.projectValue(); 2017 } 2018 2019 // Evaluate the expression directly into the destination. 2020 AggValueSlot slot = AggValueSlot::forLValue(dest, 2021 AggValueSlot::IsNotDestructed, 2022 AggValueSlot::DoesNotNeedGCBarriers, 2023 AggValueSlot::IsNotAliased, 2024 AggValueSlot::DoesNotOverlap, 2025 Zeroed ? AggValueSlot::IsZeroed : 2026 AggValueSlot::IsNotZeroed); 2027 2028 EmitAggExpr(init, slot); 2029 return; 2030 } 2031 } 2032 llvm_unreachable("bad evaluation kind"); 2033 } 2034