1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the code for emitting atomic operations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CGCall.h" 15 #include "CGRecordLayout.h" 16 #include "CodeGenFunction.h" 17 #include "CodeGenModule.h" 18 #include "TargetInfo.h" 19 #include "clang/AST/ASTContext.h" 20 #include "clang/CodeGen/CGFunctionInfo.h" 21 #include "llvm/IR/DataLayout.h" 22 #include "llvm/IR/Intrinsics.h" 23 #include "llvm/IR/Operator.h" 24 25 using namespace clang; 26 using namespace CodeGen; 27 28 namespace { 29 class AtomicInfo { 30 CodeGenFunction &CGF; 31 QualType AtomicTy; 32 QualType ValueTy; 33 uint64_t AtomicSizeInBits; 34 uint64_t ValueSizeInBits; 35 CharUnits AtomicAlign; 36 CharUnits ValueAlign; 37 CharUnits LValueAlign; 38 TypeEvaluationKind EvaluationKind; 39 bool UseLibcall; 40 LValue LVal; 41 CGBitFieldInfo BFI; 42 public: 43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 45 EvaluationKind(TEK_Scalar), UseLibcall(true) { 46 assert(!lvalue.isGlobalReg()); 47 ASTContext &C = CGF.getContext(); 48 if (lvalue.isSimple()) { 49 AtomicTy = lvalue.getType(); 50 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 51 ValueTy = ATy->getValueType(); 52 else 53 ValueTy = AtomicTy; 54 EvaluationKind = CGF.getEvaluationKind(ValueTy); 55 56 uint64_t ValueAlignInBits; 57 uint64_t AtomicAlignInBits; 58 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 59 ValueSizeInBits = ValueTI.Width; 60 ValueAlignInBits = ValueTI.Align; 61 62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 63 AtomicSizeInBits = AtomicTI.Width; 64 AtomicAlignInBits = AtomicTI.Align; 65 66 assert(ValueSizeInBits <= AtomicSizeInBits); 67 assert(ValueAlignInBits <= AtomicAlignInBits); 68 69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 71 if (lvalue.getAlignment().isZero()) 72 lvalue.setAlignment(AtomicAlign); 73 74 LVal = lvalue; 75 } else if (lvalue.isBitField()) { 76 ValueTy = lvalue.getType(); 77 ValueSizeInBits = C.getTypeSize(ValueTy); 78 auto &OrigBFI = lvalue.getBitFieldInfo(); 79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 80 AtomicSizeInBits = C.toBits( 81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 82 .alignTo(lvalue.getAlignment())); 83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 84 auto OffsetInChars = 85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 86 lvalue.getAlignment(); 87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 88 VoidPtrAddr, OffsetInChars.getQuantity()); 89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 90 VoidPtrAddr, 91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 92 "atomic_bitfield_base"); 93 BFI = OrigBFI; 94 BFI.Offset = Offset; 95 BFI.StorageSize = AtomicSizeInBits; 96 BFI.StorageOffset += OffsetInChars; 97 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 98 BFI, lvalue.getType(), 99 lvalue.getBaseInfo()); 100 LVal.setTBAAInfo(lvalue.getTBAAInfo()); 101 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 102 if (AtomicTy.isNull()) { 103 llvm::APInt Size( 104 /*numBits=*/32, 105 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 106 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, 107 /*IndexTypeQuals=*/0); 108 } 109 AtomicAlign = ValueAlign = lvalue.getAlignment(); 110 } else if (lvalue.isVectorElt()) { 111 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); 112 ValueSizeInBits = C.getTypeSize(ValueTy); 113 AtomicTy = lvalue.getType(); 114 AtomicSizeInBits = C.getTypeSize(AtomicTy); 115 AtomicAlign = ValueAlign = lvalue.getAlignment(); 116 LVal = lvalue; 117 } else { 118 assert(lvalue.isExtVectorElt()); 119 ValueTy = lvalue.getType(); 120 ValueSizeInBits = C.getTypeSize(ValueTy); 121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 122 lvalue.getType(), lvalue.getExtVectorAddress() 123 .getElementType()->getVectorNumElements()); 124 AtomicSizeInBits = C.getTypeSize(AtomicTy); 125 AtomicAlign = ValueAlign = lvalue.getAlignment(); 126 LVal = lvalue; 127 } 128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 129 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 130 } 131 132 QualType getAtomicType() const { return AtomicTy; } 133 QualType getValueType() const { return ValueTy; } 134 CharUnits getAtomicAlignment() const { return AtomicAlign; } 135 CharUnits getValueAlignment() const { return ValueAlign; } 136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 139 bool shouldUseLibcall() const { return UseLibcall; } 140 const LValue &getAtomicLValue() const { return LVal; } 141 llvm::Value *getAtomicPointer() const { 142 if (LVal.isSimple()) 143 return LVal.getPointer(); 144 else if (LVal.isBitField()) 145 return LVal.getBitFieldPointer(); 146 else if (LVal.isVectorElt()) 147 return LVal.getVectorPointer(); 148 assert(LVal.isExtVectorElt()); 149 return LVal.getExtVectorPointer(); 150 } 151 Address getAtomicAddress() const { 152 return Address(getAtomicPointer(), getAtomicAlignment()); 153 } 154 155 Address getAtomicAddressAsAtomicIntPointer() const { 156 return emitCastToAtomicIntPointer(getAtomicAddress()); 157 } 158 159 /// Is the atomic size larger than the underlying value type? 160 /// 161 /// Note that the absence of padding does not mean that atomic 162 /// objects are completely interchangeable with non-atomic 163 /// objects: we might have promoted the alignment of a type 164 /// without making it bigger. 165 bool hasPadding() const { 166 return (ValueSizeInBits != AtomicSizeInBits); 167 } 168 169 bool emitMemSetZeroIfNecessary() const; 170 171 llvm::Value *getAtomicSizeValue() const { 172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 173 return CGF.CGM.getSize(size); 174 } 175 176 /// Cast the given pointer to an integer pointer suitable for atomic 177 /// operations if the source. 178 Address emitCastToAtomicIntPointer(Address Addr) const; 179 180 /// If Addr is compatible with the iN that will be used for an atomic 181 /// operation, bitcast it. Otherwise, create a temporary that is suitable 182 /// and copy the value across. 183 Address convertToAtomicIntPointer(Address Addr) const; 184 185 /// Turn an atomic-layout object into an r-value. 186 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 187 SourceLocation loc, bool AsValue) const; 188 189 /// \brief Converts a rvalue to integer value. 190 llvm::Value *convertRValueToInt(RValue RVal) const; 191 192 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 193 AggValueSlot ResultSlot, 194 SourceLocation Loc, bool AsValue) const; 195 196 /// Copy an atomic r-value into atomic-layout memory. 197 void emitCopyIntoMemory(RValue rvalue) const; 198 199 /// Project an l-value down to the value field. 200 LValue projectValue() const { 201 assert(LVal.isSimple()); 202 Address addr = getAtomicAddress(); 203 if (hasPadding()) 204 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 205 206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 207 LVal.getBaseInfo(), LVal.getTBAAInfo()); 208 } 209 210 /// \brief Emits atomic load. 211 /// \returns Loaded value. 212 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 213 bool AsValue, llvm::AtomicOrdering AO, 214 bool IsVolatile); 215 216 /// \brief Emits atomic compare-and-exchange sequence. 217 /// \param Expected Expected value. 218 /// \param Desired Desired value. 219 /// \param Success Atomic ordering for success operation. 220 /// \param Failure Atomic ordering for failed operation. 221 /// \param IsWeak true if atomic operation is weak, false otherwise. 222 /// \returns Pair of values: previous value from storage (value type) and 223 /// boolean flag (i1 type) with true if success and false otherwise. 224 std::pair<RValue, llvm::Value *> 225 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 226 llvm::AtomicOrdering Success = 227 llvm::AtomicOrdering::SequentiallyConsistent, 228 llvm::AtomicOrdering Failure = 229 llvm::AtomicOrdering::SequentiallyConsistent, 230 bool IsWeak = false); 231 232 /// \brief Emits atomic update. 233 /// \param AO Atomic ordering. 234 /// \param UpdateOp Update operation for the current lvalue. 235 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 236 const llvm::function_ref<RValue(RValue)> &UpdateOp, 237 bool IsVolatile); 238 /// \brief Emits atomic update. 239 /// \param AO Atomic ordering. 240 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 241 bool IsVolatile); 242 243 /// Materialize an atomic r-value in atomic-layout memory. 244 Address materializeRValue(RValue rvalue) const; 245 246 /// \brief Creates temp alloca for intermediate operations on atomic value. 247 Address CreateTempAlloca() const; 248 private: 249 bool requiresMemSetZero(llvm::Type *type) const; 250 251 252 /// \brief Emits atomic load as a libcall. 253 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 254 llvm::AtomicOrdering AO, bool IsVolatile); 255 /// \brief Emits atomic load as LLVM instruction. 256 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 257 /// \brief Emits atomic compare-and-exchange op as a libcall. 258 llvm::Value *EmitAtomicCompareExchangeLibcall( 259 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 260 llvm::AtomicOrdering Success = 261 llvm::AtomicOrdering::SequentiallyConsistent, 262 llvm::AtomicOrdering Failure = 263 llvm::AtomicOrdering::SequentiallyConsistent); 264 /// \brief Emits atomic compare-and-exchange op as LLVM instruction. 265 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 266 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 267 llvm::AtomicOrdering Success = 268 llvm::AtomicOrdering::SequentiallyConsistent, 269 llvm::AtomicOrdering Failure = 270 llvm::AtomicOrdering::SequentiallyConsistent, 271 bool IsWeak = false); 272 /// \brief Emit atomic update as libcalls. 273 void 274 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 275 const llvm::function_ref<RValue(RValue)> &UpdateOp, 276 bool IsVolatile); 277 /// \brief Emit atomic update as LLVM instructions. 278 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 279 const llvm::function_ref<RValue(RValue)> &UpdateOp, 280 bool IsVolatile); 281 /// \brief Emit atomic update as libcalls. 282 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 283 bool IsVolatile); 284 /// \brief Emit atomic update as LLVM instructions. 285 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 286 bool IsVolatile); 287 }; 288 } 289 290 Address AtomicInfo::CreateTempAlloca() const { 291 Address TempAlloca = CGF.CreateMemTemp( 292 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 293 : AtomicTy, 294 getAtomicAlignment(), 295 "atomic-temp"); 296 // Cast to pointer to value type for bitfields. 297 if (LVal.isBitField()) 298 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 299 TempAlloca, getAtomicAddress().getType()); 300 return TempAlloca; 301 } 302 303 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 304 StringRef fnName, 305 QualType resultType, 306 CallArgList &args) { 307 const CGFunctionInfo &fnInfo = 308 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 309 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 310 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 311 auto callee = CGCallee::forDirect(fn); 312 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 313 } 314 315 /// Does a store of the given IR type modify the full expected width? 316 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 317 uint64_t expectedSize) { 318 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 319 } 320 321 /// Does the atomic type require memsetting to zero before initialization? 322 /// 323 /// The IR type is provided as a way of making certain queries faster. 324 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 325 // If the atomic type has size padding, we definitely need a memset. 326 if (hasPadding()) return true; 327 328 // Otherwise, do some simple heuristics to try to avoid it: 329 switch (getEvaluationKind()) { 330 // For scalars and complexes, check whether the store size of the 331 // type uses the full size. 332 case TEK_Scalar: 333 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 334 case TEK_Complex: 335 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 336 AtomicSizeInBits / 2); 337 338 // Padding in structs has an undefined bit pattern. User beware. 339 case TEK_Aggregate: 340 return false; 341 } 342 llvm_unreachable("bad evaluation kind"); 343 } 344 345 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 346 assert(LVal.isSimple()); 347 llvm::Value *addr = LVal.getPointer(); 348 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 349 return false; 350 351 CGF.Builder.CreateMemSet( 352 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 353 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 354 LVal.getAlignment().getQuantity()); 355 return true; 356 } 357 358 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 359 Address Dest, Address Ptr, 360 Address Val1, Address Val2, 361 uint64_t Size, 362 llvm::AtomicOrdering SuccessOrder, 363 llvm::AtomicOrdering FailureOrder, 364 llvm::SyncScope::ID Scope) { 365 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 366 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 367 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 368 369 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 370 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 371 Scope); 372 Pair->setVolatile(E->isVolatile()); 373 Pair->setWeak(IsWeak); 374 375 // Cmp holds the result of the compare-exchange operation: true on success, 376 // false on failure. 377 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 378 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 379 380 // This basic block is used to hold the store instruction if the operation 381 // failed. 382 llvm::BasicBlock *StoreExpectedBB = 383 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 384 385 // This basic block is the exit point of the operation, we should end up 386 // here regardless of whether or not the operation succeeded. 387 llvm::BasicBlock *ContinueBB = 388 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 389 390 // Update Expected if Expected isn't equal to Old, otherwise branch to the 391 // exit point. 392 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 393 394 CGF.Builder.SetInsertPoint(StoreExpectedBB); 395 // Update the memory at Expected with Old's value. 396 CGF.Builder.CreateStore(Old, Val1); 397 // Finally, branch to the exit point. 398 CGF.Builder.CreateBr(ContinueBB); 399 400 CGF.Builder.SetInsertPoint(ContinueBB); 401 // Update the memory at Dest with Cmp's value. 402 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 403 } 404 405 /// Given an ordering required on success, emit all possible cmpxchg 406 /// instructions to cope with the provided (but possibly only dynamically known) 407 /// FailureOrder. 408 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 409 bool IsWeak, Address Dest, Address Ptr, 410 Address Val1, Address Val2, 411 llvm::Value *FailureOrderVal, 412 uint64_t Size, 413 llvm::AtomicOrdering SuccessOrder, 414 llvm::SyncScope::ID Scope) { 415 llvm::AtomicOrdering FailureOrder; 416 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 417 auto FOS = FO->getSExtValue(); 418 if (!llvm::isValidAtomicOrderingCABI(FOS)) 419 FailureOrder = llvm::AtomicOrdering::Monotonic; 420 else 421 switch ((llvm::AtomicOrderingCABI)FOS) { 422 case llvm::AtomicOrderingCABI::relaxed: 423 case llvm::AtomicOrderingCABI::release: 424 case llvm::AtomicOrderingCABI::acq_rel: 425 FailureOrder = llvm::AtomicOrdering::Monotonic; 426 break; 427 case llvm::AtomicOrderingCABI::consume: 428 case llvm::AtomicOrderingCABI::acquire: 429 FailureOrder = llvm::AtomicOrdering::Acquire; 430 break; 431 case llvm::AtomicOrderingCABI::seq_cst: 432 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 433 break; 434 } 435 if (isStrongerThan(FailureOrder, SuccessOrder)) { 436 // Don't assert on undefined behavior "failure argument shall be no 437 // stronger than the success argument". 438 FailureOrder = 439 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 440 } 441 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 442 FailureOrder, Scope); 443 return; 444 } 445 446 // Create all the relevant BB's 447 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 448 *SeqCstBB = nullptr; 449 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 450 if (SuccessOrder != llvm::AtomicOrdering::Monotonic && 451 SuccessOrder != llvm::AtomicOrdering::Release) 452 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 453 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 454 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 455 456 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 457 458 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 459 460 // Emit all the different atomics 461 462 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 463 // doesn't matter unless someone is crazy enough to use something that 464 // doesn't fold to a constant for the ordering. 465 CGF.Builder.SetInsertPoint(MonotonicBB); 466 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 467 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 468 CGF.Builder.CreateBr(ContBB); 469 470 if (AcquireBB) { 471 CGF.Builder.SetInsertPoint(AcquireBB); 472 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 473 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope); 474 CGF.Builder.CreateBr(ContBB); 475 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 476 AcquireBB); 477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 478 AcquireBB); 479 } 480 if (SeqCstBB) { 481 CGF.Builder.SetInsertPoint(SeqCstBB); 482 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 483 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 484 CGF.Builder.CreateBr(ContBB); 485 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 486 SeqCstBB); 487 } 488 489 CGF.Builder.SetInsertPoint(ContBB); 490 } 491 492 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 493 Address Ptr, Address Val1, Address Val2, 494 llvm::Value *IsWeak, llvm::Value *FailureOrder, 495 uint64_t Size, llvm::AtomicOrdering Order, 496 llvm::SyncScope::ID Scope) { 497 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 498 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 499 500 switch (E->getOp()) { 501 case AtomicExpr::AO__c11_atomic_init: 502 case AtomicExpr::AO__opencl_atomic_init: 503 llvm_unreachable("Already handled!"); 504 505 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 506 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 507 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 508 FailureOrder, Size, Order, Scope); 509 return; 510 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 511 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 512 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 513 FailureOrder, Size, Order, Scope); 514 return; 515 case AtomicExpr::AO__atomic_compare_exchange: 516 case AtomicExpr::AO__atomic_compare_exchange_n: { 517 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 518 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 519 Val1, Val2, FailureOrder, Size, Order, Scope); 520 } else { 521 // Create all the relevant BB's 522 llvm::BasicBlock *StrongBB = 523 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 524 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 525 llvm::BasicBlock *ContBB = 526 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 527 528 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 529 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 530 531 CGF.Builder.SetInsertPoint(StrongBB); 532 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 533 FailureOrder, Size, Order, Scope); 534 CGF.Builder.CreateBr(ContBB); 535 536 CGF.Builder.SetInsertPoint(WeakBB); 537 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 538 FailureOrder, Size, Order, Scope); 539 CGF.Builder.CreateBr(ContBB); 540 541 CGF.Builder.SetInsertPoint(ContBB); 542 } 543 return; 544 } 545 case AtomicExpr::AO__c11_atomic_load: 546 case AtomicExpr::AO__opencl_atomic_load: 547 case AtomicExpr::AO__atomic_load_n: 548 case AtomicExpr::AO__atomic_load: { 549 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 550 Load->setAtomic(Order, Scope); 551 Load->setVolatile(E->isVolatile()); 552 CGF.Builder.CreateStore(Load, Dest); 553 return; 554 } 555 556 case AtomicExpr::AO__c11_atomic_store: 557 case AtomicExpr::AO__opencl_atomic_store: 558 case AtomicExpr::AO__atomic_store: 559 case AtomicExpr::AO__atomic_store_n: { 560 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 561 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 562 Store->setAtomic(Order, Scope); 563 Store->setVolatile(E->isVolatile()); 564 return; 565 } 566 567 case AtomicExpr::AO__c11_atomic_exchange: 568 case AtomicExpr::AO__opencl_atomic_exchange: 569 case AtomicExpr::AO__atomic_exchange_n: 570 case AtomicExpr::AO__atomic_exchange: 571 Op = llvm::AtomicRMWInst::Xchg; 572 break; 573 574 case AtomicExpr::AO__atomic_add_fetch: 575 PostOp = llvm::Instruction::Add; 576 // Fall through. 577 case AtomicExpr::AO__c11_atomic_fetch_add: 578 case AtomicExpr::AO__opencl_atomic_fetch_add: 579 case AtomicExpr::AO__atomic_fetch_add: 580 Op = llvm::AtomicRMWInst::Add; 581 break; 582 583 case AtomicExpr::AO__atomic_sub_fetch: 584 PostOp = llvm::Instruction::Sub; 585 // Fall through. 586 case AtomicExpr::AO__c11_atomic_fetch_sub: 587 case AtomicExpr::AO__opencl_atomic_fetch_sub: 588 case AtomicExpr::AO__atomic_fetch_sub: 589 Op = llvm::AtomicRMWInst::Sub; 590 break; 591 592 case AtomicExpr::AO__opencl_atomic_fetch_min: 593 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min 594 : llvm::AtomicRMWInst::UMin; 595 break; 596 597 case AtomicExpr::AO__opencl_atomic_fetch_max: 598 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max 599 : llvm::AtomicRMWInst::UMax; 600 break; 601 602 case AtomicExpr::AO__atomic_and_fetch: 603 PostOp = llvm::Instruction::And; 604 // Fall through. 605 case AtomicExpr::AO__c11_atomic_fetch_and: 606 case AtomicExpr::AO__opencl_atomic_fetch_and: 607 case AtomicExpr::AO__atomic_fetch_and: 608 Op = llvm::AtomicRMWInst::And; 609 break; 610 611 case AtomicExpr::AO__atomic_or_fetch: 612 PostOp = llvm::Instruction::Or; 613 // Fall through. 614 case AtomicExpr::AO__c11_atomic_fetch_or: 615 case AtomicExpr::AO__opencl_atomic_fetch_or: 616 case AtomicExpr::AO__atomic_fetch_or: 617 Op = llvm::AtomicRMWInst::Or; 618 break; 619 620 case AtomicExpr::AO__atomic_xor_fetch: 621 PostOp = llvm::Instruction::Xor; 622 // Fall through. 623 case AtomicExpr::AO__c11_atomic_fetch_xor: 624 case AtomicExpr::AO__opencl_atomic_fetch_xor: 625 case AtomicExpr::AO__atomic_fetch_xor: 626 Op = llvm::AtomicRMWInst::Xor; 627 break; 628 629 case AtomicExpr::AO__atomic_nand_fetch: 630 PostOp = llvm::Instruction::And; // the NOT is special cased below 631 // Fall through. 632 case AtomicExpr::AO__atomic_fetch_nand: 633 Op = llvm::AtomicRMWInst::Nand; 634 break; 635 } 636 637 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 638 llvm::AtomicRMWInst *RMWI = 639 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 640 RMWI->setVolatile(E->isVolatile()); 641 642 // For __atomic_*_fetch operations, perform the operation again to 643 // determine the value which was written. 644 llvm::Value *Result = RMWI; 645 if (PostOp) 646 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 647 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 648 Result = CGF.Builder.CreateNot(Result); 649 CGF.Builder.CreateStore(Result, Dest); 650 } 651 652 // This function emits any expression (scalar, complex, or aggregate) 653 // into a temporary alloca. 654 static Address 655 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 656 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 657 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 658 /*Init*/ true); 659 return DeclPtr; 660 } 661 662 static void 663 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 664 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 665 SourceLocation Loc, CharUnits SizeInChars) { 666 if (UseOptimizedLibcall) { 667 // Load value and pass it to the function directly. 668 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 669 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 670 ValTy = 671 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 672 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 673 SizeInBits)->getPointerTo(); 674 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 675 Val = CGF.EmitLoadOfScalar(Ptr, false, 676 CGF.getContext().getPointerType(ValTy), 677 Loc); 678 // Coerce the value into an appropriately sized integer type. 679 Args.add(RValue::get(Val), ValTy); 680 } else { 681 // Non-optimized functions always take a reference. 682 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 683 CGF.getContext().VoidPtrTy); 684 } 685 } 686 687 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 688 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 689 QualType MemTy = AtomicTy; 690 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 691 MemTy = AT->getValueType(); 692 CharUnits sizeChars, alignChars; 693 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); 694 uint64_t Size = sizeChars.getQuantity(); 695 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 696 bool UseLibcall = (sizeChars != alignChars || 697 getContext().toBits(sizeChars) > MaxInlineWidthInBits); 698 699 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 700 701 Address Val1 = Address::invalid(); 702 Address Val2 = Address::invalid(); 703 Address Dest = Address::invalid(); 704 Address Ptr(EmitScalarExpr(E->getPtr()), alignChars); 705 706 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 707 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 708 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 709 EmitAtomicInit(E->getVal1(), lvalue); 710 return RValue::get(nullptr); 711 } 712 713 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 714 llvm::Value *Scope = EmitScalarExpr(E->getScope()); 715 716 switch (E->getOp()) { 717 case AtomicExpr::AO__c11_atomic_init: 718 case AtomicExpr::AO__opencl_atomic_init: 719 llvm_unreachable("Already handled above with EmitAtomicInit!"); 720 721 case AtomicExpr::AO__c11_atomic_load: 722 case AtomicExpr::AO__opencl_atomic_load: 723 case AtomicExpr::AO__atomic_load_n: 724 break; 725 726 case AtomicExpr::AO__atomic_load: 727 Dest = EmitPointerWithAlignment(E->getVal1()); 728 break; 729 730 case AtomicExpr::AO__atomic_store: 731 Val1 = EmitPointerWithAlignment(E->getVal1()); 732 break; 733 734 case AtomicExpr::AO__atomic_exchange: 735 Val1 = EmitPointerWithAlignment(E->getVal1()); 736 Dest = EmitPointerWithAlignment(E->getVal2()); 737 break; 738 739 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 740 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 741 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 742 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 743 case AtomicExpr::AO__atomic_compare_exchange_n: 744 case AtomicExpr::AO__atomic_compare_exchange: 745 Val1 = EmitPointerWithAlignment(E->getVal1()); 746 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 747 Val2 = EmitPointerWithAlignment(E->getVal2()); 748 else 749 Val2 = EmitValToTemp(*this, E->getVal2()); 750 OrderFail = EmitScalarExpr(E->getOrderFail()); 751 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 752 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 753 IsWeak = EmitScalarExpr(E->getWeak()); 754 break; 755 756 case AtomicExpr::AO__c11_atomic_fetch_add: 757 case AtomicExpr::AO__c11_atomic_fetch_sub: 758 case AtomicExpr::AO__opencl_atomic_fetch_add: 759 case AtomicExpr::AO__opencl_atomic_fetch_sub: 760 if (MemTy->isPointerType()) { 761 // For pointer arithmetic, we're required to do a bit of math: 762 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 763 // ... but only for the C11 builtins. The GNU builtins expect the 764 // user to multiply by sizeof(T). 765 QualType Val1Ty = E->getVal1()->getType(); 766 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 767 CharUnits PointeeIncAmt = 768 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 769 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 770 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 771 Val1 = Temp; 772 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 773 break; 774 } 775 // Fall through. 776 case AtomicExpr::AO__atomic_fetch_add: 777 case AtomicExpr::AO__atomic_fetch_sub: 778 case AtomicExpr::AO__atomic_add_fetch: 779 case AtomicExpr::AO__atomic_sub_fetch: 780 case AtomicExpr::AO__c11_atomic_store: 781 case AtomicExpr::AO__c11_atomic_exchange: 782 case AtomicExpr::AO__opencl_atomic_store: 783 case AtomicExpr::AO__opencl_atomic_exchange: 784 case AtomicExpr::AO__atomic_store_n: 785 case AtomicExpr::AO__atomic_exchange_n: 786 case AtomicExpr::AO__c11_atomic_fetch_and: 787 case AtomicExpr::AO__c11_atomic_fetch_or: 788 case AtomicExpr::AO__c11_atomic_fetch_xor: 789 case AtomicExpr::AO__opencl_atomic_fetch_and: 790 case AtomicExpr::AO__opencl_atomic_fetch_or: 791 case AtomicExpr::AO__opencl_atomic_fetch_xor: 792 case AtomicExpr::AO__opencl_atomic_fetch_min: 793 case AtomicExpr::AO__opencl_atomic_fetch_max: 794 case AtomicExpr::AO__atomic_fetch_and: 795 case AtomicExpr::AO__atomic_fetch_or: 796 case AtomicExpr::AO__atomic_fetch_xor: 797 case AtomicExpr::AO__atomic_fetch_nand: 798 case AtomicExpr::AO__atomic_and_fetch: 799 case AtomicExpr::AO__atomic_or_fetch: 800 case AtomicExpr::AO__atomic_xor_fetch: 801 case AtomicExpr::AO__atomic_nand_fetch: 802 Val1 = EmitValToTemp(*this, E->getVal1()); 803 break; 804 } 805 806 QualType RValTy = E->getType().getUnqualifiedType(); 807 808 // The inlined atomics only function on iN types, where N is a power of 2. We 809 // need to make sure (via temporaries if necessary) that all incoming values 810 // are compatible. 811 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 812 AtomicInfo Atomics(*this, AtomicVal); 813 814 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 815 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); 816 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); 817 if (Dest.isValid()) 818 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 819 else if (E->isCmpXChg()) 820 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 821 else if (!RValTy->isVoidType()) 822 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); 823 824 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 825 if (UseLibcall) { 826 bool UseOptimizedLibcall = false; 827 switch (E->getOp()) { 828 case AtomicExpr::AO__c11_atomic_init: 829 case AtomicExpr::AO__opencl_atomic_init: 830 llvm_unreachable("Already handled above with EmitAtomicInit!"); 831 832 case AtomicExpr::AO__c11_atomic_fetch_add: 833 case AtomicExpr::AO__opencl_atomic_fetch_add: 834 case AtomicExpr::AO__atomic_fetch_add: 835 case AtomicExpr::AO__c11_atomic_fetch_and: 836 case AtomicExpr::AO__opencl_atomic_fetch_and: 837 case AtomicExpr::AO__atomic_fetch_and: 838 case AtomicExpr::AO__c11_atomic_fetch_or: 839 case AtomicExpr::AO__opencl_atomic_fetch_or: 840 case AtomicExpr::AO__atomic_fetch_or: 841 case AtomicExpr::AO__atomic_fetch_nand: 842 case AtomicExpr::AO__c11_atomic_fetch_sub: 843 case AtomicExpr::AO__opencl_atomic_fetch_sub: 844 case AtomicExpr::AO__atomic_fetch_sub: 845 case AtomicExpr::AO__c11_atomic_fetch_xor: 846 case AtomicExpr::AO__opencl_atomic_fetch_xor: 847 case AtomicExpr::AO__opencl_atomic_fetch_min: 848 case AtomicExpr::AO__opencl_atomic_fetch_max: 849 case AtomicExpr::AO__atomic_fetch_xor: 850 case AtomicExpr::AO__atomic_add_fetch: 851 case AtomicExpr::AO__atomic_and_fetch: 852 case AtomicExpr::AO__atomic_nand_fetch: 853 case AtomicExpr::AO__atomic_or_fetch: 854 case AtomicExpr::AO__atomic_sub_fetch: 855 case AtomicExpr::AO__atomic_xor_fetch: 856 // For these, only library calls for certain sizes exist. 857 UseOptimizedLibcall = true; 858 break; 859 860 case AtomicExpr::AO__c11_atomic_load: 861 case AtomicExpr::AO__c11_atomic_store: 862 case AtomicExpr::AO__c11_atomic_exchange: 863 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 864 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 865 case AtomicExpr::AO__opencl_atomic_load: 866 case AtomicExpr::AO__opencl_atomic_store: 867 case AtomicExpr::AO__opencl_atomic_exchange: 868 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 869 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 870 case AtomicExpr::AO__atomic_load_n: 871 case AtomicExpr::AO__atomic_load: 872 case AtomicExpr::AO__atomic_store_n: 873 case AtomicExpr::AO__atomic_store: 874 case AtomicExpr::AO__atomic_exchange_n: 875 case AtomicExpr::AO__atomic_exchange: 876 case AtomicExpr::AO__atomic_compare_exchange_n: 877 case AtomicExpr::AO__atomic_compare_exchange: 878 // Only use optimized library calls for sizes for which they exist. 879 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 880 UseOptimizedLibcall = true; 881 break; 882 } 883 884 CallArgList Args; 885 if (!UseOptimizedLibcall) { 886 // For non-optimized library calls, the size is the first parameter 887 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 888 getContext().getSizeType()); 889 } 890 // Atomic address is the first or second parameter 891 // The OpenCL atomic library functions only accept pointer arguments to 892 // generic address space. 893 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 894 if (!E->isOpenCL()) 895 return V; 896 auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace(); 897 if (AS == LangAS::opencl_generic) 898 return V; 899 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 900 auto T = V->getType(); 901 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS); 902 903 return getTargetHooks().performAddrSpaceCast( 904 *this, V, AS, LangAS::opencl_generic, DestType, false); 905 }; 906 907 Args.add(RValue::get(CastToGenericAddrSpace( 908 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), 909 getContext().VoidPtrTy); 910 911 std::string LibCallName; 912 QualType LoweredMemTy = 913 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 914 QualType RetTy; 915 bool HaveRetTy = false; 916 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 917 switch (E->getOp()) { 918 case AtomicExpr::AO__c11_atomic_init: 919 case AtomicExpr::AO__opencl_atomic_init: 920 llvm_unreachable("Already handled!"); 921 922 // There is only one libcall for compare an exchange, because there is no 923 // optimisation benefit possible from a libcall version of a weak compare 924 // and exchange. 925 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 926 // void *desired, int success, int failure) 927 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 928 // int success, int failure) 929 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 930 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 931 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 932 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 933 case AtomicExpr::AO__atomic_compare_exchange: 934 case AtomicExpr::AO__atomic_compare_exchange_n: 935 LibCallName = "__atomic_compare_exchange"; 936 RetTy = getContext().BoolTy; 937 HaveRetTy = true; 938 Args.add( 939 RValue::get(CastToGenericAddrSpace( 940 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), 941 getContext().VoidPtrTy); 942 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 943 MemTy, E->getExprLoc(), sizeChars); 944 Args.add(RValue::get(Order), getContext().IntTy); 945 Order = OrderFail; 946 break; 947 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 948 // int order) 949 // T __atomic_exchange_N(T *mem, T val, int order) 950 case AtomicExpr::AO__c11_atomic_exchange: 951 case AtomicExpr::AO__opencl_atomic_exchange: 952 case AtomicExpr::AO__atomic_exchange_n: 953 case AtomicExpr::AO__atomic_exchange: 954 LibCallName = "__atomic_exchange"; 955 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 956 MemTy, E->getExprLoc(), sizeChars); 957 break; 958 // void __atomic_store(size_t size, void *mem, void *val, int order) 959 // void __atomic_store_N(T *mem, T val, int order) 960 case AtomicExpr::AO__c11_atomic_store: 961 case AtomicExpr::AO__opencl_atomic_store: 962 case AtomicExpr::AO__atomic_store: 963 case AtomicExpr::AO__atomic_store_n: 964 LibCallName = "__atomic_store"; 965 RetTy = getContext().VoidTy; 966 HaveRetTy = true; 967 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 968 MemTy, E->getExprLoc(), sizeChars); 969 break; 970 // void __atomic_load(size_t size, void *mem, void *return, int order) 971 // T __atomic_load_N(T *mem, int order) 972 case AtomicExpr::AO__c11_atomic_load: 973 case AtomicExpr::AO__opencl_atomic_load: 974 case AtomicExpr::AO__atomic_load: 975 case AtomicExpr::AO__atomic_load_n: 976 LibCallName = "__atomic_load"; 977 break; 978 // T __atomic_add_fetch_N(T *mem, T val, int order) 979 // T __atomic_fetch_add_N(T *mem, T val, int order) 980 case AtomicExpr::AO__atomic_add_fetch: 981 PostOp = llvm::Instruction::Add; 982 // Fall through. 983 case AtomicExpr::AO__c11_atomic_fetch_add: 984 case AtomicExpr::AO__opencl_atomic_fetch_add: 985 case AtomicExpr::AO__atomic_fetch_add: 986 LibCallName = "__atomic_fetch_add"; 987 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 988 LoweredMemTy, E->getExprLoc(), sizeChars); 989 break; 990 // T __atomic_and_fetch_N(T *mem, T val, int order) 991 // T __atomic_fetch_and_N(T *mem, T val, int order) 992 case AtomicExpr::AO__atomic_and_fetch: 993 PostOp = llvm::Instruction::And; 994 // Fall through. 995 case AtomicExpr::AO__c11_atomic_fetch_and: 996 case AtomicExpr::AO__opencl_atomic_fetch_and: 997 case AtomicExpr::AO__atomic_fetch_and: 998 LibCallName = "__atomic_fetch_and"; 999 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1000 MemTy, E->getExprLoc(), sizeChars); 1001 break; 1002 // T __atomic_or_fetch_N(T *mem, T val, int order) 1003 // T __atomic_fetch_or_N(T *mem, T val, int order) 1004 case AtomicExpr::AO__atomic_or_fetch: 1005 PostOp = llvm::Instruction::Or; 1006 // Fall through. 1007 case AtomicExpr::AO__c11_atomic_fetch_or: 1008 case AtomicExpr::AO__opencl_atomic_fetch_or: 1009 case AtomicExpr::AO__atomic_fetch_or: 1010 LibCallName = "__atomic_fetch_or"; 1011 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1012 MemTy, E->getExprLoc(), sizeChars); 1013 break; 1014 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1015 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1016 case AtomicExpr::AO__atomic_sub_fetch: 1017 PostOp = llvm::Instruction::Sub; 1018 // Fall through. 1019 case AtomicExpr::AO__c11_atomic_fetch_sub: 1020 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1021 case AtomicExpr::AO__atomic_fetch_sub: 1022 LibCallName = "__atomic_fetch_sub"; 1023 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1024 LoweredMemTy, E->getExprLoc(), sizeChars); 1025 break; 1026 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1027 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1028 case AtomicExpr::AO__atomic_xor_fetch: 1029 PostOp = llvm::Instruction::Xor; 1030 // Fall through. 1031 case AtomicExpr::AO__c11_atomic_fetch_xor: 1032 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1033 case AtomicExpr::AO__atomic_fetch_xor: 1034 LibCallName = "__atomic_fetch_xor"; 1035 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1036 MemTy, E->getExprLoc(), sizeChars); 1037 break; 1038 case AtomicExpr::AO__opencl_atomic_fetch_min: 1039 LibCallName = E->getValueType()->isSignedIntegerType() 1040 ? "__atomic_fetch_min" 1041 : "__atomic_fetch_umin"; 1042 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1043 LoweredMemTy, E->getExprLoc(), sizeChars); 1044 break; 1045 case AtomicExpr::AO__opencl_atomic_fetch_max: 1046 LibCallName = E->getValueType()->isSignedIntegerType() 1047 ? "__atomic_fetch_max" 1048 : "__atomic_fetch_umax"; 1049 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1050 LoweredMemTy, E->getExprLoc(), sizeChars); 1051 break; 1052 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1053 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1054 case AtomicExpr::AO__atomic_nand_fetch: 1055 PostOp = llvm::Instruction::And; // the NOT is special cased below 1056 // Fall through. 1057 case AtomicExpr::AO__atomic_fetch_nand: 1058 LibCallName = "__atomic_fetch_nand"; 1059 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1060 MemTy, E->getExprLoc(), sizeChars); 1061 break; 1062 } 1063 1064 if (E->isOpenCL()) { 1065 LibCallName = std::string("__opencl") + 1066 StringRef(LibCallName).drop_front(1).str(); 1067 1068 } 1069 // Optimized functions have the size in their name. 1070 if (UseOptimizedLibcall) 1071 LibCallName += "_" + llvm::utostr(Size); 1072 // By default, assume we return a value of the atomic type. 1073 if (!HaveRetTy) { 1074 if (UseOptimizedLibcall) { 1075 // Value is returned directly. 1076 // The function returns an appropriately sized integer type. 1077 RetTy = getContext().getIntTypeForBitwidth( 1078 getContext().toBits(sizeChars), /*Signed=*/false); 1079 } else { 1080 // Value is returned through parameter before the order. 1081 RetTy = getContext().VoidTy; 1082 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 1083 getContext().VoidPtrTy); 1084 } 1085 } 1086 // order is always the last parameter 1087 Args.add(RValue::get(Order), 1088 getContext().IntTy); 1089 if (E->isOpenCL()) 1090 Args.add(RValue::get(Scope), getContext().IntTy); 1091 1092 // PostOp is only needed for the atomic_*_fetch operations, and 1093 // thus is only needed for and implemented in the 1094 // UseOptimizedLibcall codepath. 1095 assert(UseOptimizedLibcall || !PostOp); 1096 1097 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1098 // The value is returned directly from the libcall. 1099 if (E->isCmpXChg()) 1100 return Res; 1101 1102 // The value is returned directly for optimized libcalls but the expr 1103 // provided an out-param. 1104 if (UseOptimizedLibcall && Res.getScalarVal()) { 1105 llvm::Value *ResVal = Res.getScalarVal(); 1106 if (PostOp) { 1107 llvm::Value *LoadVal1 = Args[1].RV.getScalarVal(); 1108 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1109 } 1110 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1111 ResVal = Builder.CreateNot(ResVal); 1112 1113 Builder.CreateStore( 1114 ResVal, 1115 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 1116 } 1117 1118 if (RValTy->isVoidType()) 1119 return RValue::get(nullptr); 1120 1121 return convertTempToRValue( 1122 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1123 RValTy, E->getExprLoc()); 1124 } 1125 1126 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1127 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1128 E->getOp() == AtomicExpr::AO__atomic_store || 1129 E->getOp() == AtomicExpr::AO__atomic_store_n; 1130 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1131 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1132 E->getOp() == AtomicExpr::AO__atomic_load || 1133 E->getOp() == AtomicExpr::AO__atomic_load_n; 1134 1135 assert(isa<llvm::ConstantInt>(Scope) && 1136 "Non-constant synchronization scope not supported"); 1137 auto SCID = getTargetHooks().getLLVMSyncScopeID( 1138 static_cast<SyncScope>(cast<llvm::ConstantInt>(Scope)->getZExtValue()), 1139 getLLVMContext()); 1140 1141 if (isa<llvm::ConstantInt>(Order)) { 1142 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1143 // We should not ever get to a case where the ordering isn't a valid C ABI 1144 // value, but it's hard to enforce that in general. 1145 if (llvm::isValidAtomicOrderingCABI(ord)) 1146 switch ((llvm::AtomicOrderingCABI)ord) { 1147 case llvm::AtomicOrderingCABI::relaxed: 1148 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1149 llvm::AtomicOrdering::Monotonic, SCID); 1150 break; 1151 case llvm::AtomicOrderingCABI::consume: 1152 case llvm::AtomicOrderingCABI::acquire: 1153 if (IsStore) 1154 break; // Avoid crashing on code with undefined behavior 1155 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1156 llvm::AtomicOrdering::Acquire, SCID); 1157 break; 1158 case llvm::AtomicOrderingCABI::release: 1159 if (IsLoad) 1160 break; // Avoid crashing on code with undefined behavior 1161 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1162 llvm::AtomicOrdering::Release, SCID); 1163 break; 1164 case llvm::AtomicOrderingCABI::acq_rel: 1165 if (IsLoad || IsStore) 1166 break; // Avoid crashing on code with undefined behavior 1167 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1168 llvm::AtomicOrdering::AcquireRelease, SCID); 1169 break; 1170 case llvm::AtomicOrderingCABI::seq_cst: 1171 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1172 llvm::AtomicOrdering::SequentiallyConsistent, SCID); 1173 break; 1174 } 1175 if (RValTy->isVoidType()) 1176 return RValue::get(nullptr); 1177 1178 return convertTempToRValue( 1179 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1180 RValTy, E->getExprLoc()); 1181 } 1182 1183 // Long case, when Order isn't obviously constant. 1184 1185 // Create all the relevant BB's 1186 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1187 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1188 *SeqCstBB = nullptr; 1189 MonotonicBB = createBasicBlock("monotonic", CurFn); 1190 if (!IsStore) 1191 AcquireBB = createBasicBlock("acquire", CurFn); 1192 if (!IsLoad) 1193 ReleaseBB = createBasicBlock("release", CurFn); 1194 if (!IsLoad && !IsStore) 1195 AcqRelBB = createBasicBlock("acqrel", CurFn); 1196 SeqCstBB = createBasicBlock("seqcst", CurFn); 1197 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1198 1199 // Create the switch for the split 1200 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1201 // doesn't matter unless someone is crazy enough to use something that 1202 // doesn't fold to a constant for the ordering. 1203 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1204 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1205 1206 // Emit all the different atomics 1207 Builder.SetInsertPoint(MonotonicBB); 1208 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1209 llvm::AtomicOrdering::Monotonic, SCID); 1210 Builder.CreateBr(ContBB); 1211 if (!IsStore) { 1212 Builder.SetInsertPoint(AcquireBB); 1213 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1214 llvm::AtomicOrdering::Acquire, SCID); 1215 Builder.CreateBr(ContBB); 1216 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1217 AcquireBB); 1218 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1219 AcquireBB); 1220 } 1221 if (!IsLoad) { 1222 Builder.SetInsertPoint(ReleaseBB); 1223 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1224 llvm::AtomicOrdering::Release, SCID); 1225 Builder.CreateBr(ContBB); 1226 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1227 ReleaseBB); 1228 } 1229 if (!IsLoad && !IsStore) { 1230 Builder.SetInsertPoint(AcqRelBB); 1231 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1232 llvm::AtomicOrdering::AcquireRelease, SCID); 1233 Builder.CreateBr(ContBB); 1234 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1235 AcqRelBB); 1236 } 1237 Builder.SetInsertPoint(SeqCstBB); 1238 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1239 llvm::AtomicOrdering::SequentiallyConsistent, SCID); 1240 Builder.CreateBr(ContBB); 1241 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1242 SeqCstBB); 1243 1244 // Cleanup and return 1245 Builder.SetInsertPoint(ContBB); 1246 if (RValTy->isVoidType()) 1247 return RValue::get(nullptr); 1248 1249 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1250 return convertTempToRValue( 1251 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1252 RValTy, E->getExprLoc()); 1253 } 1254 1255 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1256 unsigned addrspace = 1257 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 1258 llvm::IntegerType *ty = 1259 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1260 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 1261 } 1262 1263 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1264 llvm::Type *Ty = Addr.getElementType(); 1265 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1266 if (SourceSizeInBits != AtomicSizeInBits) { 1267 Address Tmp = CreateTempAlloca(); 1268 CGF.Builder.CreateMemCpy(Tmp, Addr, 1269 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1270 Addr = Tmp; 1271 } 1272 1273 return emitCastToAtomicIntPointer(Addr); 1274 } 1275 1276 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1277 AggValueSlot resultSlot, 1278 SourceLocation loc, 1279 bool asValue) const { 1280 if (LVal.isSimple()) { 1281 if (EvaluationKind == TEK_Aggregate) 1282 return resultSlot.asRValue(); 1283 1284 // Drill into the padding structure if we have one. 1285 if (hasPadding()) 1286 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 1287 1288 // Otherwise, just convert the temporary to an r-value using the 1289 // normal conversion routine. 1290 return CGF.convertTempToRValue(addr, getValueType(), loc); 1291 } 1292 if (!asValue) 1293 // Get RValue from temp memory as atomic for non-simple lvalues 1294 return RValue::get(CGF.Builder.CreateLoad(addr)); 1295 if (LVal.isBitField()) 1296 return CGF.EmitLoadOfBitfieldLValue( 1297 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1298 LVal.getBaseInfo()), loc); 1299 if (LVal.isVectorElt()) 1300 return CGF.EmitLoadOfLValue( 1301 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1302 LVal.getBaseInfo()), loc); 1303 assert(LVal.isExtVectorElt()); 1304 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1305 addr, LVal.getExtVectorElts(), LVal.getType(), 1306 LVal.getBaseInfo())); 1307 } 1308 1309 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1310 AggValueSlot ResultSlot, 1311 SourceLocation Loc, 1312 bool AsValue) const { 1313 // Try not to in some easy cases. 1314 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1315 if (getEvaluationKind() == TEK_Scalar && 1316 (((!LVal.isBitField() || 1317 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1318 !hasPadding()) || 1319 !AsValue)) { 1320 auto *ValTy = AsValue 1321 ? CGF.ConvertTypeForMem(ValueTy) 1322 : getAtomicAddress().getType()->getPointerElementType(); 1323 if (ValTy->isIntegerTy()) { 1324 assert(IntVal->getType() == ValTy && "Different integer types."); 1325 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1326 } else if (ValTy->isPointerTy()) 1327 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1328 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1329 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1330 } 1331 1332 // Create a temporary. This needs to be big enough to hold the 1333 // atomic integer. 1334 Address Temp = Address::invalid(); 1335 bool TempIsVolatile = false; 1336 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1337 assert(!ResultSlot.isIgnored()); 1338 Temp = ResultSlot.getAddress(); 1339 TempIsVolatile = ResultSlot.isVolatile(); 1340 } else { 1341 Temp = CreateTempAlloca(); 1342 } 1343 1344 // Slam the integer into the temporary. 1345 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1346 CGF.Builder.CreateStore(IntVal, CastTemp) 1347 ->setVolatile(TempIsVolatile); 1348 1349 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1350 } 1351 1352 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1353 llvm::AtomicOrdering AO, bool) { 1354 // void __atomic_load(size_t size, void *mem, void *return, int order); 1355 CallArgList Args; 1356 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1357 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1358 CGF.getContext().VoidPtrTy); 1359 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1360 CGF.getContext().VoidPtrTy); 1361 Args.add( 1362 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1363 CGF.getContext().IntTy); 1364 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1365 } 1366 1367 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1368 bool IsVolatile) { 1369 // Okay, we're doing this natively. 1370 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1371 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1372 Load->setAtomic(AO); 1373 1374 // Other decoration. 1375 if (IsVolatile) 1376 Load->setVolatile(true); 1377 if (LVal.getTBAAInfo()) 1378 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1379 return Load; 1380 } 1381 1382 /// An LValue is a candidate for having its loads and stores be made atomic if 1383 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1384 /// performing such an operation can be performed without a libcall. 1385 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1386 if (!CGM.getCodeGenOpts().MSVolatile) return false; 1387 AtomicInfo AI(*this, LV); 1388 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1389 // An atomic is inline if we don't need to use a libcall. 1390 bool AtomicIsInline = !AI.shouldUseLibcall(); 1391 // MSVC doesn't seem to do this for types wider than a pointer. 1392 if (getContext().getTypeSize(LV.getType()) > 1393 getContext().getTypeSize(getContext().getIntPtrType())) 1394 return false; 1395 return IsVolatile && AtomicIsInline; 1396 } 1397 1398 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1399 AggValueSlot Slot) { 1400 llvm::AtomicOrdering AO; 1401 bool IsVolatile = LV.isVolatileQualified(); 1402 if (LV.getType()->isAtomicType()) { 1403 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1404 } else { 1405 AO = llvm::AtomicOrdering::Acquire; 1406 IsVolatile = true; 1407 } 1408 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1409 } 1410 1411 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1412 bool AsValue, llvm::AtomicOrdering AO, 1413 bool IsVolatile) { 1414 // Check whether we should use a library call. 1415 if (shouldUseLibcall()) { 1416 Address TempAddr = Address::invalid(); 1417 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1418 assert(getEvaluationKind() == TEK_Aggregate); 1419 TempAddr = ResultSlot.getAddress(); 1420 } else 1421 TempAddr = CreateTempAlloca(); 1422 1423 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1424 1425 // Okay, turn that back into the original value or whole atomic (for 1426 // non-simple lvalues) type. 1427 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1428 } 1429 1430 // Okay, we're doing this natively. 1431 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1432 1433 // If we're ignoring an aggregate return, don't do anything. 1434 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1435 return RValue::getAggregate(Address::invalid(), false); 1436 1437 // Okay, turn that back into the original value or atomic (for non-simple 1438 // lvalues) type. 1439 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1440 } 1441 1442 /// Emit a load from an l-value of atomic type. Note that the r-value 1443 /// we produce is an r-value of the atomic *value* type. 1444 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1445 llvm::AtomicOrdering AO, bool IsVolatile, 1446 AggValueSlot resultSlot) { 1447 AtomicInfo Atomics(*this, src); 1448 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1449 IsVolatile); 1450 } 1451 1452 /// Copy an r-value into memory as part of storing to an atomic type. 1453 /// This needs to create a bit-pattern suitable for atomic operations. 1454 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1455 assert(LVal.isSimple()); 1456 // If we have an r-value, the rvalue should be of the atomic type, 1457 // which means that the caller is responsible for having zeroed 1458 // any padding. Just do an aggregate copy of that type. 1459 if (rvalue.isAggregate()) { 1460 CGF.EmitAggregateCopy(getAtomicAddress(), 1461 rvalue.getAggregateAddress(), 1462 getAtomicType(), 1463 (rvalue.isVolatileQualified() 1464 || LVal.isVolatileQualified())); 1465 return; 1466 } 1467 1468 // Okay, otherwise we're copying stuff. 1469 1470 // Zero out the buffer if necessary. 1471 emitMemSetZeroIfNecessary(); 1472 1473 // Drill past the padding if present. 1474 LValue TempLVal = projectValue(); 1475 1476 // Okay, store the rvalue in. 1477 if (rvalue.isScalar()) { 1478 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1479 } else { 1480 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1481 } 1482 } 1483 1484 1485 /// Materialize an r-value into memory for the purposes of storing it 1486 /// to an atomic type. 1487 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1488 // Aggregate r-values are already in memory, and EmitAtomicStore 1489 // requires them to be values of the atomic type. 1490 if (rvalue.isAggregate()) 1491 return rvalue.getAggregateAddress(); 1492 1493 // Otherwise, make a temporary and materialize into it. 1494 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1495 AtomicInfo Atomics(CGF, TempLV); 1496 Atomics.emitCopyIntoMemory(rvalue); 1497 return TempLV.getAddress(); 1498 } 1499 1500 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1501 // If we've got a scalar value of the right size, try to avoid going 1502 // through memory. 1503 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1504 llvm::Value *Value = RVal.getScalarVal(); 1505 if (isa<llvm::IntegerType>(Value->getType())) 1506 return CGF.EmitToMemory(Value, ValueTy); 1507 else { 1508 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1509 CGF.getLLVMContext(), 1510 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1511 if (isa<llvm::PointerType>(Value->getType())) 1512 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1513 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1514 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1515 } 1516 } 1517 // Otherwise, we need to go through memory. 1518 // Put the r-value in memory. 1519 Address Addr = materializeRValue(RVal); 1520 1521 // Cast the temporary to the atomic int type and pull a value out. 1522 Addr = emitCastToAtomicIntPointer(Addr); 1523 return CGF.Builder.CreateLoad(Addr); 1524 } 1525 1526 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1527 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1528 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1529 // Do the atomic store. 1530 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1531 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1532 ExpectedVal, DesiredVal, 1533 Success, Failure); 1534 // Other decoration. 1535 Inst->setVolatile(LVal.isVolatileQualified()); 1536 Inst->setWeak(IsWeak); 1537 1538 // Okay, turn that back into the original value type. 1539 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1540 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1541 return std::make_pair(PreviousVal, SuccessFailureVal); 1542 } 1543 1544 llvm::Value * 1545 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1546 llvm::Value *DesiredAddr, 1547 llvm::AtomicOrdering Success, 1548 llvm::AtomicOrdering Failure) { 1549 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1550 // void *desired, int success, int failure); 1551 CallArgList Args; 1552 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1553 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1554 CGF.getContext().VoidPtrTy); 1555 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1556 CGF.getContext().VoidPtrTy); 1557 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1558 CGF.getContext().VoidPtrTy); 1559 Args.add(RValue::get( 1560 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1561 CGF.getContext().IntTy); 1562 Args.add(RValue::get( 1563 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1564 CGF.getContext().IntTy); 1565 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1566 CGF.getContext().BoolTy, Args); 1567 1568 return SuccessFailureRVal.getScalarVal(); 1569 } 1570 1571 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1572 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1573 llvm::AtomicOrdering Failure, bool IsWeak) { 1574 if (isStrongerThan(Failure, Success)) 1575 // Don't assert on undefined behavior "failure argument shall be no stronger 1576 // than the success argument". 1577 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 1578 1579 // Check whether we should use a library call. 1580 if (shouldUseLibcall()) { 1581 // Produce a source address. 1582 Address ExpectedAddr = materializeRValue(Expected); 1583 Address DesiredAddr = materializeRValue(Desired); 1584 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1585 DesiredAddr.getPointer(), 1586 Success, Failure); 1587 return std::make_pair( 1588 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1589 SourceLocation(), /*AsValue=*/false), 1590 Res); 1591 } 1592 1593 // If we've got a scalar value of the right size, try to avoid going 1594 // through memory. 1595 auto *ExpectedVal = convertRValueToInt(Expected); 1596 auto *DesiredVal = convertRValueToInt(Desired); 1597 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1598 Failure, IsWeak); 1599 return std::make_pair( 1600 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1601 SourceLocation(), /*AsValue=*/false), 1602 Res.second); 1603 } 1604 1605 static void 1606 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1607 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1608 Address DesiredAddr) { 1609 RValue UpRVal; 1610 LValue AtomicLVal = Atomics.getAtomicLValue(); 1611 LValue DesiredLVal; 1612 if (AtomicLVal.isSimple()) { 1613 UpRVal = OldRVal; 1614 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1615 } else { 1616 // Build new lvalue for temp address 1617 Address Ptr = Atomics.materializeRValue(OldRVal); 1618 LValue UpdateLVal; 1619 if (AtomicLVal.isBitField()) { 1620 UpdateLVal = 1621 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1622 AtomicLVal.getType(), 1623 AtomicLVal.getBaseInfo()); 1624 DesiredLVal = 1625 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1626 AtomicLVal.getType(), 1627 AtomicLVal.getBaseInfo()); 1628 } else if (AtomicLVal.isVectorElt()) { 1629 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1630 AtomicLVal.getType(), 1631 AtomicLVal.getBaseInfo()); 1632 DesiredLVal = LValue::MakeVectorElt( 1633 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1634 AtomicLVal.getBaseInfo()); 1635 } else { 1636 assert(AtomicLVal.isExtVectorElt()); 1637 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1638 AtomicLVal.getType(), 1639 AtomicLVal.getBaseInfo()); 1640 DesiredLVal = LValue::MakeExtVectorElt( 1641 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1642 AtomicLVal.getBaseInfo()); 1643 } 1644 UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1645 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1646 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1647 } 1648 // Store new value in the corresponding memory area 1649 RValue NewRVal = UpdateOp(UpRVal); 1650 if (NewRVal.isScalar()) { 1651 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1652 } else { 1653 assert(NewRVal.isComplex()); 1654 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1655 /*isInit=*/false); 1656 } 1657 } 1658 1659 void AtomicInfo::EmitAtomicUpdateLibcall( 1660 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1661 bool IsVolatile) { 1662 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1663 1664 Address ExpectedAddr = CreateTempAlloca(); 1665 1666 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1667 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1668 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1669 CGF.EmitBlock(ContBB); 1670 Address DesiredAddr = CreateTempAlloca(); 1671 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1672 requiresMemSetZero(getAtomicAddress().getElementType())) { 1673 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1674 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1675 } 1676 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1677 AggValueSlot::ignored(), 1678 SourceLocation(), /*AsValue=*/false); 1679 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1680 auto *Res = 1681 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1682 DesiredAddr.getPointer(), 1683 AO, Failure); 1684 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1685 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1686 } 1687 1688 void AtomicInfo::EmitAtomicUpdateOp( 1689 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1690 bool IsVolatile) { 1691 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1692 1693 // Do the atomic load. 1694 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1695 // For non-simple lvalues perform compare-and-swap procedure. 1696 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1697 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1698 auto *CurBB = CGF.Builder.GetInsertBlock(); 1699 CGF.EmitBlock(ContBB); 1700 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1701 /*NumReservedValues=*/2); 1702 PHI->addIncoming(OldVal, CurBB); 1703 Address NewAtomicAddr = CreateTempAlloca(); 1704 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1705 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1706 requiresMemSetZero(getAtomicAddress().getElementType())) { 1707 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1708 } 1709 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1710 SourceLocation(), /*AsValue=*/false); 1711 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1712 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1713 // Try to write new value using cmpxchg operation 1714 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1715 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1716 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1717 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1718 } 1719 1720 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1721 RValue UpdateRVal, Address DesiredAddr) { 1722 LValue AtomicLVal = Atomics.getAtomicLValue(); 1723 LValue DesiredLVal; 1724 // Build new lvalue for temp address 1725 if (AtomicLVal.isBitField()) { 1726 DesiredLVal = 1727 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1728 AtomicLVal.getType(), 1729 AtomicLVal.getBaseInfo()); 1730 } else if (AtomicLVal.isVectorElt()) { 1731 DesiredLVal = 1732 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1733 AtomicLVal.getType(), 1734 AtomicLVal.getBaseInfo()); 1735 } else { 1736 assert(AtomicLVal.isExtVectorElt()); 1737 DesiredLVal = LValue::MakeExtVectorElt( 1738 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1739 AtomicLVal.getBaseInfo()); 1740 } 1741 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1742 // Store new value in the corresponding memory area 1743 assert(UpdateRVal.isScalar()); 1744 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1745 } 1746 1747 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1748 RValue UpdateRVal, bool IsVolatile) { 1749 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1750 1751 Address ExpectedAddr = CreateTempAlloca(); 1752 1753 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1754 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1755 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1756 CGF.EmitBlock(ContBB); 1757 Address DesiredAddr = CreateTempAlloca(); 1758 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1759 requiresMemSetZero(getAtomicAddress().getElementType())) { 1760 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1761 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1762 } 1763 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1764 auto *Res = 1765 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1766 DesiredAddr.getPointer(), 1767 AO, Failure); 1768 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1769 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1770 } 1771 1772 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1773 bool IsVolatile) { 1774 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1775 1776 // Do the atomic load. 1777 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1778 // For non-simple lvalues perform compare-and-swap procedure. 1779 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1780 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1781 auto *CurBB = CGF.Builder.GetInsertBlock(); 1782 CGF.EmitBlock(ContBB); 1783 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1784 /*NumReservedValues=*/2); 1785 PHI->addIncoming(OldVal, CurBB); 1786 Address NewAtomicAddr = CreateTempAlloca(); 1787 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1788 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1789 requiresMemSetZero(getAtomicAddress().getElementType())) { 1790 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1791 } 1792 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1793 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1794 // Try to write new value using cmpxchg operation 1795 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1796 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1797 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1798 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1799 } 1800 1801 void AtomicInfo::EmitAtomicUpdate( 1802 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1803 bool IsVolatile) { 1804 if (shouldUseLibcall()) { 1805 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1806 } else { 1807 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1808 } 1809 } 1810 1811 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1812 bool IsVolatile) { 1813 if (shouldUseLibcall()) { 1814 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1815 } else { 1816 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1817 } 1818 } 1819 1820 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1821 bool isInit) { 1822 bool IsVolatile = lvalue.isVolatileQualified(); 1823 llvm::AtomicOrdering AO; 1824 if (lvalue.getType()->isAtomicType()) { 1825 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1826 } else { 1827 AO = llvm::AtomicOrdering::Release; 1828 IsVolatile = true; 1829 } 1830 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1831 } 1832 1833 /// Emit a store to an l-value of atomic type. 1834 /// 1835 /// Note that the r-value is expected to be an r-value *of the atomic 1836 /// type*; this means that for aggregate r-values, it should include 1837 /// storage for any padding that was necessary. 1838 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1839 llvm::AtomicOrdering AO, bool IsVolatile, 1840 bool isInit) { 1841 // If this is an aggregate r-value, it should agree in type except 1842 // maybe for address-space qualification. 1843 assert(!rvalue.isAggregate() || 1844 rvalue.getAggregateAddress().getElementType() 1845 == dest.getAddress().getElementType()); 1846 1847 AtomicInfo atomics(*this, dest); 1848 LValue LVal = atomics.getAtomicLValue(); 1849 1850 // If this is an initialization, just put the value there normally. 1851 if (LVal.isSimple()) { 1852 if (isInit) { 1853 atomics.emitCopyIntoMemory(rvalue); 1854 return; 1855 } 1856 1857 // Check whether we should use a library call. 1858 if (atomics.shouldUseLibcall()) { 1859 // Produce a source address. 1860 Address srcAddr = atomics.materializeRValue(rvalue); 1861 1862 // void __atomic_store(size_t size, void *mem, void *val, int order) 1863 CallArgList args; 1864 args.add(RValue::get(atomics.getAtomicSizeValue()), 1865 getContext().getSizeType()); 1866 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 1867 getContext().VoidPtrTy); 1868 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 1869 getContext().VoidPtrTy); 1870 args.add( 1871 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 1872 getContext().IntTy); 1873 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 1874 return; 1875 } 1876 1877 // Okay, we're doing this natively. 1878 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 1879 1880 // Do the atomic store. 1881 Address addr = 1882 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 1883 intValue = Builder.CreateIntCast( 1884 intValue, addr.getElementType(), /*isSigned=*/false); 1885 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 1886 1887 // Initializations don't need to be atomic. 1888 if (!isInit) 1889 store->setAtomic(AO); 1890 1891 // Other decoration. 1892 if (IsVolatile) 1893 store->setVolatile(true); 1894 if (dest.getTBAAInfo()) 1895 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 1896 return; 1897 } 1898 1899 // Emit simple atomic update operation. 1900 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 1901 } 1902 1903 /// Emit a compare-and-exchange op for atomic type. 1904 /// 1905 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 1906 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 1907 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 1908 AggValueSlot Slot) { 1909 // If this is an aggregate r-value, it should agree in type except 1910 // maybe for address-space qualification. 1911 assert(!Expected.isAggregate() || 1912 Expected.getAggregateAddress().getElementType() == 1913 Obj.getAddress().getElementType()); 1914 assert(!Desired.isAggregate() || 1915 Desired.getAggregateAddress().getElementType() == 1916 Obj.getAddress().getElementType()); 1917 AtomicInfo Atomics(*this, Obj); 1918 1919 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 1920 IsWeak); 1921 } 1922 1923 void CodeGenFunction::EmitAtomicUpdate( 1924 LValue LVal, llvm::AtomicOrdering AO, 1925 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 1926 AtomicInfo Atomics(*this, LVal); 1927 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 1928 } 1929 1930 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 1931 AtomicInfo atomics(*this, dest); 1932 1933 switch (atomics.getEvaluationKind()) { 1934 case TEK_Scalar: { 1935 llvm::Value *value = EmitScalarExpr(init); 1936 atomics.emitCopyIntoMemory(RValue::get(value)); 1937 return; 1938 } 1939 1940 case TEK_Complex: { 1941 ComplexPairTy value = EmitComplexExpr(init); 1942 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 1943 return; 1944 } 1945 1946 case TEK_Aggregate: { 1947 // Fix up the destination if the initializer isn't an expression 1948 // of atomic type. 1949 bool Zeroed = false; 1950 if (!init->getType()->isAtomicType()) { 1951 Zeroed = atomics.emitMemSetZeroIfNecessary(); 1952 dest = atomics.projectValue(); 1953 } 1954 1955 // Evaluate the expression directly into the destination. 1956 AggValueSlot slot = AggValueSlot::forLValue(dest, 1957 AggValueSlot::IsNotDestructed, 1958 AggValueSlot::DoesNotNeedGCBarriers, 1959 AggValueSlot::IsNotAliased, 1960 Zeroed ? AggValueSlot::IsZeroed : 1961 AggValueSlot::IsNotZeroed); 1962 1963 EmitAggExpr(init, slot); 1964 return; 1965 } 1966 } 1967 llvm_unreachable("bad evaluation kind"); 1968 } 1969