1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the code for emitting atomic operations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CGCall.h" 15 #include "CGRecordLayout.h" 16 #include "CodeGenFunction.h" 17 #include "CodeGenModule.h" 18 #include "TargetInfo.h" 19 #include "clang/AST/ASTContext.h" 20 #include "clang/CodeGen/CGFunctionInfo.h" 21 #include "clang/Sema/SemaDiagnostic.h" 22 #include "llvm/ADT/DenseMap.h" 23 #include "llvm/IR/DataLayout.h" 24 #include "llvm/IR/Intrinsics.h" 25 #include "llvm/IR/Operator.h" 26 27 using namespace clang; 28 using namespace CodeGen; 29 30 namespace { 31 class AtomicInfo { 32 CodeGenFunction &CGF; 33 QualType AtomicTy; 34 QualType ValueTy; 35 uint64_t AtomicSizeInBits; 36 uint64_t ValueSizeInBits; 37 CharUnits AtomicAlign; 38 CharUnits ValueAlign; 39 CharUnits LValueAlign; 40 TypeEvaluationKind EvaluationKind; 41 bool UseLibcall; 42 LValue LVal; 43 CGBitFieldInfo BFI; 44 public: 45 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 46 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 47 EvaluationKind(TEK_Scalar), UseLibcall(true) { 48 assert(!lvalue.isGlobalReg()); 49 ASTContext &C = CGF.getContext(); 50 if (lvalue.isSimple()) { 51 AtomicTy = lvalue.getType(); 52 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 53 ValueTy = ATy->getValueType(); 54 else 55 ValueTy = AtomicTy; 56 EvaluationKind = CGF.getEvaluationKind(ValueTy); 57 58 uint64_t ValueAlignInBits; 59 uint64_t AtomicAlignInBits; 60 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 61 ValueSizeInBits = ValueTI.Width; 62 ValueAlignInBits = ValueTI.Align; 63 64 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 65 AtomicSizeInBits = AtomicTI.Width; 66 AtomicAlignInBits = AtomicTI.Align; 67 68 assert(ValueSizeInBits <= AtomicSizeInBits); 69 assert(ValueAlignInBits <= AtomicAlignInBits); 70 71 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 72 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 73 if (lvalue.getAlignment().isZero()) 74 lvalue.setAlignment(AtomicAlign); 75 76 LVal = lvalue; 77 } else if (lvalue.isBitField()) { 78 ValueTy = lvalue.getType(); 79 ValueSizeInBits = C.getTypeSize(ValueTy); 80 auto &OrigBFI = lvalue.getBitFieldInfo(); 81 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 82 AtomicSizeInBits = C.toBits( 83 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 84 .alignTo(lvalue.getAlignment())); 85 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 86 auto OffsetInChars = 87 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 88 lvalue.getAlignment(); 89 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 90 VoidPtrAddr, OffsetInChars.getQuantity()); 91 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 92 VoidPtrAddr, 93 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 94 "atomic_bitfield_base"); 95 BFI = OrigBFI; 96 BFI.Offset = Offset; 97 BFI.StorageSize = AtomicSizeInBits; 98 BFI.StorageOffset += OffsetInChars; 99 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 100 BFI, lvalue.getType(), lvalue.getBaseInfo(), 101 lvalue.getTBAAInfo()); 102 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 103 if (AtomicTy.isNull()) { 104 llvm::APInt Size( 105 /*numBits=*/32, 106 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 107 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, 108 /*IndexTypeQuals=*/0); 109 } 110 AtomicAlign = ValueAlign = lvalue.getAlignment(); 111 } else if (lvalue.isVectorElt()) { 112 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); 113 ValueSizeInBits = C.getTypeSize(ValueTy); 114 AtomicTy = lvalue.getType(); 115 AtomicSizeInBits = C.getTypeSize(AtomicTy); 116 AtomicAlign = ValueAlign = lvalue.getAlignment(); 117 LVal = lvalue; 118 } else { 119 assert(lvalue.isExtVectorElt()); 120 ValueTy = lvalue.getType(); 121 ValueSizeInBits = C.getTypeSize(ValueTy); 122 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 123 lvalue.getType(), lvalue.getExtVectorAddress() 124 .getElementType()->getVectorNumElements()); 125 AtomicSizeInBits = C.getTypeSize(AtomicTy); 126 AtomicAlign = ValueAlign = lvalue.getAlignment(); 127 LVal = lvalue; 128 } 129 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 130 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 131 } 132 133 QualType getAtomicType() const { return AtomicTy; } 134 QualType getValueType() const { return ValueTy; } 135 CharUnits getAtomicAlignment() const { return AtomicAlign; } 136 CharUnits getValueAlignment() const { return ValueAlign; } 137 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 138 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 139 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 140 bool shouldUseLibcall() const { return UseLibcall; } 141 const LValue &getAtomicLValue() const { return LVal; } 142 llvm::Value *getAtomicPointer() const { 143 if (LVal.isSimple()) 144 return LVal.getPointer(); 145 else if (LVal.isBitField()) 146 return LVal.getBitFieldPointer(); 147 else if (LVal.isVectorElt()) 148 return LVal.getVectorPointer(); 149 assert(LVal.isExtVectorElt()); 150 return LVal.getExtVectorPointer(); 151 } 152 Address getAtomicAddress() const { 153 return Address(getAtomicPointer(), getAtomicAlignment()); 154 } 155 156 Address getAtomicAddressAsAtomicIntPointer() const { 157 return emitCastToAtomicIntPointer(getAtomicAddress()); 158 } 159 160 /// Is the atomic size larger than the underlying value type? 161 /// 162 /// Note that the absence of padding does not mean that atomic 163 /// objects are completely interchangeable with non-atomic 164 /// objects: we might have promoted the alignment of a type 165 /// without making it bigger. 166 bool hasPadding() const { 167 return (ValueSizeInBits != AtomicSizeInBits); 168 } 169 170 bool emitMemSetZeroIfNecessary() const; 171 172 llvm::Value *getAtomicSizeValue() const { 173 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 174 return CGF.CGM.getSize(size); 175 } 176 177 /// Cast the given pointer to an integer pointer suitable for atomic 178 /// operations if the source. 179 Address emitCastToAtomicIntPointer(Address Addr) const; 180 181 /// If Addr is compatible with the iN that will be used for an atomic 182 /// operation, bitcast it. Otherwise, create a temporary that is suitable 183 /// and copy the value across. 184 Address convertToAtomicIntPointer(Address Addr) const; 185 186 /// Turn an atomic-layout object into an r-value. 187 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 188 SourceLocation loc, bool AsValue) const; 189 190 /// Converts a rvalue to integer value. 191 llvm::Value *convertRValueToInt(RValue RVal) const; 192 193 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 194 AggValueSlot ResultSlot, 195 SourceLocation Loc, bool AsValue) const; 196 197 /// Copy an atomic r-value into atomic-layout memory. 198 void emitCopyIntoMemory(RValue rvalue) const; 199 200 /// Project an l-value down to the value field. 201 LValue projectValue() const { 202 assert(LVal.isSimple()); 203 Address addr = getAtomicAddress(); 204 if (hasPadding()) 205 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 206 207 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 208 LVal.getBaseInfo(), LVal.getTBAAInfo()); 209 } 210 211 /// Emits atomic load. 212 /// \returns Loaded value. 213 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 214 bool AsValue, llvm::AtomicOrdering AO, 215 bool IsVolatile); 216 217 /// Emits atomic compare-and-exchange sequence. 218 /// \param Expected Expected value. 219 /// \param Desired Desired value. 220 /// \param Success Atomic ordering for success operation. 221 /// \param Failure Atomic ordering for failed operation. 222 /// \param IsWeak true if atomic operation is weak, false otherwise. 223 /// \returns Pair of values: previous value from storage (value type) and 224 /// boolean flag (i1 type) with true if success and false otherwise. 225 std::pair<RValue, llvm::Value *> 226 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 227 llvm::AtomicOrdering Success = 228 llvm::AtomicOrdering::SequentiallyConsistent, 229 llvm::AtomicOrdering Failure = 230 llvm::AtomicOrdering::SequentiallyConsistent, 231 bool IsWeak = false); 232 233 /// Emits atomic update. 234 /// \param AO Atomic ordering. 235 /// \param UpdateOp Update operation for the current lvalue. 236 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 237 const llvm::function_ref<RValue(RValue)> &UpdateOp, 238 bool IsVolatile); 239 /// Emits atomic update. 240 /// \param AO Atomic ordering. 241 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 242 bool IsVolatile); 243 244 /// Materialize an atomic r-value in atomic-layout memory. 245 Address materializeRValue(RValue rvalue) const; 246 247 /// Creates temp alloca for intermediate operations on atomic value. 248 Address CreateTempAlloca() const; 249 private: 250 bool requiresMemSetZero(llvm::Type *type) const; 251 252 253 /// Emits atomic load as a libcall. 254 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 255 llvm::AtomicOrdering AO, bool IsVolatile); 256 /// Emits atomic load as LLVM instruction. 257 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 258 /// Emits atomic compare-and-exchange op as a libcall. 259 llvm::Value *EmitAtomicCompareExchangeLibcall( 260 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 261 llvm::AtomicOrdering Success = 262 llvm::AtomicOrdering::SequentiallyConsistent, 263 llvm::AtomicOrdering Failure = 264 llvm::AtomicOrdering::SequentiallyConsistent); 265 /// Emits atomic compare-and-exchange op as LLVM instruction. 266 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 267 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 268 llvm::AtomicOrdering Success = 269 llvm::AtomicOrdering::SequentiallyConsistent, 270 llvm::AtomicOrdering Failure = 271 llvm::AtomicOrdering::SequentiallyConsistent, 272 bool IsWeak = false); 273 /// Emit atomic update as libcalls. 274 void 275 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 276 const llvm::function_ref<RValue(RValue)> &UpdateOp, 277 bool IsVolatile); 278 /// Emit atomic update as LLVM instructions. 279 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 280 const llvm::function_ref<RValue(RValue)> &UpdateOp, 281 bool IsVolatile); 282 /// Emit atomic update as libcalls. 283 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 284 bool IsVolatile); 285 /// Emit atomic update as LLVM instructions. 286 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 287 bool IsVolatile); 288 }; 289 } 290 291 Address AtomicInfo::CreateTempAlloca() const { 292 Address TempAlloca = CGF.CreateMemTemp( 293 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 294 : AtomicTy, 295 getAtomicAlignment(), 296 "atomic-temp"); 297 // Cast to pointer to value type for bitfields. 298 if (LVal.isBitField()) 299 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 300 TempAlloca, getAtomicAddress().getType()); 301 return TempAlloca; 302 } 303 304 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 305 StringRef fnName, 306 QualType resultType, 307 CallArgList &args) { 308 const CGFunctionInfo &fnInfo = 309 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 310 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 311 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 312 auto callee = CGCallee::forDirect(fn); 313 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 314 } 315 316 /// Does a store of the given IR type modify the full expected width? 317 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 318 uint64_t expectedSize) { 319 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 320 } 321 322 /// Does the atomic type require memsetting to zero before initialization? 323 /// 324 /// The IR type is provided as a way of making certain queries faster. 325 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 326 // If the atomic type has size padding, we definitely need a memset. 327 if (hasPadding()) return true; 328 329 // Otherwise, do some simple heuristics to try to avoid it: 330 switch (getEvaluationKind()) { 331 // For scalars and complexes, check whether the store size of the 332 // type uses the full size. 333 case TEK_Scalar: 334 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 335 case TEK_Complex: 336 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 337 AtomicSizeInBits / 2); 338 339 // Padding in structs has an undefined bit pattern. User beware. 340 case TEK_Aggregate: 341 return false; 342 } 343 llvm_unreachable("bad evaluation kind"); 344 } 345 346 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 347 assert(LVal.isSimple()); 348 llvm::Value *addr = LVal.getPointer(); 349 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 350 return false; 351 352 CGF.Builder.CreateMemSet( 353 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 354 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 355 LVal.getAlignment().getQuantity()); 356 return true; 357 } 358 359 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 360 Address Dest, Address Ptr, 361 Address Val1, Address Val2, 362 uint64_t Size, 363 llvm::AtomicOrdering SuccessOrder, 364 llvm::AtomicOrdering FailureOrder, 365 llvm::SyncScope::ID Scope) { 366 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 367 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 368 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 369 370 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 371 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 372 Scope); 373 Pair->setVolatile(E->isVolatile()); 374 Pair->setWeak(IsWeak); 375 376 // Cmp holds the result of the compare-exchange operation: true on success, 377 // false on failure. 378 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 379 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 380 381 // This basic block is used to hold the store instruction if the operation 382 // failed. 383 llvm::BasicBlock *StoreExpectedBB = 384 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 385 386 // This basic block is the exit point of the operation, we should end up 387 // here regardless of whether or not the operation succeeded. 388 llvm::BasicBlock *ContinueBB = 389 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 390 391 // Update Expected if Expected isn't equal to Old, otherwise branch to the 392 // exit point. 393 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 394 395 CGF.Builder.SetInsertPoint(StoreExpectedBB); 396 // Update the memory at Expected with Old's value. 397 CGF.Builder.CreateStore(Old, Val1); 398 // Finally, branch to the exit point. 399 CGF.Builder.CreateBr(ContinueBB); 400 401 CGF.Builder.SetInsertPoint(ContinueBB); 402 // Update the memory at Dest with Cmp's value. 403 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 404 } 405 406 /// Given an ordering required on success, emit all possible cmpxchg 407 /// instructions to cope with the provided (but possibly only dynamically known) 408 /// FailureOrder. 409 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 410 bool IsWeak, Address Dest, Address Ptr, 411 Address Val1, Address Val2, 412 llvm::Value *FailureOrderVal, 413 uint64_t Size, 414 llvm::AtomicOrdering SuccessOrder, 415 llvm::SyncScope::ID Scope) { 416 llvm::AtomicOrdering FailureOrder; 417 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 418 auto FOS = FO->getSExtValue(); 419 if (!llvm::isValidAtomicOrderingCABI(FOS)) 420 FailureOrder = llvm::AtomicOrdering::Monotonic; 421 else 422 switch ((llvm::AtomicOrderingCABI)FOS) { 423 case llvm::AtomicOrderingCABI::relaxed: 424 case llvm::AtomicOrderingCABI::release: 425 case llvm::AtomicOrderingCABI::acq_rel: 426 FailureOrder = llvm::AtomicOrdering::Monotonic; 427 break; 428 case llvm::AtomicOrderingCABI::consume: 429 case llvm::AtomicOrderingCABI::acquire: 430 FailureOrder = llvm::AtomicOrdering::Acquire; 431 break; 432 case llvm::AtomicOrderingCABI::seq_cst: 433 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 434 break; 435 } 436 if (isStrongerThan(FailureOrder, SuccessOrder)) { 437 // Don't assert on undefined behavior "failure argument shall be no 438 // stronger than the success argument". 439 FailureOrder = 440 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 441 } 442 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 443 FailureOrder, Scope); 444 return; 445 } 446 447 // Create all the relevant BB's 448 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 449 *SeqCstBB = nullptr; 450 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 451 if (SuccessOrder != llvm::AtomicOrdering::Monotonic && 452 SuccessOrder != llvm::AtomicOrdering::Release) 453 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 454 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 455 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 456 457 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 458 459 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 460 461 // Emit all the different atomics 462 463 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 464 // doesn't matter unless someone is crazy enough to use something that 465 // doesn't fold to a constant for the ordering. 466 CGF.Builder.SetInsertPoint(MonotonicBB); 467 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 468 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 469 CGF.Builder.CreateBr(ContBB); 470 471 if (AcquireBB) { 472 CGF.Builder.SetInsertPoint(AcquireBB); 473 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 474 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope); 475 CGF.Builder.CreateBr(ContBB); 476 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 477 AcquireBB); 478 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 479 AcquireBB); 480 } 481 if (SeqCstBB) { 482 CGF.Builder.SetInsertPoint(SeqCstBB); 483 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 484 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 485 CGF.Builder.CreateBr(ContBB); 486 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 487 SeqCstBB); 488 } 489 490 CGF.Builder.SetInsertPoint(ContBB); 491 } 492 493 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 494 Address Ptr, Address Val1, Address Val2, 495 llvm::Value *IsWeak, llvm::Value *FailureOrder, 496 uint64_t Size, llvm::AtomicOrdering Order, 497 llvm::SyncScope::ID Scope) { 498 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 499 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 500 501 switch (E->getOp()) { 502 case AtomicExpr::AO__c11_atomic_init: 503 case AtomicExpr::AO__opencl_atomic_init: 504 llvm_unreachable("Already handled!"); 505 506 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 507 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 508 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 509 FailureOrder, Size, Order, Scope); 510 return; 511 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 512 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 513 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 514 FailureOrder, Size, Order, Scope); 515 return; 516 case AtomicExpr::AO__atomic_compare_exchange: 517 case AtomicExpr::AO__atomic_compare_exchange_n: { 518 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 519 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 520 Val1, Val2, FailureOrder, Size, Order, Scope); 521 } else { 522 // Create all the relevant BB's 523 llvm::BasicBlock *StrongBB = 524 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 525 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 526 llvm::BasicBlock *ContBB = 527 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 528 529 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 530 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 531 532 CGF.Builder.SetInsertPoint(StrongBB); 533 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 534 FailureOrder, Size, Order, Scope); 535 CGF.Builder.CreateBr(ContBB); 536 537 CGF.Builder.SetInsertPoint(WeakBB); 538 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 539 FailureOrder, Size, Order, Scope); 540 CGF.Builder.CreateBr(ContBB); 541 542 CGF.Builder.SetInsertPoint(ContBB); 543 } 544 return; 545 } 546 case AtomicExpr::AO__c11_atomic_load: 547 case AtomicExpr::AO__opencl_atomic_load: 548 case AtomicExpr::AO__atomic_load_n: 549 case AtomicExpr::AO__atomic_load: { 550 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 551 Load->setAtomic(Order, Scope); 552 Load->setVolatile(E->isVolatile()); 553 CGF.Builder.CreateStore(Load, Dest); 554 return; 555 } 556 557 case AtomicExpr::AO__c11_atomic_store: 558 case AtomicExpr::AO__opencl_atomic_store: 559 case AtomicExpr::AO__atomic_store: 560 case AtomicExpr::AO__atomic_store_n: { 561 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 562 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 563 Store->setAtomic(Order, Scope); 564 Store->setVolatile(E->isVolatile()); 565 return; 566 } 567 568 case AtomicExpr::AO__c11_atomic_exchange: 569 case AtomicExpr::AO__opencl_atomic_exchange: 570 case AtomicExpr::AO__atomic_exchange_n: 571 case AtomicExpr::AO__atomic_exchange: 572 Op = llvm::AtomicRMWInst::Xchg; 573 break; 574 575 case AtomicExpr::AO__atomic_add_fetch: 576 PostOp = llvm::Instruction::Add; 577 LLVM_FALLTHROUGH; 578 case AtomicExpr::AO__c11_atomic_fetch_add: 579 case AtomicExpr::AO__opencl_atomic_fetch_add: 580 case AtomicExpr::AO__atomic_fetch_add: 581 Op = llvm::AtomicRMWInst::Add; 582 break; 583 584 case AtomicExpr::AO__atomic_sub_fetch: 585 PostOp = llvm::Instruction::Sub; 586 LLVM_FALLTHROUGH; 587 case AtomicExpr::AO__c11_atomic_fetch_sub: 588 case AtomicExpr::AO__opencl_atomic_fetch_sub: 589 case AtomicExpr::AO__atomic_fetch_sub: 590 Op = llvm::AtomicRMWInst::Sub; 591 break; 592 593 case AtomicExpr::AO__opencl_atomic_fetch_min: 594 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min 595 : llvm::AtomicRMWInst::UMin; 596 break; 597 598 case AtomicExpr::AO__opencl_atomic_fetch_max: 599 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max 600 : llvm::AtomicRMWInst::UMax; 601 break; 602 603 case AtomicExpr::AO__atomic_and_fetch: 604 PostOp = llvm::Instruction::And; 605 LLVM_FALLTHROUGH; 606 case AtomicExpr::AO__c11_atomic_fetch_and: 607 case AtomicExpr::AO__opencl_atomic_fetch_and: 608 case AtomicExpr::AO__atomic_fetch_and: 609 Op = llvm::AtomicRMWInst::And; 610 break; 611 612 case AtomicExpr::AO__atomic_or_fetch: 613 PostOp = llvm::Instruction::Or; 614 LLVM_FALLTHROUGH; 615 case AtomicExpr::AO__c11_atomic_fetch_or: 616 case AtomicExpr::AO__opencl_atomic_fetch_or: 617 case AtomicExpr::AO__atomic_fetch_or: 618 Op = llvm::AtomicRMWInst::Or; 619 break; 620 621 case AtomicExpr::AO__atomic_xor_fetch: 622 PostOp = llvm::Instruction::Xor; 623 LLVM_FALLTHROUGH; 624 case AtomicExpr::AO__c11_atomic_fetch_xor: 625 case AtomicExpr::AO__opencl_atomic_fetch_xor: 626 case AtomicExpr::AO__atomic_fetch_xor: 627 Op = llvm::AtomicRMWInst::Xor; 628 break; 629 630 case AtomicExpr::AO__atomic_nand_fetch: 631 PostOp = llvm::Instruction::And; // the NOT is special cased below 632 LLVM_FALLTHROUGH; 633 case AtomicExpr::AO__atomic_fetch_nand: 634 Op = llvm::AtomicRMWInst::Nand; 635 break; 636 } 637 638 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 639 llvm::AtomicRMWInst *RMWI = 640 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 641 RMWI->setVolatile(E->isVolatile()); 642 643 // For __atomic_*_fetch operations, perform the operation again to 644 // determine the value which was written. 645 llvm::Value *Result = RMWI; 646 if (PostOp) 647 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 648 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 649 Result = CGF.Builder.CreateNot(Result); 650 CGF.Builder.CreateStore(Result, Dest); 651 } 652 653 // This function emits any expression (scalar, complex, or aggregate) 654 // into a temporary alloca. 655 static Address 656 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 657 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 658 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 659 /*Init*/ true); 660 return DeclPtr; 661 } 662 663 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 664 Address Ptr, Address Val1, Address Val2, 665 llvm::Value *IsWeak, llvm::Value *FailureOrder, 666 uint64_t Size, llvm::AtomicOrdering Order, 667 llvm::Value *Scope) { 668 auto ScopeModel = Expr->getScopeModel(); 669 670 // LLVM atomic instructions always have synch scope. If clang atomic 671 // expression has no scope operand, use default LLVM synch scope. 672 if (!ScopeModel) { 673 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 674 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 675 return; 676 } 677 678 // Handle constant scope. 679 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 680 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 681 ScopeModel->map(SC->getZExtValue()), CGF.CGM.getLLVMContext()); 682 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 683 Order, SCID); 684 return; 685 } 686 687 // Handle non-constant scope. 688 auto &Builder = CGF.Builder; 689 auto Scopes = ScopeModel->getRuntimeValues(); 690 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 691 for (auto S : Scopes) 692 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 693 694 llvm::BasicBlock *ContBB = 695 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 696 697 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 698 // If unsupported synch scope is encountered at run time, assume a fallback 699 // synch scope value. 700 auto FallBack = ScopeModel->getFallBackValue(); 701 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 702 for (auto S : Scopes) { 703 auto *B = BB[S]; 704 if (S != FallBack) 705 SI->addCase(Builder.getInt32(S), B); 706 707 Builder.SetInsertPoint(B); 708 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 709 Order, 710 CGF.getTargetHooks().getLLVMSyncScopeID(ScopeModel->map(S), 711 CGF.getLLVMContext())); 712 Builder.CreateBr(ContBB); 713 } 714 715 Builder.SetInsertPoint(ContBB); 716 } 717 718 static void 719 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 720 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 721 SourceLocation Loc, CharUnits SizeInChars) { 722 if (UseOptimizedLibcall) { 723 // Load value and pass it to the function directly. 724 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 725 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 726 ValTy = 727 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 728 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 729 SizeInBits)->getPointerTo(); 730 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 731 Val = CGF.EmitLoadOfScalar(Ptr, false, 732 CGF.getContext().getPointerType(ValTy), 733 Loc); 734 // Coerce the value into an appropriately sized integer type. 735 Args.add(RValue::get(Val), ValTy); 736 } else { 737 // Non-optimized functions always take a reference. 738 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 739 CGF.getContext().VoidPtrTy); 740 } 741 } 742 743 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 744 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 745 QualType MemTy = AtomicTy; 746 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 747 MemTy = AT->getValueType(); 748 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 749 750 Address Val1 = Address::invalid(); 751 Address Val2 = Address::invalid(); 752 Address Dest = Address::invalid(); 753 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 754 755 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 756 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 757 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 758 EmitAtomicInit(E->getVal1(), lvalue); 759 return RValue::get(nullptr); 760 } 761 762 CharUnits sizeChars, alignChars; 763 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); 764 uint64_t Size = sizeChars.getQuantity(); 765 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 766 bool UseLibcall = ((Ptr.getAlignment() % sizeChars) != 0 || 767 getContext().toBits(sizeChars) > MaxInlineWidthInBits); 768 769 if (UseLibcall) 770 CGM.getDiags().Report(E->getLocStart(), diag::warn_atomic_op_misaligned); 771 772 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 773 llvm::Value *Scope = 774 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 775 776 switch (E->getOp()) { 777 case AtomicExpr::AO__c11_atomic_init: 778 case AtomicExpr::AO__opencl_atomic_init: 779 llvm_unreachable("Already handled above with EmitAtomicInit!"); 780 781 case AtomicExpr::AO__c11_atomic_load: 782 case AtomicExpr::AO__opencl_atomic_load: 783 case AtomicExpr::AO__atomic_load_n: 784 break; 785 786 case AtomicExpr::AO__atomic_load: 787 Dest = EmitPointerWithAlignment(E->getVal1()); 788 break; 789 790 case AtomicExpr::AO__atomic_store: 791 Val1 = EmitPointerWithAlignment(E->getVal1()); 792 break; 793 794 case AtomicExpr::AO__atomic_exchange: 795 Val1 = EmitPointerWithAlignment(E->getVal1()); 796 Dest = EmitPointerWithAlignment(E->getVal2()); 797 break; 798 799 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 800 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 801 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 802 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 803 case AtomicExpr::AO__atomic_compare_exchange_n: 804 case AtomicExpr::AO__atomic_compare_exchange: 805 Val1 = EmitPointerWithAlignment(E->getVal1()); 806 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 807 Val2 = EmitPointerWithAlignment(E->getVal2()); 808 else 809 Val2 = EmitValToTemp(*this, E->getVal2()); 810 OrderFail = EmitScalarExpr(E->getOrderFail()); 811 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 812 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 813 IsWeak = EmitScalarExpr(E->getWeak()); 814 break; 815 816 case AtomicExpr::AO__c11_atomic_fetch_add: 817 case AtomicExpr::AO__c11_atomic_fetch_sub: 818 case AtomicExpr::AO__opencl_atomic_fetch_add: 819 case AtomicExpr::AO__opencl_atomic_fetch_sub: 820 if (MemTy->isPointerType()) { 821 // For pointer arithmetic, we're required to do a bit of math: 822 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 823 // ... but only for the C11 builtins. The GNU builtins expect the 824 // user to multiply by sizeof(T). 825 QualType Val1Ty = E->getVal1()->getType(); 826 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 827 CharUnits PointeeIncAmt = 828 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 829 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 830 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 831 Val1 = Temp; 832 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 833 break; 834 } 835 LLVM_FALLTHROUGH; 836 case AtomicExpr::AO__atomic_fetch_add: 837 case AtomicExpr::AO__atomic_fetch_sub: 838 case AtomicExpr::AO__atomic_add_fetch: 839 case AtomicExpr::AO__atomic_sub_fetch: 840 case AtomicExpr::AO__c11_atomic_store: 841 case AtomicExpr::AO__c11_atomic_exchange: 842 case AtomicExpr::AO__opencl_atomic_store: 843 case AtomicExpr::AO__opencl_atomic_exchange: 844 case AtomicExpr::AO__atomic_store_n: 845 case AtomicExpr::AO__atomic_exchange_n: 846 case AtomicExpr::AO__c11_atomic_fetch_and: 847 case AtomicExpr::AO__c11_atomic_fetch_or: 848 case AtomicExpr::AO__c11_atomic_fetch_xor: 849 case AtomicExpr::AO__opencl_atomic_fetch_and: 850 case AtomicExpr::AO__opencl_atomic_fetch_or: 851 case AtomicExpr::AO__opencl_atomic_fetch_xor: 852 case AtomicExpr::AO__opencl_atomic_fetch_min: 853 case AtomicExpr::AO__opencl_atomic_fetch_max: 854 case AtomicExpr::AO__atomic_fetch_and: 855 case AtomicExpr::AO__atomic_fetch_or: 856 case AtomicExpr::AO__atomic_fetch_xor: 857 case AtomicExpr::AO__atomic_fetch_nand: 858 case AtomicExpr::AO__atomic_and_fetch: 859 case AtomicExpr::AO__atomic_or_fetch: 860 case AtomicExpr::AO__atomic_xor_fetch: 861 case AtomicExpr::AO__atomic_nand_fetch: 862 Val1 = EmitValToTemp(*this, E->getVal1()); 863 break; 864 } 865 866 QualType RValTy = E->getType().getUnqualifiedType(); 867 868 // The inlined atomics only function on iN types, where N is a power of 2. We 869 // need to make sure (via temporaries if necessary) that all incoming values 870 // are compatible. 871 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 872 AtomicInfo Atomics(*this, AtomicVal); 873 874 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 875 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); 876 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); 877 if (Dest.isValid()) 878 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 879 else if (E->isCmpXChg()) 880 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 881 else if (!RValTy->isVoidType()) 882 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); 883 884 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 885 if (UseLibcall) { 886 bool UseOptimizedLibcall = false; 887 switch (E->getOp()) { 888 case AtomicExpr::AO__c11_atomic_init: 889 case AtomicExpr::AO__opencl_atomic_init: 890 llvm_unreachable("Already handled above with EmitAtomicInit!"); 891 892 case AtomicExpr::AO__c11_atomic_fetch_add: 893 case AtomicExpr::AO__opencl_atomic_fetch_add: 894 case AtomicExpr::AO__atomic_fetch_add: 895 case AtomicExpr::AO__c11_atomic_fetch_and: 896 case AtomicExpr::AO__opencl_atomic_fetch_and: 897 case AtomicExpr::AO__atomic_fetch_and: 898 case AtomicExpr::AO__c11_atomic_fetch_or: 899 case AtomicExpr::AO__opencl_atomic_fetch_or: 900 case AtomicExpr::AO__atomic_fetch_or: 901 case AtomicExpr::AO__atomic_fetch_nand: 902 case AtomicExpr::AO__c11_atomic_fetch_sub: 903 case AtomicExpr::AO__opencl_atomic_fetch_sub: 904 case AtomicExpr::AO__atomic_fetch_sub: 905 case AtomicExpr::AO__c11_atomic_fetch_xor: 906 case AtomicExpr::AO__opencl_atomic_fetch_xor: 907 case AtomicExpr::AO__opencl_atomic_fetch_min: 908 case AtomicExpr::AO__opencl_atomic_fetch_max: 909 case AtomicExpr::AO__atomic_fetch_xor: 910 case AtomicExpr::AO__atomic_add_fetch: 911 case AtomicExpr::AO__atomic_and_fetch: 912 case AtomicExpr::AO__atomic_nand_fetch: 913 case AtomicExpr::AO__atomic_or_fetch: 914 case AtomicExpr::AO__atomic_sub_fetch: 915 case AtomicExpr::AO__atomic_xor_fetch: 916 // For these, only library calls for certain sizes exist. 917 UseOptimizedLibcall = true; 918 break; 919 920 case AtomicExpr::AO__c11_atomic_load: 921 case AtomicExpr::AO__c11_atomic_store: 922 case AtomicExpr::AO__c11_atomic_exchange: 923 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 924 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 925 case AtomicExpr::AO__opencl_atomic_load: 926 case AtomicExpr::AO__opencl_atomic_store: 927 case AtomicExpr::AO__opencl_atomic_exchange: 928 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 929 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 930 case AtomicExpr::AO__atomic_load_n: 931 case AtomicExpr::AO__atomic_load: 932 case AtomicExpr::AO__atomic_store_n: 933 case AtomicExpr::AO__atomic_store: 934 case AtomicExpr::AO__atomic_exchange_n: 935 case AtomicExpr::AO__atomic_exchange: 936 case AtomicExpr::AO__atomic_compare_exchange_n: 937 case AtomicExpr::AO__atomic_compare_exchange: 938 // Only use optimized library calls for sizes for which they exist. 939 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 940 UseOptimizedLibcall = true; 941 break; 942 } 943 944 CallArgList Args; 945 if (!UseOptimizedLibcall) { 946 // For non-optimized library calls, the size is the first parameter 947 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 948 getContext().getSizeType()); 949 } 950 // Atomic address is the first or second parameter 951 // The OpenCL atomic library functions only accept pointer arguments to 952 // generic address space. 953 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 954 if (!E->isOpenCL()) 955 return V; 956 auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace(); 957 if (AS == LangAS::opencl_generic) 958 return V; 959 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 960 auto T = V->getType(); 961 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS); 962 963 return getTargetHooks().performAddrSpaceCast( 964 *this, V, AS, LangAS::opencl_generic, DestType, false); 965 }; 966 967 Args.add(RValue::get(CastToGenericAddrSpace( 968 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), 969 getContext().VoidPtrTy); 970 971 std::string LibCallName; 972 QualType LoweredMemTy = 973 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 974 QualType RetTy; 975 bool HaveRetTy = false; 976 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 977 switch (E->getOp()) { 978 case AtomicExpr::AO__c11_atomic_init: 979 case AtomicExpr::AO__opencl_atomic_init: 980 llvm_unreachable("Already handled!"); 981 982 // There is only one libcall for compare an exchange, because there is no 983 // optimisation benefit possible from a libcall version of a weak compare 984 // and exchange. 985 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 986 // void *desired, int success, int failure) 987 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 988 // int success, int failure) 989 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 990 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 991 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 992 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 993 case AtomicExpr::AO__atomic_compare_exchange: 994 case AtomicExpr::AO__atomic_compare_exchange_n: 995 LibCallName = "__atomic_compare_exchange"; 996 RetTy = getContext().BoolTy; 997 HaveRetTy = true; 998 Args.add( 999 RValue::get(CastToGenericAddrSpace( 1000 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), 1001 getContext().VoidPtrTy); 1002 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 1003 MemTy, E->getExprLoc(), sizeChars); 1004 Args.add(RValue::get(Order), getContext().IntTy); 1005 Order = OrderFail; 1006 break; 1007 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1008 // int order) 1009 // T __atomic_exchange_N(T *mem, T val, int order) 1010 case AtomicExpr::AO__c11_atomic_exchange: 1011 case AtomicExpr::AO__opencl_atomic_exchange: 1012 case AtomicExpr::AO__atomic_exchange_n: 1013 case AtomicExpr::AO__atomic_exchange: 1014 LibCallName = "__atomic_exchange"; 1015 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1016 MemTy, E->getExprLoc(), sizeChars); 1017 break; 1018 // void __atomic_store(size_t size, void *mem, void *val, int order) 1019 // void __atomic_store_N(T *mem, T val, int order) 1020 case AtomicExpr::AO__c11_atomic_store: 1021 case AtomicExpr::AO__opencl_atomic_store: 1022 case AtomicExpr::AO__atomic_store: 1023 case AtomicExpr::AO__atomic_store_n: 1024 LibCallName = "__atomic_store"; 1025 RetTy = getContext().VoidTy; 1026 HaveRetTy = true; 1027 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1028 MemTy, E->getExprLoc(), sizeChars); 1029 break; 1030 // void __atomic_load(size_t size, void *mem, void *return, int order) 1031 // T __atomic_load_N(T *mem, int order) 1032 case AtomicExpr::AO__c11_atomic_load: 1033 case AtomicExpr::AO__opencl_atomic_load: 1034 case AtomicExpr::AO__atomic_load: 1035 case AtomicExpr::AO__atomic_load_n: 1036 LibCallName = "__atomic_load"; 1037 break; 1038 // T __atomic_add_fetch_N(T *mem, T val, int order) 1039 // T __atomic_fetch_add_N(T *mem, T val, int order) 1040 case AtomicExpr::AO__atomic_add_fetch: 1041 PostOp = llvm::Instruction::Add; 1042 LLVM_FALLTHROUGH; 1043 case AtomicExpr::AO__c11_atomic_fetch_add: 1044 case AtomicExpr::AO__opencl_atomic_fetch_add: 1045 case AtomicExpr::AO__atomic_fetch_add: 1046 LibCallName = "__atomic_fetch_add"; 1047 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1048 LoweredMemTy, E->getExprLoc(), sizeChars); 1049 break; 1050 // T __atomic_and_fetch_N(T *mem, T val, int order) 1051 // T __atomic_fetch_and_N(T *mem, T val, int order) 1052 case AtomicExpr::AO__atomic_and_fetch: 1053 PostOp = llvm::Instruction::And; 1054 LLVM_FALLTHROUGH; 1055 case AtomicExpr::AO__c11_atomic_fetch_and: 1056 case AtomicExpr::AO__opencl_atomic_fetch_and: 1057 case AtomicExpr::AO__atomic_fetch_and: 1058 LibCallName = "__atomic_fetch_and"; 1059 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1060 MemTy, E->getExprLoc(), sizeChars); 1061 break; 1062 // T __atomic_or_fetch_N(T *mem, T val, int order) 1063 // T __atomic_fetch_or_N(T *mem, T val, int order) 1064 case AtomicExpr::AO__atomic_or_fetch: 1065 PostOp = llvm::Instruction::Or; 1066 LLVM_FALLTHROUGH; 1067 case AtomicExpr::AO__c11_atomic_fetch_or: 1068 case AtomicExpr::AO__opencl_atomic_fetch_or: 1069 case AtomicExpr::AO__atomic_fetch_or: 1070 LibCallName = "__atomic_fetch_or"; 1071 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1072 MemTy, E->getExprLoc(), sizeChars); 1073 break; 1074 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1075 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1076 case AtomicExpr::AO__atomic_sub_fetch: 1077 PostOp = llvm::Instruction::Sub; 1078 LLVM_FALLTHROUGH; 1079 case AtomicExpr::AO__c11_atomic_fetch_sub: 1080 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1081 case AtomicExpr::AO__atomic_fetch_sub: 1082 LibCallName = "__atomic_fetch_sub"; 1083 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1084 LoweredMemTy, E->getExprLoc(), sizeChars); 1085 break; 1086 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1087 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1088 case AtomicExpr::AO__atomic_xor_fetch: 1089 PostOp = llvm::Instruction::Xor; 1090 LLVM_FALLTHROUGH; 1091 case AtomicExpr::AO__c11_atomic_fetch_xor: 1092 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1093 case AtomicExpr::AO__atomic_fetch_xor: 1094 LibCallName = "__atomic_fetch_xor"; 1095 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1096 MemTy, E->getExprLoc(), sizeChars); 1097 break; 1098 case AtomicExpr::AO__opencl_atomic_fetch_min: 1099 LibCallName = E->getValueType()->isSignedIntegerType() 1100 ? "__atomic_fetch_min" 1101 : "__atomic_fetch_umin"; 1102 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1103 LoweredMemTy, E->getExprLoc(), sizeChars); 1104 break; 1105 case AtomicExpr::AO__opencl_atomic_fetch_max: 1106 LibCallName = E->getValueType()->isSignedIntegerType() 1107 ? "__atomic_fetch_max" 1108 : "__atomic_fetch_umax"; 1109 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1110 LoweredMemTy, E->getExprLoc(), sizeChars); 1111 break; 1112 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1113 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1114 case AtomicExpr::AO__atomic_nand_fetch: 1115 PostOp = llvm::Instruction::And; // the NOT is special cased below 1116 LLVM_FALLTHROUGH; 1117 case AtomicExpr::AO__atomic_fetch_nand: 1118 LibCallName = "__atomic_fetch_nand"; 1119 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1120 MemTy, E->getExprLoc(), sizeChars); 1121 break; 1122 } 1123 1124 if (E->isOpenCL()) { 1125 LibCallName = std::string("__opencl") + 1126 StringRef(LibCallName).drop_front(1).str(); 1127 1128 } 1129 // Optimized functions have the size in their name. 1130 if (UseOptimizedLibcall) 1131 LibCallName += "_" + llvm::utostr(Size); 1132 // By default, assume we return a value of the atomic type. 1133 if (!HaveRetTy) { 1134 if (UseOptimizedLibcall) { 1135 // Value is returned directly. 1136 // The function returns an appropriately sized integer type. 1137 RetTy = getContext().getIntTypeForBitwidth( 1138 getContext().toBits(sizeChars), /*Signed=*/false); 1139 } else { 1140 // Value is returned through parameter before the order. 1141 RetTy = getContext().VoidTy; 1142 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 1143 getContext().VoidPtrTy); 1144 } 1145 } 1146 // order is always the last parameter 1147 Args.add(RValue::get(Order), 1148 getContext().IntTy); 1149 if (E->isOpenCL()) 1150 Args.add(RValue::get(Scope), getContext().IntTy); 1151 1152 // PostOp is only needed for the atomic_*_fetch operations, and 1153 // thus is only needed for and implemented in the 1154 // UseOptimizedLibcall codepath. 1155 assert(UseOptimizedLibcall || !PostOp); 1156 1157 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1158 // The value is returned directly from the libcall. 1159 if (E->isCmpXChg()) 1160 return Res; 1161 1162 // The value is returned directly for optimized libcalls but the expr 1163 // provided an out-param. 1164 if (UseOptimizedLibcall && Res.getScalarVal()) { 1165 llvm::Value *ResVal = Res.getScalarVal(); 1166 if (PostOp) { 1167 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1168 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1169 } 1170 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1171 ResVal = Builder.CreateNot(ResVal); 1172 1173 Builder.CreateStore( 1174 ResVal, 1175 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 1176 } 1177 1178 if (RValTy->isVoidType()) 1179 return RValue::get(nullptr); 1180 1181 return convertTempToRValue( 1182 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1183 RValTy, E->getExprLoc()); 1184 } 1185 1186 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1187 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1188 E->getOp() == AtomicExpr::AO__atomic_store || 1189 E->getOp() == AtomicExpr::AO__atomic_store_n; 1190 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1191 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1192 E->getOp() == AtomicExpr::AO__atomic_load || 1193 E->getOp() == AtomicExpr::AO__atomic_load_n; 1194 1195 if (isa<llvm::ConstantInt>(Order)) { 1196 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1197 // We should not ever get to a case where the ordering isn't a valid C ABI 1198 // value, but it's hard to enforce that in general. 1199 if (llvm::isValidAtomicOrderingCABI(ord)) 1200 switch ((llvm::AtomicOrderingCABI)ord) { 1201 case llvm::AtomicOrderingCABI::relaxed: 1202 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1203 llvm::AtomicOrdering::Monotonic, Scope); 1204 break; 1205 case llvm::AtomicOrderingCABI::consume: 1206 case llvm::AtomicOrderingCABI::acquire: 1207 if (IsStore) 1208 break; // Avoid crashing on code with undefined behavior 1209 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1210 llvm::AtomicOrdering::Acquire, Scope); 1211 break; 1212 case llvm::AtomicOrderingCABI::release: 1213 if (IsLoad) 1214 break; // Avoid crashing on code with undefined behavior 1215 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1216 llvm::AtomicOrdering::Release, Scope); 1217 break; 1218 case llvm::AtomicOrderingCABI::acq_rel: 1219 if (IsLoad || IsStore) 1220 break; // Avoid crashing on code with undefined behavior 1221 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1222 llvm::AtomicOrdering::AcquireRelease, Scope); 1223 break; 1224 case llvm::AtomicOrderingCABI::seq_cst: 1225 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1226 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1227 break; 1228 } 1229 if (RValTy->isVoidType()) 1230 return RValue::get(nullptr); 1231 1232 return convertTempToRValue( 1233 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1234 Dest.getAddressSpace())), 1235 RValTy, E->getExprLoc()); 1236 } 1237 1238 // Long case, when Order isn't obviously constant. 1239 1240 // Create all the relevant BB's 1241 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1242 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1243 *SeqCstBB = nullptr; 1244 MonotonicBB = createBasicBlock("monotonic", CurFn); 1245 if (!IsStore) 1246 AcquireBB = createBasicBlock("acquire", CurFn); 1247 if (!IsLoad) 1248 ReleaseBB = createBasicBlock("release", CurFn); 1249 if (!IsLoad && !IsStore) 1250 AcqRelBB = createBasicBlock("acqrel", CurFn); 1251 SeqCstBB = createBasicBlock("seqcst", CurFn); 1252 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1253 1254 // Create the switch for the split 1255 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1256 // doesn't matter unless someone is crazy enough to use something that 1257 // doesn't fold to a constant for the ordering. 1258 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1259 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1260 1261 // Emit all the different atomics 1262 Builder.SetInsertPoint(MonotonicBB); 1263 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1264 llvm::AtomicOrdering::Monotonic, Scope); 1265 Builder.CreateBr(ContBB); 1266 if (!IsStore) { 1267 Builder.SetInsertPoint(AcquireBB); 1268 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1269 llvm::AtomicOrdering::Acquire, Scope); 1270 Builder.CreateBr(ContBB); 1271 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1272 AcquireBB); 1273 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1274 AcquireBB); 1275 } 1276 if (!IsLoad) { 1277 Builder.SetInsertPoint(ReleaseBB); 1278 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1279 llvm::AtomicOrdering::Release, Scope); 1280 Builder.CreateBr(ContBB); 1281 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1282 ReleaseBB); 1283 } 1284 if (!IsLoad && !IsStore) { 1285 Builder.SetInsertPoint(AcqRelBB); 1286 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1287 llvm::AtomicOrdering::AcquireRelease, Scope); 1288 Builder.CreateBr(ContBB); 1289 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1290 AcqRelBB); 1291 } 1292 Builder.SetInsertPoint(SeqCstBB); 1293 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1294 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1295 Builder.CreateBr(ContBB); 1296 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1297 SeqCstBB); 1298 1299 // Cleanup and return 1300 Builder.SetInsertPoint(ContBB); 1301 if (RValTy->isVoidType()) 1302 return RValue::get(nullptr); 1303 1304 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1305 return convertTempToRValue( 1306 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1307 Dest.getAddressSpace())), 1308 RValTy, E->getExprLoc()); 1309 } 1310 1311 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1312 unsigned addrspace = 1313 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 1314 llvm::IntegerType *ty = 1315 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1316 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 1317 } 1318 1319 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1320 llvm::Type *Ty = Addr.getElementType(); 1321 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1322 if (SourceSizeInBits != AtomicSizeInBits) { 1323 Address Tmp = CreateTempAlloca(); 1324 CGF.Builder.CreateMemCpy(Tmp, Addr, 1325 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1326 Addr = Tmp; 1327 } 1328 1329 return emitCastToAtomicIntPointer(Addr); 1330 } 1331 1332 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1333 AggValueSlot resultSlot, 1334 SourceLocation loc, 1335 bool asValue) const { 1336 if (LVal.isSimple()) { 1337 if (EvaluationKind == TEK_Aggregate) 1338 return resultSlot.asRValue(); 1339 1340 // Drill into the padding structure if we have one. 1341 if (hasPadding()) 1342 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 1343 1344 // Otherwise, just convert the temporary to an r-value using the 1345 // normal conversion routine. 1346 return CGF.convertTempToRValue(addr, getValueType(), loc); 1347 } 1348 if (!asValue) 1349 // Get RValue from temp memory as atomic for non-simple lvalues 1350 return RValue::get(CGF.Builder.CreateLoad(addr)); 1351 if (LVal.isBitField()) 1352 return CGF.EmitLoadOfBitfieldLValue( 1353 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1354 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1355 if (LVal.isVectorElt()) 1356 return CGF.EmitLoadOfLValue( 1357 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1358 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1359 assert(LVal.isExtVectorElt()); 1360 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1361 addr, LVal.getExtVectorElts(), LVal.getType(), 1362 LVal.getBaseInfo(), TBAAAccessInfo())); 1363 } 1364 1365 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1366 AggValueSlot ResultSlot, 1367 SourceLocation Loc, 1368 bool AsValue) const { 1369 // Try not to in some easy cases. 1370 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1371 if (getEvaluationKind() == TEK_Scalar && 1372 (((!LVal.isBitField() || 1373 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1374 !hasPadding()) || 1375 !AsValue)) { 1376 auto *ValTy = AsValue 1377 ? CGF.ConvertTypeForMem(ValueTy) 1378 : getAtomicAddress().getType()->getPointerElementType(); 1379 if (ValTy->isIntegerTy()) { 1380 assert(IntVal->getType() == ValTy && "Different integer types."); 1381 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1382 } else if (ValTy->isPointerTy()) 1383 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1384 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1385 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1386 } 1387 1388 // Create a temporary. This needs to be big enough to hold the 1389 // atomic integer. 1390 Address Temp = Address::invalid(); 1391 bool TempIsVolatile = false; 1392 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1393 assert(!ResultSlot.isIgnored()); 1394 Temp = ResultSlot.getAddress(); 1395 TempIsVolatile = ResultSlot.isVolatile(); 1396 } else { 1397 Temp = CreateTempAlloca(); 1398 } 1399 1400 // Slam the integer into the temporary. 1401 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1402 CGF.Builder.CreateStore(IntVal, CastTemp) 1403 ->setVolatile(TempIsVolatile); 1404 1405 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1406 } 1407 1408 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1409 llvm::AtomicOrdering AO, bool) { 1410 // void __atomic_load(size_t size, void *mem, void *return, int order); 1411 CallArgList Args; 1412 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1413 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1414 CGF.getContext().VoidPtrTy); 1415 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1416 CGF.getContext().VoidPtrTy); 1417 Args.add( 1418 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1419 CGF.getContext().IntTy); 1420 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1421 } 1422 1423 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1424 bool IsVolatile) { 1425 // Okay, we're doing this natively. 1426 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1427 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1428 Load->setAtomic(AO); 1429 1430 // Other decoration. 1431 if (IsVolatile) 1432 Load->setVolatile(true); 1433 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1434 return Load; 1435 } 1436 1437 /// An LValue is a candidate for having its loads and stores be made atomic if 1438 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1439 /// performing such an operation can be performed without a libcall. 1440 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1441 if (!CGM.getCodeGenOpts().MSVolatile) return false; 1442 AtomicInfo AI(*this, LV); 1443 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1444 // An atomic is inline if we don't need to use a libcall. 1445 bool AtomicIsInline = !AI.shouldUseLibcall(); 1446 // MSVC doesn't seem to do this for types wider than a pointer. 1447 if (getContext().getTypeSize(LV.getType()) > 1448 getContext().getTypeSize(getContext().getIntPtrType())) 1449 return false; 1450 return IsVolatile && AtomicIsInline; 1451 } 1452 1453 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1454 AggValueSlot Slot) { 1455 llvm::AtomicOrdering AO; 1456 bool IsVolatile = LV.isVolatileQualified(); 1457 if (LV.getType()->isAtomicType()) { 1458 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1459 } else { 1460 AO = llvm::AtomicOrdering::Acquire; 1461 IsVolatile = true; 1462 } 1463 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1464 } 1465 1466 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1467 bool AsValue, llvm::AtomicOrdering AO, 1468 bool IsVolatile) { 1469 // Check whether we should use a library call. 1470 if (shouldUseLibcall()) { 1471 Address TempAddr = Address::invalid(); 1472 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1473 assert(getEvaluationKind() == TEK_Aggregate); 1474 TempAddr = ResultSlot.getAddress(); 1475 } else 1476 TempAddr = CreateTempAlloca(); 1477 1478 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1479 1480 // Okay, turn that back into the original value or whole atomic (for 1481 // non-simple lvalues) type. 1482 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1483 } 1484 1485 // Okay, we're doing this natively. 1486 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1487 1488 // If we're ignoring an aggregate return, don't do anything. 1489 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1490 return RValue::getAggregate(Address::invalid(), false); 1491 1492 // Okay, turn that back into the original value or atomic (for non-simple 1493 // lvalues) type. 1494 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1495 } 1496 1497 /// Emit a load from an l-value of atomic type. Note that the r-value 1498 /// we produce is an r-value of the atomic *value* type. 1499 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1500 llvm::AtomicOrdering AO, bool IsVolatile, 1501 AggValueSlot resultSlot) { 1502 AtomicInfo Atomics(*this, src); 1503 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1504 IsVolatile); 1505 } 1506 1507 /// Copy an r-value into memory as part of storing to an atomic type. 1508 /// This needs to create a bit-pattern suitable for atomic operations. 1509 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1510 assert(LVal.isSimple()); 1511 // If we have an r-value, the rvalue should be of the atomic type, 1512 // which means that the caller is responsible for having zeroed 1513 // any padding. Just do an aggregate copy of that type. 1514 if (rvalue.isAggregate()) { 1515 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1516 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1517 getAtomicType()); 1518 bool IsVolatile = rvalue.isVolatileQualified() || 1519 LVal.isVolatileQualified(); 1520 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1521 AggValueSlot::DoesNotOverlap, IsVolatile); 1522 return; 1523 } 1524 1525 // Okay, otherwise we're copying stuff. 1526 1527 // Zero out the buffer if necessary. 1528 emitMemSetZeroIfNecessary(); 1529 1530 // Drill past the padding if present. 1531 LValue TempLVal = projectValue(); 1532 1533 // Okay, store the rvalue in. 1534 if (rvalue.isScalar()) { 1535 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1536 } else { 1537 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1538 } 1539 } 1540 1541 1542 /// Materialize an r-value into memory for the purposes of storing it 1543 /// to an atomic type. 1544 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1545 // Aggregate r-values are already in memory, and EmitAtomicStore 1546 // requires them to be values of the atomic type. 1547 if (rvalue.isAggregate()) 1548 return rvalue.getAggregateAddress(); 1549 1550 // Otherwise, make a temporary and materialize into it. 1551 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1552 AtomicInfo Atomics(CGF, TempLV); 1553 Atomics.emitCopyIntoMemory(rvalue); 1554 return TempLV.getAddress(); 1555 } 1556 1557 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1558 // If we've got a scalar value of the right size, try to avoid going 1559 // through memory. 1560 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1561 llvm::Value *Value = RVal.getScalarVal(); 1562 if (isa<llvm::IntegerType>(Value->getType())) 1563 return CGF.EmitToMemory(Value, ValueTy); 1564 else { 1565 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1566 CGF.getLLVMContext(), 1567 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1568 if (isa<llvm::PointerType>(Value->getType())) 1569 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1570 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1571 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1572 } 1573 } 1574 // Otherwise, we need to go through memory. 1575 // Put the r-value in memory. 1576 Address Addr = materializeRValue(RVal); 1577 1578 // Cast the temporary to the atomic int type and pull a value out. 1579 Addr = emitCastToAtomicIntPointer(Addr); 1580 return CGF.Builder.CreateLoad(Addr); 1581 } 1582 1583 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1584 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1585 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1586 // Do the atomic store. 1587 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1588 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1589 ExpectedVal, DesiredVal, 1590 Success, Failure); 1591 // Other decoration. 1592 Inst->setVolatile(LVal.isVolatileQualified()); 1593 Inst->setWeak(IsWeak); 1594 1595 // Okay, turn that back into the original value type. 1596 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1597 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1598 return std::make_pair(PreviousVal, SuccessFailureVal); 1599 } 1600 1601 llvm::Value * 1602 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1603 llvm::Value *DesiredAddr, 1604 llvm::AtomicOrdering Success, 1605 llvm::AtomicOrdering Failure) { 1606 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1607 // void *desired, int success, int failure); 1608 CallArgList Args; 1609 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1610 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1611 CGF.getContext().VoidPtrTy); 1612 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1613 CGF.getContext().VoidPtrTy); 1614 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1615 CGF.getContext().VoidPtrTy); 1616 Args.add(RValue::get( 1617 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1618 CGF.getContext().IntTy); 1619 Args.add(RValue::get( 1620 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1621 CGF.getContext().IntTy); 1622 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1623 CGF.getContext().BoolTy, Args); 1624 1625 return SuccessFailureRVal.getScalarVal(); 1626 } 1627 1628 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1629 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1630 llvm::AtomicOrdering Failure, bool IsWeak) { 1631 if (isStrongerThan(Failure, Success)) 1632 // Don't assert on undefined behavior "failure argument shall be no stronger 1633 // than the success argument". 1634 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 1635 1636 // Check whether we should use a library call. 1637 if (shouldUseLibcall()) { 1638 // Produce a source address. 1639 Address ExpectedAddr = materializeRValue(Expected); 1640 Address DesiredAddr = materializeRValue(Desired); 1641 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1642 DesiredAddr.getPointer(), 1643 Success, Failure); 1644 return std::make_pair( 1645 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1646 SourceLocation(), /*AsValue=*/false), 1647 Res); 1648 } 1649 1650 // If we've got a scalar value of the right size, try to avoid going 1651 // through memory. 1652 auto *ExpectedVal = convertRValueToInt(Expected); 1653 auto *DesiredVal = convertRValueToInt(Desired); 1654 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1655 Failure, IsWeak); 1656 return std::make_pair( 1657 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1658 SourceLocation(), /*AsValue=*/false), 1659 Res.second); 1660 } 1661 1662 static void 1663 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1664 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1665 Address DesiredAddr) { 1666 RValue UpRVal; 1667 LValue AtomicLVal = Atomics.getAtomicLValue(); 1668 LValue DesiredLVal; 1669 if (AtomicLVal.isSimple()) { 1670 UpRVal = OldRVal; 1671 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1672 } else { 1673 // Build new lvalue for temp address 1674 Address Ptr = Atomics.materializeRValue(OldRVal); 1675 LValue UpdateLVal; 1676 if (AtomicLVal.isBitField()) { 1677 UpdateLVal = 1678 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1679 AtomicLVal.getType(), 1680 AtomicLVal.getBaseInfo(), 1681 AtomicLVal.getTBAAInfo()); 1682 DesiredLVal = 1683 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1684 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1685 AtomicLVal.getTBAAInfo()); 1686 } else if (AtomicLVal.isVectorElt()) { 1687 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1688 AtomicLVal.getType(), 1689 AtomicLVal.getBaseInfo(), 1690 AtomicLVal.getTBAAInfo()); 1691 DesiredLVal = LValue::MakeVectorElt( 1692 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1693 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1694 } else { 1695 assert(AtomicLVal.isExtVectorElt()); 1696 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1697 AtomicLVal.getType(), 1698 AtomicLVal.getBaseInfo(), 1699 AtomicLVal.getTBAAInfo()); 1700 DesiredLVal = LValue::MakeExtVectorElt( 1701 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1702 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1703 } 1704 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1705 } 1706 // Store new value in the corresponding memory area 1707 RValue NewRVal = UpdateOp(UpRVal); 1708 if (NewRVal.isScalar()) { 1709 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1710 } else { 1711 assert(NewRVal.isComplex()); 1712 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1713 /*isInit=*/false); 1714 } 1715 } 1716 1717 void AtomicInfo::EmitAtomicUpdateLibcall( 1718 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1719 bool IsVolatile) { 1720 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1721 1722 Address ExpectedAddr = CreateTempAlloca(); 1723 1724 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1725 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1726 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1727 CGF.EmitBlock(ContBB); 1728 Address DesiredAddr = CreateTempAlloca(); 1729 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1730 requiresMemSetZero(getAtomicAddress().getElementType())) { 1731 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1732 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1733 } 1734 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1735 AggValueSlot::ignored(), 1736 SourceLocation(), /*AsValue=*/false); 1737 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1738 auto *Res = 1739 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1740 DesiredAddr.getPointer(), 1741 AO, Failure); 1742 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1743 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1744 } 1745 1746 void AtomicInfo::EmitAtomicUpdateOp( 1747 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1748 bool IsVolatile) { 1749 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1750 1751 // Do the atomic load. 1752 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1753 // For non-simple lvalues perform compare-and-swap procedure. 1754 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1755 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1756 auto *CurBB = CGF.Builder.GetInsertBlock(); 1757 CGF.EmitBlock(ContBB); 1758 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1759 /*NumReservedValues=*/2); 1760 PHI->addIncoming(OldVal, CurBB); 1761 Address NewAtomicAddr = CreateTempAlloca(); 1762 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1763 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1764 requiresMemSetZero(getAtomicAddress().getElementType())) { 1765 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1766 } 1767 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1768 SourceLocation(), /*AsValue=*/false); 1769 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1770 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1771 // Try to write new value using cmpxchg operation 1772 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1773 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1774 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1775 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1776 } 1777 1778 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1779 RValue UpdateRVal, Address DesiredAddr) { 1780 LValue AtomicLVal = Atomics.getAtomicLValue(); 1781 LValue DesiredLVal; 1782 // Build new lvalue for temp address 1783 if (AtomicLVal.isBitField()) { 1784 DesiredLVal = 1785 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1786 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1787 AtomicLVal.getTBAAInfo()); 1788 } else if (AtomicLVal.isVectorElt()) { 1789 DesiredLVal = 1790 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1791 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1792 AtomicLVal.getTBAAInfo()); 1793 } else { 1794 assert(AtomicLVal.isExtVectorElt()); 1795 DesiredLVal = LValue::MakeExtVectorElt( 1796 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1797 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1798 } 1799 // Store new value in the corresponding memory area 1800 assert(UpdateRVal.isScalar()); 1801 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1802 } 1803 1804 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1805 RValue UpdateRVal, bool IsVolatile) { 1806 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1807 1808 Address ExpectedAddr = CreateTempAlloca(); 1809 1810 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1811 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1812 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1813 CGF.EmitBlock(ContBB); 1814 Address DesiredAddr = CreateTempAlloca(); 1815 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1816 requiresMemSetZero(getAtomicAddress().getElementType())) { 1817 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1818 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1819 } 1820 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1821 auto *Res = 1822 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1823 DesiredAddr.getPointer(), 1824 AO, Failure); 1825 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1826 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1827 } 1828 1829 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1830 bool IsVolatile) { 1831 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1832 1833 // Do the atomic load. 1834 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1835 // For non-simple lvalues perform compare-and-swap procedure. 1836 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1837 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1838 auto *CurBB = CGF.Builder.GetInsertBlock(); 1839 CGF.EmitBlock(ContBB); 1840 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1841 /*NumReservedValues=*/2); 1842 PHI->addIncoming(OldVal, CurBB); 1843 Address NewAtomicAddr = CreateTempAlloca(); 1844 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1845 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1846 requiresMemSetZero(getAtomicAddress().getElementType())) { 1847 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1848 } 1849 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1850 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1851 // Try to write new value using cmpxchg operation 1852 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1853 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1854 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1855 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1856 } 1857 1858 void AtomicInfo::EmitAtomicUpdate( 1859 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1860 bool IsVolatile) { 1861 if (shouldUseLibcall()) { 1862 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1863 } else { 1864 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1865 } 1866 } 1867 1868 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1869 bool IsVolatile) { 1870 if (shouldUseLibcall()) { 1871 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1872 } else { 1873 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1874 } 1875 } 1876 1877 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1878 bool isInit) { 1879 bool IsVolatile = lvalue.isVolatileQualified(); 1880 llvm::AtomicOrdering AO; 1881 if (lvalue.getType()->isAtomicType()) { 1882 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1883 } else { 1884 AO = llvm::AtomicOrdering::Release; 1885 IsVolatile = true; 1886 } 1887 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1888 } 1889 1890 /// Emit a store to an l-value of atomic type. 1891 /// 1892 /// Note that the r-value is expected to be an r-value *of the atomic 1893 /// type*; this means that for aggregate r-values, it should include 1894 /// storage for any padding that was necessary. 1895 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1896 llvm::AtomicOrdering AO, bool IsVolatile, 1897 bool isInit) { 1898 // If this is an aggregate r-value, it should agree in type except 1899 // maybe for address-space qualification. 1900 assert(!rvalue.isAggregate() || 1901 rvalue.getAggregateAddress().getElementType() 1902 == dest.getAddress().getElementType()); 1903 1904 AtomicInfo atomics(*this, dest); 1905 LValue LVal = atomics.getAtomicLValue(); 1906 1907 // If this is an initialization, just put the value there normally. 1908 if (LVal.isSimple()) { 1909 if (isInit) { 1910 atomics.emitCopyIntoMemory(rvalue); 1911 return; 1912 } 1913 1914 // Check whether we should use a library call. 1915 if (atomics.shouldUseLibcall()) { 1916 // Produce a source address. 1917 Address srcAddr = atomics.materializeRValue(rvalue); 1918 1919 // void __atomic_store(size_t size, void *mem, void *val, int order) 1920 CallArgList args; 1921 args.add(RValue::get(atomics.getAtomicSizeValue()), 1922 getContext().getSizeType()); 1923 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 1924 getContext().VoidPtrTy); 1925 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 1926 getContext().VoidPtrTy); 1927 args.add( 1928 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 1929 getContext().IntTy); 1930 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 1931 return; 1932 } 1933 1934 // Okay, we're doing this natively. 1935 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 1936 1937 // Do the atomic store. 1938 Address addr = 1939 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 1940 intValue = Builder.CreateIntCast( 1941 intValue, addr.getElementType(), /*isSigned=*/false); 1942 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 1943 1944 // Initializations don't need to be atomic. 1945 if (!isInit) 1946 store->setAtomic(AO); 1947 1948 // Other decoration. 1949 if (IsVolatile) 1950 store->setVolatile(true); 1951 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 1952 return; 1953 } 1954 1955 // Emit simple atomic update operation. 1956 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 1957 } 1958 1959 /// Emit a compare-and-exchange op for atomic type. 1960 /// 1961 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 1962 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 1963 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 1964 AggValueSlot Slot) { 1965 // If this is an aggregate r-value, it should agree in type except 1966 // maybe for address-space qualification. 1967 assert(!Expected.isAggregate() || 1968 Expected.getAggregateAddress().getElementType() == 1969 Obj.getAddress().getElementType()); 1970 assert(!Desired.isAggregate() || 1971 Desired.getAggregateAddress().getElementType() == 1972 Obj.getAddress().getElementType()); 1973 AtomicInfo Atomics(*this, Obj); 1974 1975 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 1976 IsWeak); 1977 } 1978 1979 void CodeGenFunction::EmitAtomicUpdate( 1980 LValue LVal, llvm::AtomicOrdering AO, 1981 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 1982 AtomicInfo Atomics(*this, LVal); 1983 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 1984 } 1985 1986 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 1987 AtomicInfo atomics(*this, dest); 1988 1989 switch (atomics.getEvaluationKind()) { 1990 case TEK_Scalar: { 1991 llvm::Value *value = EmitScalarExpr(init); 1992 atomics.emitCopyIntoMemory(RValue::get(value)); 1993 return; 1994 } 1995 1996 case TEK_Complex: { 1997 ComplexPairTy value = EmitComplexExpr(init); 1998 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 1999 return; 2000 } 2001 2002 case TEK_Aggregate: { 2003 // Fix up the destination if the initializer isn't an expression 2004 // of atomic type. 2005 bool Zeroed = false; 2006 if (!init->getType()->isAtomicType()) { 2007 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2008 dest = atomics.projectValue(); 2009 } 2010 2011 // Evaluate the expression directly into the destination. 2012 AggValueSlot slot = AggValueSlot::forLValue(dest, 2013 AggValueSlot::IsNotDestructed, 2014 AggValueSlot::DoesNotNeedGCBarriers, 2015 AggValueSlot::IsNotAliased, 2016 AggValueSlot::DoesNotOverlap, 2017 Zeroed ? AggValueSlot::IsZeroed : 2018 AggValueSlot::IsNotZeroed); 2019 2020 EmitAggExpr(init, slot); 2021 return; 2022 } 2023 } 2024 llvm_unreachable("bad evaluation kind"); 2025 } 2026