1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the code for emitting atomic operations. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCall.h" 14 #include "CGRecordLayout.h" 15 #include "CodeGenFunction.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "clang/Frontend/FrontendDiagnostic.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/IR/DataLayout.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/Operator.h" 25 26 using namespace clang; 27 using namespace CodeGen; 28 29 namespace { 30 class AtomicInfo { 31 CodeGenFunction &CGF; 32 QualType AtomicTy; 33 QualType ValueTy; 34 uint64_t AtomicSizeInBits; 35 uint64_t ValueSizeInBits; 36 CharUnits AtomicAlign; 37 CharUnits ValueAlign; 38 CharUnits LValueAlign; 39 TypeEvaluationKind EvaluationKind; 40 bool UseLibcall; 41 LValue LVal; 42 CGBitFieldInfo BFI; 43 public: 44 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 45 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 46 EvaluationKind(TEK_Scalar), UseLibcall(true) { 47 assert(!lvalue.isGlobalReg()); 48 ASTContext &C = CGF.getContext(); 49 if (lvalue.isSimple()) { 50 AtomicTy = lvalue.getType(); 51 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 52 ValueTy = ATy->getValueType(); 53 else 54 ValueTy = AtomicTy; 55 EvaluationKind = CGF.getEvaluationKind(ValueTy); 56 57 uint64_t ValueAlignInBits; 58 uint64_t AtomicAlignInBits; 59 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 60 ValueSizeInBits = ValueTI.Width; 61 ValueAlignInBits = ValueTI.Align; 62 63 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 64 AtomicSizeInBits = AtomicTI.Width; 65 AtomicAlignInBits = AtomicTI.Align; 66 67 assert(ValueSizeInBits <= AtomicSizeInBits); 68 assert(ValueAlignInBits <= AtomicAlignInBits); 69 70 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 71 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 72 if (lvalue.getAlignment().isZero()) 73 lvalue.setAlignment(AtomicAlign); 74 75 LVal = lvalue; 76 } else if (lvalue.isBitField()) { 77 ValueTy = lvalue.getType(); 78 ValueSizeInBits = C.getTypeSize(ValueTy); 79 auto &OrigBFI = lvalue.getBitFieldInfo(); 80 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 81 AtomicSizeInBits = C.toBits( 82 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 83 .alignTo(lvalue.getAlignment())); 84 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 85 auto OffsetInChars = 86 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 87 lvalue.getAlignment(); 88 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 89 VoidPtrAddr, OffsetInChars.getQuantity()); 90 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 91 VoidPtrAddr, 92 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 93 "atomic_bitfield_base"); 94 BFI = OrigBFI; 95 BFI.Offset = Offset; 96 BFI.StorageSize = AtomicSizeInBits; 97 BFI.StorageOffset += OffsetInChars; 98 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 99 BFI, lvalue.getType(), lvalue.getBaseInfo(), 100 lvalue.getTBAAInfo()); 101 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 102 if (AtomicTy.isNull()) { 103 llvm::APInt Size( 104 /*numBits=*/32, 105 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 106 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, 107 /*IndexTypeQuals=*/0); 108 } 109 AtomicAlign = ValueAlign = lvalue.getAlignment(); 110 } else if (lvalue.isVectorElt()) { 111 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); 112 ValueSizeInBits = C.getTypeSize(ValueTy); 113 AtomicTy = lvalue.getType(); 114 AtomicSizeInBits = C.getTypeSize(AtomicTy); 115 AtomicAlign = ValueAlign = lvalue.getAlignment(); 116 LVal = lvalue; 117 } else { 118 assert(lvalue.isExtVectorElt()); 119 ValueTy = lvalue.getType(); 120 ValueSizeInBits = C.getTypeSize(ValueTy); 121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 122 lvalue.getType(), lvalue.getExtVectorAddress() 123 .getElementType()->getVectorNumElements()); 124 AtomicSizeInBits = C.getTypeSize(AtomicTy); 125 AtomicAlign = ValueAlign = lvalue.getAlignment(); 126 LVal = lvalue; 127 } 128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 129 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 130 } 131 132 QualType getAtomicType() const { return AtomicTy; } 133 QualType getValueType() const { return ValueTy; } 134 CharUnits getAtomicAlignment() const { return AtomicAlign; } 135 CharUnits getValueAlignment() const { return ValueAlign; } 136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 139 bool shouldUseLibcall() const { return UseLibcall; } 140 const LValue &getAtomicLValue() const { return LVal; } 141 llvm::Value *getAtomicPointer() const { 142 if (LVal.isSimple()) 143 return LVal.getPointer(); 144 else if (LVal.isBitField()) 145 return LVal.getBitFieldPointer(); 146 else if (LVal.isVectorElt()) 147 return LVal.getVectorPointer(); 148 assert(LVal.isExtVectorElt()); 149 return LVal.getExtVectorPointer(); 150 } 151 Address getAtomicAddress() const { 152 return Address(getAtomicPointer(), getAtomicAlignment()); 153 } 154 155 Address getAtomicAddressAsAtomicIntPointer() const { 156 return emitCastToAtomicIntPointer(getAtomicAddress()); 157 } 158 159 /// Is the atomic size larger than the underlying value type? 160 /// 161 /// Note that the absence of padding does not mean that atomic 162 /// objects are completely interchangeable with non-atomic 163 /// objects: we might have promoted the alignment of a type 164 /// without making it bigger. 165 bool hasPadding() const { 166 return (ValueSizeInBits != AtomicSizeInBits); 167 } 168 169 bool emitMemSetZeroIfNecessary() const; 170 171 llvm::Value *getAtomicSizeValue() const { 172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 173 return CGF.CGM.getSize(size); 174 } 175 176 /// Cast the given pointer to an integer pointer suitable for atomic 177 /// operations if the source. 178 Address emitCastToAtomicIntPointer(Address Addr) const; 179 180 /// If Addr is compatible with the iN that will be used for an atomic 181 /// operation, bitcast it. Otherwise, create a temporary that is suitable 182 /// and copy the value across. 183 Address convertToAtomicIntPointer(Address Addr) const; 184 185 /// Turn an atomic-layout object into an r-value. 186 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 187 SourceLocation loc, bool AsValue) const; 188 189 /// Converts a rvalue to integer value. 190 llvm::Value *convertRValueToInt(RValue RVal) const; 191 192 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 193 AggValueSlot ResultSlot, 194 SourceLocation Loc, bool AsValue) const; 195 196 /// Copy an atomic r-value into atomic-layout memory. 197 void emitCopyIntoMemory(RValue rvalue) const; 198 199 /// Project an l-value down to the value field. 200 LValue projectValue() const { 201 assert(LVal.isSimple()); 202 Address addr = getAtomicAddress(); 203 if (hasPadding()) 204 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 205 206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 207 LVal.getBaseInfo(), LVal.getTBAAInfo()); 208 } 209 210 /// Emits atomic load. 211 /// \returns Loaded value. 212 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 213 bool AsValue, llvm::AtomicOrdering AO, 214 bool IsVolatile); 215 216 /// Emits atomic compare-and-exchange sequence. 217 /// \param Expected Expected value. 218 /// \param Desired Desired value. 219 /// \param Success Atomic ordering for success operation. 220 /// \param Failure Atomic ordering for failed operation. 221 /// \param IsWeak true if atomic operation is weak, false otherwise. 222 /// \returns Pair of values: previous value from storage (value type) and 223 /// boolean flag (i1 type) with true if success and false otherwise. 224 std::pair<RValue, llvm::Value *> 225 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 226 llvm::AtomicOrdering Success = 227 llvm::AtomicOrdering::SequentiallyConsistent, 228 llvm::AtomicOrdering Failure = 229 llvm::AtomicOrdering::SequentiallyConsistent, 230 bool IsWeak = false); 231 232 /// Emits atomic update. 233 /// \param AO Atomic ordering. 234 /// \param UpdateOp Update operation for the current lvalue. 235 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 236 const llvm::function_ref<RValue(RValue)> &UpdateOp, 237 bool IsVolatile); 238 /// Emits atomic update. 239 /// \param AO Atomic ordering. 240 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 241 bool IsVolatile); 242 243 /// Materialize an atomic r-value in atomic-layout memory. 244 Address materializeRValue(RValue rvalue) const; 245 246 /// Creates temp alloca for intermediate operations on atomic value. 247 Address CreateTempAlloca() const; 248 private: 249 bool requiresMemSetZero(llvm::Type *type) const; 250 251 252 /// Emits atomic load as a libcall. 253 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 254 llvm::AtomicOrdering AO, bool IsVolatile); 255 /// Emits atomic load as LLVM instruction. 256 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 257 /// Emits atomic compare-and-exchange op as a libcall. 258 llvm::Value *EmitAtomicCompareExchangeLibcall( 259 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 260 llvm::AtomicOrdering Success = 261 llvm::AtomicOrdering::SequentiallyConsistent, 262 llvm::AtomicOrdering Failure = 263 llvm::AtomicOrdering::SequentiallyConsistent); 264 /// Emits atomic compare-and-exchange op as LLVM instruction. 265 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 266 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 267 llvm::AtomicOrdering Success = 268 llvm::AtomicOrdering::SequentiallyConsistent, 269 llvm::AtomicOrdering Failure = 270 llvm::AtomicOrdering::SequentiallyConsistent, 271 bool IsWeak = false); 272 /// Emit atomic update as libcalls. 273 void 274 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 275 const llvm::function_ref<RValue(RValue)> &UpdateOp, 276 bool IsVolatile); 277 /// Emit atomic update as LLVM instructions. 278 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 279 const llvm::function_ref<RValue(RValue)> &UpdateOp, 280 bool IsVolatile); 281 /// Emit atomic update as libcalls. 282 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 283 bool IsVolatile); 284 /// Emit atomic update as LLVM instructions. 285 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 286 bool IsVolatile); 287 }; 288 } 289 290 Address AtomicInfo::CreateTempAlloca() const { 291 Address TempAlloca = CGF.CreateMemTemp( 292 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 293 : AtomicTy, 294 getAtomicAlignment(), 295 "atomic-temp"); 296 // Cast to pointer to value type for bitfields. 297 if (LVal.isBitField()) 298 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 299 TempAlloca, getAtomicAddress().getType()); 300 return TempAlloca; 301 } 302 303 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 304 StringRef fnName, 305 QualType resultType, 306 CallArgList &args) { 307 const CGFunctionInfo &fnInfo = 308 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 309 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 310 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 311 auto callee = CGCallee::forDirect(fn); 312 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 313 } 314 315 /// Does a store of the given IR type modify the full expected width? 316 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 317 uint64_t expectedSize) { 318 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 319 } 320 321 /// Does the atomic type require memsetting to zero before initialization? 322 /// 323 /// The IR type is provided as a way of making certain queries faster. 324 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 325 // If the atomic type has size padding, we definitely need a memset. 326 if (hasPadding()) return true; 327 328 // Otherwise, do some simple heuristics to try to avoid it: 329 switch (getEvaluationKind()) { 330 // For scalars and complexes, check whether the store size of the 331 // type uses the full size. 332 case TEK_Scalar: 333 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 334 case TEK_Complex: 335 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 336 AtomicSizeInBits / 2); 337 338 // Padding in structs has an undefined bit pattern. User beware. 339 case TEK_Aggregate: 340 return false; 341 } 342 llvm_unreachable("bad evaluation kind"); 343 } 344 345 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 346 assert(LVal.isSimple()); 347 llvm::Value *addr = LVal.getPointer(); 348 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 349 return false; 350 351 CGF.Builder.CreateMemSet( 352 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 353 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 354 LVal.getAlignment().getQuantity()); 355 return true; 356 } 357 358 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 359 Address Dest, Address Ptr, 360 Address Val1, Address Val2, 361 uint64_t Size, 362 llvm::AtomicOrdering SuccessOrder, 363 llvm::AtomicOrdering FailureOrder, 364 llvm::SyncScope::ID Scope) { 365 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 366 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 367 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 368 369 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 370 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 371 Scope); 372 Pair->setVolatile(E->isVolatile()); 373 Pair->setWeak(IsWeak); 374 375 // Cmp holds the result of the compare-exchange operation: true on success, 376 // false on failure. 377 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 378 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 379 380 // This basic block is used to hold the store instruction if the operation 381 // failed. 382 llvm::BasicBlock *StoreExpectedBB = 383 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 384 385 // This basic block is the exit point of the operation, we should end up 386 // here regardless of whether or not the operation succeeded. 387 llvm::BasicBlock *ContinueBB = 388 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 389 390 // Update Expected if Expected isn't equal to Old, otherwise branch to the 391 // exit point. 392 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 393 394 CGF.Builder.SetInsertPoint(StoreExpectedBB); 395 // Update the memory at Expected with Old's value. 396 CGF.Builder.CreateStore(Old, Val1); 397 // Finally, branch to the exit point. 398 CGF.Builder.CreateBr(ContinueBB); 399 400 CGF.Builder.SetInsertPoint(ContinueBB); 401 // Update the memory at Dest with Cmp's value. 402 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 403 } 404 405 /// Given an ordering required on success, emit all possible cmpxchg 406 /// instructions to cope with the provided (but possibly only dynamically known) 407 /// FailureOrder. 408 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 409 bool IsWeak, Address Dest, Address Ptr, 410 Address Val1, Address Val2, 411 llvm::Value *FailureOrderVal, 412 uint64_t Size, 413 llvm::AtomicOrdering SuccessOrder, 414 llvm::SyncScope::ID Scope) { 415 llvm::AtomicOrdering FailureOrder; 416 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 417 auto FOS = FO->getSExtValue(); 418 if (!llvm::isValidAtomicOrderingCABI(FOS)) 419 FailureOrder = llvm::AtomicOrdering::Monotonic; 420 else 421 switch ((llvm::AtomicOrderingCABI)FOS) { 422 case llvm::AtomicOrderingCABI::relaxed: 423 case llvm::AtomicOrderingCABI::release: 424 case llvm::AtomicOrderingCABI::acq_rel: 425 FailureOrder = llvm::AtomicOrdering::Monotonic; 426 break; 427 case llvm::AtomicOrderingCABI::consume: 428 case llvm::AtomicOrderingCABI::acquire: 429 FailureOrder = llvm::AtomicOrdering::Acquire; 430 break; 431 case llvm::AtomicOrderingCABI::seq_cst: 432 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 433 break; 434 } 435 if (isStrongerThan(FailureOrder, SuccessOrder)) { 436 // Don't assert on undefined behavior "failure argument shall be no 437 // stronger than the success argument". 438 FailureOrder = 439 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 440 } 441 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 442 FailureOrder, Scope); 443 return; 444 } 445 446 // Create all the relevant BB's 447 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 448 *SeqCstBB = nullptr; 449 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 450 if (SuccessOrder != llvm::AtomicOrdering::Monotonic && 451 SuccessOrder != llvm::AtomicOrdering::Release) 452 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 453 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 454 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 455 456 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 457 458 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 459 460 // Emit all the different atomics 461 462 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 463 // doesn't matter unless someone is crazy enough to use something that 464 // doesn't fold to a constant for the ordering. 465 CGF.Builder.SetInsertPoint(MonotonicBB); 466 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 467 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 468 CGF.Builder.CreateBr(ContBB); 469 470 if (AcquireBB) { 471 CGF.Builder.SetInsertPoint(AcquireBB); 472 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 473 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope); 474 CGF.Builder.CreateBr(ContBB); 475 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 476 AcquireBB); 477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 478 AcquireBB); 479 } 480 if (SeqCstBB) { 481 CGF.Builder.SetInsertPoint(SeqCstBB); 482 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 483 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 484 CGF.Builder.CreateBr(ContBB); 485 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 486 SeqCstBB); 487 } 488 489 CGF.Builder.SetInsertPoint(ContBB); 490 } 491 492 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 493 Address Ptr, Address Val1, Address Val2, 494 llvm::Value *IsWeak, llvm::Value *FailureOrder, 495 uint64_t Size, llvm::AtomicOrdering Order, 496 llvm::SyncScope::ID Scope) { 497 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 498 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 499 500 switch (E->getOp()) { 501 case AtomicExpr::AO__c11_atomic_init: 502 case AtomicExpr::AO__opencl_atomic_init: 503 llvm_unreachable("Already handled!"); 504 505 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 506 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 507 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 508 FailureOrder, Size, Order, Scope); 509 return; 510 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 511 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 512 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 513 FailureOrder, Size, Order, Scope); 514 return; 515 case AtomicExpr::AO__atomic_compare_exchange: 516 case AtomicExpr::AO__atomic_compare_exchange_n: { 517 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 518 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 519 Val1, Val2, FailureOrder, Size, Order, Scope); 520 } else { 521 // Create all the relevant BB's 522 llvm::BasicBlock *StrongBB = 523 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 524 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 525 llvm::BasicBlock *ContBB = 526 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 527 528 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 529 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 530 531 CGF.Builder.SetInsertPoint(StrongBB); 532 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 533 FailureOrder, Size, Order, Scope); 534 CGF.Builder.CreateBr(ContBB); 535 536 CGF.Builder.SetInsertPoint(WeakBB); 537 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 538 FailureOrder, Size, Order, Scope); 539 CGF.Builder.CreateBr(ContBB); 540 541 CGF.Builder.SetInsertPoint(ContBB); 542 } 543 return; 544 } 545 case AtomicExpr::AO__c11_atomic_load: 546 case AtomicExpr::AO__opencl_atomic_load: 547 case AtomicExpr::AO__atomic_load_n: 548 case AtomicExpr::AO__atomic_load: { 549 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 550 Load->setAtomic(Order, Scope); 551 Load->setVolatile(E->isVolatile()); 552 CGF.Builder.CreateStore(Load, Dest); 553 return; 554 } 555 556 case AtomicExpr::AO__c11_atomic_store: 557 case AtomicExpr::AO__opencl_atomic_store: 558 case AtomicExpr::AO__atomic_store: 559 case AtomicExpr::AO__atomic_store_n: { 560 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 561 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 562 Store->setAtomic(Order, Scope); 563 Store->setVolatile(E->isVolatile()); 564 return; 565 } 566 567 case AtomicExpr::AO__c11_atomic_exchange: 568 case AtomicExpr::AO__opencl_atomic_exchange: 569 case AtomicExpr::AO__atomic_exchange_n: 570 case AtomicExpr::AO__atomic_exchange: 571 Op = llvm::AtomicRMWInst::Xchg; 572 break; 573 574 case AtomicExpr::AO__atomic_add_fetch: 575 PostOp = llvm::Instruction::Add; 576 LLVM_FALLTHROUGH; 577 case AtomicExpr::AO__c11_atomic_fetch_add: 578 case AtomicExpr::AO__opencl_atomic_fetch_add: 579 case AtomicExpr::AO__atomic_fetch_add: 580 Op = llvm::AtomicRMWInst::Add; 581 break; 582 583 case AtomicExpr::AO__atomic_sub_fetch: 584 PostOp = llvm::Instruction::Sub; 585 LLVM_FALLTHROUGH; 586 case AtomicExpr::AO__c11_atomic_fetch_sub: 587 case AtomicExpr::AO__opencl_atomic_fetch_sub: 588 case AtomicExpr::AO__atomic_fetch_sub: 589 Op = llvm::AtomicRMWInst::Sub; 590 break; 591 592 case AtomicExpr::AO__opencl_atomic_fetch_min: 593 case AtomicExpr::AO__atomic_fetch_min: 594 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min 595 : llvm::AtomicRMWInst::UMin; 596 break; 597 598 case AtomicExpr::AO__opencl_atomic_fetch_max: 599 case AtomicExpr::AO__atomic_fetch_max: 600 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max 601 : llvm::AtomicRMWInst::UMax; 602 break; 603 604 case AtomicExpr::AO__atomic_and_fetch: 605 PostOp = llvm::Instruction::And; 606 LLVM_FALLTHROUGH; 607 case AtomicExpr::AO__c11_atomic_fetch_and: 608 case AtomicExpr::AO__opencl_atomic_fetch_and: 609 case AtomicExpr::AO__atomic_fetch_and: 610 Op = llvm::AtomicRMWInst::And; 611 break; 612 613 case AtomicExpr::AO__atomic_or_fetch: 614 PostOp = llvm::Instruction::Or; 615 LLVM_FALLTHROUGH; 616 case AtomicExpr::AO__c11_atomic_fetch_or: 617 case AtomicExpr::AO__opencl_atomic_fetch_or: 618 case AtomicExpr::AO__atomic_fetch_or: 619 Op = llvm::AtomicRMWInst::Or; 620 break; 621 622 case AtomicExpr::AO__atomic_xor_fetch: 623 PostOp = llvm::Instruction::Xor; 624 LLVM_FALLTHROUGH; 625 case AtomicExpr::AO__c11_atomic_fetch_xor: 626 case AtomicExpr::AO__opencl_atomic_fetch_xor: 627 case AtomicExpr::AO__atomic_fetch_xor: 628 Op = llvm::AtomicRMWInst::Xor; 629 break; 630 631 case AtomicExpr::AO__atomic_nand_fetch: 632 PostOp = llvm::Instruction::And; // the NOT is special cased below 633 LLVM_FALLTHROUGH; 634 case AtomicExpr::AO__atomic_fetch_nand: 635 Op = llvm::AtomicRMWInst::Nand; 636 break; 637 } 638 639 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 640 llvm::AtomicRMWInst *RMWI = 641 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 642 RMWI->setVolatile(E->isVolatile()); 643 644 // For __atomic_*_fetch operations, perform the operation again to 645 // determine the value which was written. 646 llvm::Value *Result = RMWI; 647 if (PostOp) 648 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 649 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 650 Result = CGF.Builder.CreateNot(Result); 651 CGF.Builder.CreateStore(Result, Dest); 652 } 653 654 // This function emits any expression (scalar, complex, or aggregate) 655 // into a temporary alloca. 656 static Address 657 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 658 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 659 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 660 /*Init*/ true); 661 return DeclPtr; 662 } 663 664 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 665 Address Ptr, Address Val1, Address Val2, 666 llvm::Value *IsWeak, llvm::Value *FailureOrder, 667 uint64_t Size, llvm::AtomicOrdering Order, 668 llvm::Value *Scope) { 669 auto ScopeModel = Expr->getScopeModel(); 670 671 // LLVM atomic instructions always have synch scope. If clang atomic 672 // expression has no scope operand, use default LLVM synch scope. 673 if (!ScopeModel) { 674 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 675 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 676 return; 677 } 678 679 // Handle constant scope. 680 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 681 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 682 ScopeModel->map(SC->getZExtValue()), CGF.CGM.getLLVMContext()); 683 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 684 Order, SCID); 685 return; 686 } 687 688 // Handle non-constant scope. 689 auto &Builder = CGF.Builder; 690 auto Scopes = ScopeModel->getRuntimeValues(); 691 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 692 for (auto S : Scopes) 693 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 694 695 llvm::BasicBlock *ContBB = 696 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 697 698 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 699 // If unsupported synch scope is encountered at run time, assume a fallback 700 // synch scope value. 701 auto FallBack = ScopeModel->getFallBackValue(); 702 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 703 for (auto S : Scopes) { 704 auto *B = BB[S]; 705 if (S != FallBack) 706 SI->addCase(Builder.getInt32(S), B); 707 708 Builder.SetInsertPoint(B); 709 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 710 Order, 711 CGF.getTargetHooks().getLLVMSyncScopeID(ScopeModel->map(S), 712 CGF.getLLVMContext())); 713 Builder.CreateBr(ContBB); 714 } 715 716 Builder.SetInsertPoint(ContBB); 717 } 718 719 static void 720 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 721 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 722 SourceLocation Loc, CharUnits SizeInChars) { 723 if (UseOptimizedLibcall) { 724 // Load value and pass it to the function directly. 725 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 726 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 727 ValTy = 728 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 729 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 730 SizeInBits)->getPointerTo(); 731 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 732 Val = CGF.EmitLoadOfScalar(Ptr, false, 733 CGF.getContext().getPointerType(ValTy), 734 Loc); 735 // Coerce the value into an appropriately sized integer type. 736 Args.add(RValue::get(Val), ValTy); 737 } else { 738 // Non-optimized functions always take a reference. 739 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 740 CGF.getContext().VoidPtrTy); 741 } 742 } 743 744 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 745 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 746 QualType MemTy = AtomicTy; 747 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 748 MemTy = AT->getValueType(); 749 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 750 751 Address Val1 = Address::invalid(); 752 Address Val2 = Address::invalid(); 753 Address Dest = Address::invalid(); 754 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 755 756 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 757 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 758 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 759 EmitAtomicInit(E->getVal1(), lvalue); 760 return RValue::get(nullptr); 761 } 762 763 CharUnits sizeChars, alignChars; 764 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); 765 uint64_t Size = sizeChars.getQuantity(); 766 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 767 768 bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits; 769 bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0; 770 bool UseLibcall = Misaligned | Oversized; 771 772 if (UseLibcall) { 773 CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) 774 << !Oversized; 775 } 776 777 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 778 llvm::Value *Scope = 779 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 780 781 switch (E->getOp()) { 782 case AtomicExpr::AO__c11_atomic_init: 783 case AtomicExpr::AO__opencl_atomic_init: 784 llvm_unreachable("Already handled above with EmitAtomicInit!"); 785 786 case AtomicExpr::AO__c11_atomic_load: 787 case AtomicExpr::AO__opencl_atomic_load: 788 case AtomicExpr::AO__atomic_load_n: 789 break; 790 791 case AtomicExpr::AO__atomic_load: 792 Dest = EmitPointerWithAlignment(E->getVal1()); 793 break; 794 795 case AtomicExpr::AO__atomic_store: 796 Val1 = EmitPointerWithAlignment(E->getVal1()); 797 break; 798 799 case AtomicExpr::AO__atomic_exchange: 800 Val1 = EmitPointerWithAlignment(E->getVal1()); 801 Dest = EmitPointerWithAlignment(E->getVal2()); 802 break; 803 804 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 805 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 806 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 807 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 808 case AtomicExpr::AO__atomic_compare_exchange_n: 809 case AtomicExpr::AO__atomic_compare_exchange: 810 Val1 = EmitPointerWithAlignment(E->getVal1()); 811 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 812 Val2 = EmitPointerWithAlignment(E->getVal2()); 813 else 814 Val2 = EmitValToTemp(*this, E->getVal2()); 815 OrderFail = EmitScalarExpr(E->getOrderFail()); 816 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 817 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 818 IsWeak = EmitScalarExpr(E->getWeak()); 819 break; 820 821 case AtomicExpr::AO__c11_atomic_fetch_add: 822 case AtomicExpr::AO__c11_atomic_fetch_sub: 823 case AtomicExpr::AO__opencl_atomic_fetch_add: 824 case AtomicExpr::AO__opencl_atomic_fetch_sub: 825 if (MemTy->isPointerType()) { 826 // For pointer arithmetic, we're required to do a bit of math: 827 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 828 // ... but only for the C11 builtins. The GNU builtins expect the 829 // user to multiply by sizeof(T). 830 QualType Val1Ty = E->getVal1()->getType(); 831 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 832 CharUnits PointeeIncAmt = 833 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 834 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 835 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 836 Val1 = Temp; 837 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 838 break; 839 } 840 LLVM_FALLTHROUGH; 841 case AtomicExpr::AO__atomic_fetch_add: 842 case AtomicExpr::AO__atomic_fetch_sub: 843 case AtomicExpr::AO__atomic_add_fetch: 844 case AtomicExpr::AO__atomic_sub_fetch: 845 case AtomicExpr::AO__c11_atomic_store: 846 case AtomicExpr::AO__c11_atomic_exchange: 847 case AtomicExpr::AO__opencl_atomic_store: 848 case AtomicExpr::AO__opencl_atomic_exchange: 849 case AtomicExpr::AO__atomic_store_n: 850 case AtomicExpr::AO__atomic_exchange_n: 851 case AtomicExpr::AO__c11_atomic_fetch_and: 852 case AtomicExpr::AO__c11_atomic_fetch_or: 853 case AtomicExpr::AO__c11_atomic_fetch_xor: 854 case AtomicExpr::AO__opencl_atomic_fetch_and: 855 case AtomicExpr::AO__opencl_atomic_fetch_or: 856 case AtomicExpr::AO__opencl_atomic_fetch_xor: 857 case AtomicExpr::AO__opencl_atomic_fetch_min: 858 case AtomicExpr::AO__opencl_atomic_fetch_max: 859 case AtomicExpr::AO__atomic_fetch_and: 860 case AtomicExpr::AO__atomic_fetch_or: 861 case AtomicExpr::AO__atomic_fetch_xor: 862 case AtomicExpr::AO__atomic_fetch_nand: 863 case AtomicExpr::AO__atomic_and_fetch: 864 case AtomicExpr::AO__atomic_or_fetch: 865 case AtomicExpr::AO__atomic_xor_fetch: 866 case AtomicExpr::AO__atomic_nand_fetch: 867 case AtomicExpr::AO__atomic_fetch_min: 868 case AtomicExpr::AO__atomic_fetch_max: 869 Val1 = EmitValToTemp(*this, E->getVal1()); 870 break; 871 } 872 873 QualType RValTy = E->getType().getUnqualifiedType(); 874 875 // The inlined atomics only function on iN types, where N is a power of 2. We 876 // need to make sure (via temporaries if necessary) that all incoming values 877 // are compatible. 878 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 879 AtomicInfo Atomics(*this, AtomicVal); 880 881 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 882 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); 883 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); 884 if (Dest.isValid()) 885 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 886 else if (E->isCmpXChg()) 887 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 888 else if (!RValTy->isVoidType()) 889 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); 890 891 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 892 if (UseLibcall) { 893 bool UseOptimizedLibcall = false; 894 switch (E->getOp()) { 895 case AtomicExpr::AO__c11_atomic_init: 896 case AtomicExpr::AO__opencl_atomic_init: 897 llvm_unreachable("Already handled above with EmitAtomicInit!"); 898 899 case AtomicExpr::AO__c11_atomic_fetch_add: 900 case AtomicExpr::AO__opencl_atomic_fetch_add: 901 case AtomicExpr::AO__atomic_fetch_add: 902 case AtomicExpr::AO__c11_atomic_fetch_and: 903 case AtomicExpr::AO__opencl_atomic_fetch_and: 904 case AtomicExpr::AO__atomic_fetch_and: 905 case AtomicExpr::AO__c11_atomic_fetch_or: 906 case AtomicExpr::AO__opencl_atomic_fetch_or: 907 case AtomicExpr::AO__atomic_fetch_or: 908 case AtomicExpr::AO__atomic_fetch_nand: 909 case AtomicExpr::AO__c11_atomic_fetch_sub: 910 case AtomicExpr::AO__opencl_atomic_fetch_sub: 911 case AtomicExpr::AO__atomic_fetch_sub: 912 case AtomicExpr::AO__c11_atomic_fetch_xor: 913 case AtomicExpr::AO__opencl_atomic_fetch_xor: 914 case AtomicExpr::AO__opencl_atomic_fetch_min: 915 case AtomicExpr::AO__opencl_atomic_fetch_max: 916 case AtomicExpr::AO__atomic_fetch_xor: 917 case AtomicExpr::AO__atomic_add_fetch: 918 case AtomicExpr::AO__atomic_and_fetch: 919 case AtomicExpr::AO__atomic_nand_fetch: 920 case AtomicExpr::AO__atomic_or_fetch: 921 case AtomicExpr::AO__atomic_sub_fetch: 922 case AtomicExpr::AO__atomic_xor_fetch: 923 case AtomicExpr::AO__atomic_fetch_min: 924 case AtomicExpr::AO__atomic_fetch_max: 925 // For these, only library calls for certain sizes exist. 926 UseOptimizedLibcall = true; 927 break; 928 929 case AtomicExpr::AO__atomic_load: 930 case AtomicExpr::AO__atomic_store: 931 case AtomicExpr::AO__atomic_exchange: 932 case AtomicExpr::AO__atomic_compare_exchange: 933 // Use the generic version if we don't know that the operand will be 934 // suitably aligned for the optimized version. 935 if (Misaligned) 936 break; 937 LLVM_FALLTHROUGH; 938 case AtomicExpr::AO__c11_atomic_load: 939 case AtomicExpr::AO__c11_atomic_store: 940 case AtomicExpr::AO__c11_atomic_exchange: 941 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 942 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 943 case AtomicExpr::AO__opencl_atomic_load: 944 case AtomicExpr::AO__opencl_atomic_store: 945 case AtomicExpr::AO__opencl_atomic_exchange: 946 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 947 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 948 case AtomicExpr::AO__atomic_load_n: 949 case AtomicExpr::AO__atomic_store_n: 950 case AtomicExpr::AO__atomic_exchange_n: 951 case AtomicExpr::AO__atomic_compare_exchange_n: 952 // Only use optimized library calls for sizes for which they exist. 953 // FIXME: Size == 16 optimized library functions exist too. 954 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 955 UseOptimizedLibcall = true; 956 break; 957 } 958 959 CallArgList Args; 960 if (!UseOptimizedLibcall) { 961 // For non-optimized library calls, the size is the first parameter 962 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 963 getContext().getSizeType()); 964 } 965 // Atomic address is the first or second parameter 966 // The OpenCL atomic library functions only accept pointer arguments to 967 // generic address space. 968 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 969 if (!E->isOpenCL()) 970 return V; 971 auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace(); 972 if (AS == LangAS::opencl_generic) 973 return V; 974 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 975 auto T = V->getType(); 976 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS); 977 978 return getTargetHooks().performAddrSpaceCast( 979 *this, V, AS, LangAS::opencl_generic, DestType, false); 980 }; 981 982 Args.add(RValue::get(CastToGenericAddrSpace( 983 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), 984 getContext().VoidPtrTy); 985 986 std::string LibCallName; 987 QualType LoweredMemTy = 988 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 989 QualType RetTy; 990 bool HaveRetTy = false; 991 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 992 switch (E->getOp()) { 993 case AtomicExpr::AO__c11_atomic_init: 994 case AtomicExpr::AO__opencl_atomic_init: 995 llvm_unreachable("Already handled!"); 996 997 // There is only one libcall for compare an exchange, because there is no 998 // optimisation benefit possible from a libcall version of a weak compare 999 // and exchange. 1000 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 1001 // void *desired, int success, int failure) 1002 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 1003 // int success, int failure) 1004 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1005 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1006 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1007 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1008 case AtomicExpr::AO__atomic_compare_exchange: 1009 case AtomicExpr::AO__atomic_compare_exchange_n: 1010 LibCallName = "__atomic_compare_exchange"; 1011 RetTy = getContext().BoolTy; 1012 HaveRetTy = true; 1013 Args.add( 1014 RValue::get(CastToGenericAddrSpace( 1015 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), 1016 getContext().VoidPtrTy); 1017 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 1018 MemTy, E->getExprLoc(), sizeChars); 1019 Args.add(RValue::get(Order), getContext().IntTy); 1020 Order = OrderFail; 1021 break; 1022 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1023 // int order) 1024 // T __atomic_exchange_N(T *mem, T val, int order) 1025 case AtomicExpr::AO__c11_atomic_exchange: 1026 case AtomicExpr::AO__opencl_atomic_exchange: 1027 case AtomicExpr::AO__atomic_exchange_n: 1028 case AtomicExpr::AO__atomic_exchange: 1029 LibCallName = "__atomic_exchange"; 1030 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1031 MemTy, E->getExprLoc(), sizeChars); 1032 break; 1033 // void __atomic_store(size_t size, void *mem, void *val, int order) 1034 // void __atomic_store_N(T *mem, T val, int order) 1035 case AtomicExpr::AO__c11_atomic_store: 1036 case AtomicExpr::AO__opencl_atomic_store: 1037 case AtomicExpr::AO__atomic_store: 1038 case AtomicExpr::AO__atomic_store_n: 1039 LibCallName = "__atomic_store"; 1040 RetTy = getContext().VoidTy; 1041 HaveRetTy = true; 1042 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1043 MemTy, E->getExprLoc(), sizeChars); 1044 break; 1045 // void __atomic_load(size_t size, void *mem, void *return, int order) 1046 // T __atomic_load_N(T *mem, int order) 1047 case AtomicExpr::AO__c11_atomic_load: 1048 case AtomicExpr::AO__opencl_atomic_load: 1049 case AtomicExpr::AO__atomic_load: 1050 case AtomicExpr::AO__atomic_load_n: 1051 LibCallName = "__atomic_load"; 1052 break; 1053 // T __atomic_add_fetch_N(T *mem, T val, int order) 1054 // T __atomic_fetch_add_N(T *mem, T val, int order) 1055 case AtomicExpr::AO__atomic_add_fetch: 1056 PostOp = llvm::Instruction::Add; 1057 LLVM_FALLTHROUGH; 1058 case AtomicExpr::AO__c11_atomic_fetch_add: 1059 case AtomicExpr::AO__opencl_atomic_fetch_add: 1060 case AtomicExpr::AO__atomic_fetch_add: 1061 LibCallName = "__atomic_fetch_add"; 1062 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1063 LoweredMemTy, E->getExprLoc(), sizeChars); 1064 break; 1065 // T __atomic_and_fetch_N(T *mem, T val, int order) 1066 // T __atomic_fetch_and_N(T *mem, T val, int order) 1067 case AtomicExpr::AO__atomic_and_fetch: 1068 PostOp = llvm::Instruction::And; 1069 LLVM_FALLTHROUGH; 1070 case AtomicExpr::AO__c11_atomic_fetch_and: 1071 case AtomicExpr::AO__opencl_atomic_fetch_and: 1072 case AtomicExpr::AO__atomic_fetch_and: 1073 LibCallName = "__atomic_fetch_and"; 1074 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1075 MemTy, E->getExprLoc(), sizeChars); 1076 break; 1077 // T __atomic_or_fetch_N(T *mem, T val, int order) 1078 // T __atomic_fetch_or_N(T *mem, T val, int order) 1079 case AtomicExpr::AO__atomic_or_fetch: 1080 PostOp = llvm::Instruction::Or; 1081 LLVM_FALLTHROUGH; 1082 case AtomicExpr::AO__c11_atomic_fetch_or: 1083 case AtomicExpr::AO__opencl_atomic_fetch_or: 1084 case AtomicExpr::AO__atomic_fetch_or: 1085 LibCallName = "__atomic_fetch_or"; 1086 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1087 MemTy, E->getExprLoc(), sizeChars); 1088 break; 1089 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1090 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1091 case AtomicExpr::AO__atomic_sub_fetch: 1092 PostOp = llvm::Instruction::Sub; 1093 LLVM_FALLTHROUGH; 1094 case AtomicExpr::AO__c11_atomic_fetch_sub: 1095 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1096 case AtomicExpr::AO__atomic_fetch_sub: 1097 LibCallName = "__atomic_fetch_sub"; 1098 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1099 LoweredMemTy, E->getExprLoc(), sizeChars); 1100 break; 1101 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1102 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1103 case AtomicExpr::AO__atomic_xor_fetch: 1104 PostOp = llvm::Instruction::Xor; 1105 LLVM_FALLTHROUGH; 1106 case AtomicExpr::AO__c11_atomic_fetch_xor: 1107 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1108 case AtomicExpr::AO__atomic_fetch_xor: 1109 LibCallName = "__atomic_fetch_xor"; 1110 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1111 MemTy, E->getExprLoc(), sizeChars); 1112 break; 1113 case AtomicExpr::AO__atomic_fetch_min: 1114 case AtomicExpr::AO__opencl_atomic_fetch_min: 1115 LibCallName = E->getValueType()->isSignedIntegerType() 1116 ? "__atomic_fetch_min" 1117 : "__atomic_fetch_umin"; 1118 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1119 LoweredMemTy, E->getExprLoc(), sizeChars); 1120 break; 1121 case AtomicExpr::AO__atomic_fetch_max: 1122 case AtomicExpr::AO__opencl_atomic_fetch_max: 1123 LibCallName = E->getValueType()->isSignedIntegerType() 1124 ? "__atomic_fetch_max" 1125 : "__atomic_fetch_umax"; 1126 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1127 LoweredMemTy, E->getExprLoc(), sizeChars); 1128 break; 1129 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1130 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1131 case AtomicExpr::AO__atomic_nand_fetch: 1132 PostOp = llvm::Instruction::And; // the NOT is special cased below 1133 LLVM_FALLTHROUGH; 1134 case AtomicExpr::AO__atomic_fetch_nand: 1135 LibCallName = "__atomic_fetch_nand"; 1136 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1137 MemTy, E->getExprLoc(), sizeChars); 1138 break; 1139 } 1140 1141 if (E->isOpenCL()) { 1142 LibCallName = std::string("__opencl") + 1143 StringRef(LibCallName).drop_front(1).str(); 1144 1145 } 1146 // Optimized functions have the size in their name. 1147 if (UseOptimizedLibcall) 1148 LibCallName += "_" + llvm::utostr(Size); 1149 // By default, assume we return a value of the atomic type. 1150 if (!HaveRetTy) { 1151 if (UseOptimizedLibcall) { 1152 // Value is returned directly. 1153 // The function returns an appropriately sized integer type. 1154 RetTy = getContext().getIntTypeForBitwidth( 1155 getContext().toBits(sizeChars), /*Signed=*/false); 1156 } else { 1157 // Value is returned through parameter before the order. 1158 RetTy = getContext().VoidTy; 1159 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 1160 getContext().VoidPtrTy); 1161 } 1162 } 1163 // order is always the last parameter 1164 Args.add(RValue::get(Order), 1165 getContext().IntTy); 1166 if (E->isOpenCL()) 1167 Args.add(RValue::get(Scope), getContext().IntTy); 1168 1169 // PostOp is only needed for the atomic_*_fetch operations, and 1170 // thus is only needed for and implemented in the 1171 // UseOptimizedLibcall codepath. 1172 assert(UseOptimizedLibcall || !PostOp); 1173 1174 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1175 // The value is returned directly from the libcall. 1176 if (E->isCmpXChg()) 1177 return Res; 1178 1179 // The value is returned directly for optimized libcalls but the expr 1180 // provided an out-param. 1181 if (UseOptimizedLibcall && Res.getScalarVal()) { 1182 llvm::Value *ResVal = Res.getScalarVal(); 1183 if (PostOp) { 1184 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1185 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1186 } 1187 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1188 ResVal = Builder.CreateNot(ResVal); 1189 1190 Builder.CreateStore( 1191 ResVal, 1192 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 1193 } 1194 1195 if (RValTy->isVoidType()) 1196 return RValue::get(nullptr); 1197 1198 return convertTempToRValue( 1199 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1200 RValTy, E->getExprLoc()); 1201 } 1202 1203 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1204 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1205 E->getOp() == AtomicExpr::AO__atomic_store || 1206 E->getOp() == AtomicExpr::AO__atomic_store_n; 1207 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1208 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1209 E->getOp() == AtomicExpr::AO__atomic_load || 1210 E->getOp() == AtomicExpr::AO__atomic_load_n; 1211 1212 if (isa<llvm::ConstantInt>(Order)) { 1213 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1214 // We should not ever get to a case where the ordering isn't a valid C ABI 1215 // value, but it's hard to enforce that in general. 1216 if (llvm::isValidAtomicOrderingCABI(ord)) 1217 switch ((llvm::AtomicOrderingCABI)ord) { 1218 case llvm::AtomicOrderingCABI::relaxed: 1219 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1220 llvm::AtomicOrdering::Monotonic, Scope); 1221 break; 1222 case llvm::AtomicOrderingCABI::consume: 1223 case llvm::AtomicOrderingCABI::acquire: 1224 if (IsStore) 1225 break; // Avoid crashing on code with undefined behavior 1226 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1227 llvm::AtomicOrdering::Acquire, Scope); 1228 break; 1229 case llvm::AtomicOrderingCABI::release: 1230 if (IsLoad) 1231 break; // Avoid crashing on code with undefined behavior 1232 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1233 llvm::AtomicOrdering::Release, Scope); 1234 break; 1235 case llvm::AtomicOrderingCABI::acq_rel: 1236 if (IsLoad || IsStore) 1237 break; // Avoid crashing on code with undefined behavior 1238 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1239 llvm::AtomicOrdering::AcquireRelease, Scope); 1240 break; 1241 case llvm::AtomicOrderingCABI::seq_cst: 1242 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1243 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1244 break; 1245 } 1246 if (RValTy->isVoidType()) 1247 return RValue::get(nullptr); 1248 1249 return convertTempToRValue( 1250 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1251 Dest.getAddressSpace())), 1252 RValTy, E->getExprLoc()); 1253 } 1254 1255 // Long case, when Order isn't obviously constant. 1256 1257 // Create all the relevant BB's 1258 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1259 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1260 *SeqCstBB = nullptr; 1261 MonotonicBB = createBasicBlock("monotonic", CurFn); 1262 if (!IsStore) 1263 AcquireBB = createBasicBlock("acquire", CurFn); 1264 if (!IsLoad) 1265 ReleaseBB = createBasicBlock("release", CurFn); 1266 if (!IsLoad && !IsStore) 1267 AcqRelBB = createBasicBlock("acqrel", CurFn); 1268 SeqCstBB = createBasicBlock("seqcst", CurFn); 1269 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1270 1271 // Create the switch for the split 1272 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1273 // doesn't matter unless someone is crazy enough to use something that 1274 // doesn't fold to a constant for the ordering. 1275 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1276 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1277 1278 // Emit all the different atomics 1279 Builder.SetInsertPoint(MonotonicBB); 1280 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1281 llvm::AtomicOrdering::Monotonic, Scope); 1282 Builder.CreateBr(ContBB); 1283 if (!IsStore) { 1284 Builder.SetInsertPoint(AcquireBB); 1285 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1286 llvm::AtomicOrdering::Acquire, Scope); 1287 Builder.CreateBr(ContBB); 1288 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1289 AcquireBB); 1290 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1291 AcquireBB); 1292 } 1293 if (!IsLoad) { 1294 Builder.SetInsertPoint(ReleaseBB); 1295 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1296 llvm::AtomicOrdering::Release, Scope); 1297 Builder.CreateBr(ContBB); 1298 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1299 ReleaseBB); 1300 } 1301 if (!IsLoad && !IsStore) { 1302 Builder.SetInsertPoint(AcqRelBB); 1303 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1304 llvm::AtomicOrdering::AcquireRelease, Scope); 1305 Builder.CreateBr(ContBB); 1306 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1307 AcqRelBB); 1308 } 1309 Builder.SetInsertPoint(SeqCstBB); 1310 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1311 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1312 Builder.CreateBr(ContBB); 1313 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1314 SeqCstBB); 1315 1316 // Cleanup and return 1317 Builder.SetInsertPoint(ContBB); 1318 if (RValTy->isVoidType()) 1319 return RValue::get(nullptr); 1320 1321 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1322 return convertTempToRValue( 1323 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1324 Dest.getAddressSpace())), 1325 RValTy, E->getExprLoc()); 1326 } 1327 1328 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1329 unsigned addrspace = 1330 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 1331 llvm::IntegerType *ty = 1332 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1333 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 1334 } 1335 1336 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1337 llvm::Type *Ty = Addr.getElementType(); 1338 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1339 if (SourceSizeInBits != AtomicSizeInBits) { 1340 Address Tmp = CreateTempAlloca(); 1341 CGF.Builder.CreateMemCpy(Tmp, Addr, 1342 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1343 Addr = Tmp; 1344 } 1345 1346 return emitCastToAtomicIntPointer(Addr); 1347 } 1348 1349 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1350 AggValueSlot resultSlot, 1351 SourceLocation loc, 1352 bool asValue) const { 1353 if (LVal.isSimple()) { 1354 if (EvaluationKind == TEK_Aggregate) 1355 return resultSlot.asRValue(); 1356 1357 // Drill into the padding structure if we have one. 1358 if (hasPadding()) 1359 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 1360 1361 // Otherwise, just convert the temporary to an r-value using the 1362 // normal conversion routine. 1363 return CGF.convertTempToRValue(addr, getValueType(), loc); 1364 } 1365 if (!asValue) 1366 // Get RValue from temp memory as atomic for non-simple lvalues 1367 return RValue::get(CGF.Builder.CreateLoad(addr)); 1368 if (LVal.isBitField()) 1369 return CGF.EmitLoadOfBitfieldLValue( 1370 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1371 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1372 if (LVal.isVectorElt()) 1373 return CGF.EmitLoadOfLValue( 1374 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1375 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1376 assert(LVal.isExtVectorElt()); 1377 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1378 addr, LVal.getExtVectorElts(), LVal.getType(), 1379 LVal.getBaseInfo(), TBAAAccessInfo())); 1380 } 1381 1382 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1383 AggValueSlot ResultSlot, 1384 SourceLocation Loc, 1385 bool AsValue) const { 1386 // Try not to in some easy cases. 1387 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1388 if (getEvaluationKind() == TEK_Scalar && 1389 (((!LVal.isBitField() || 1390 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1391 !hasPadding()) || 1392 !AsValue)) { 1393 auto *ValTy = AsValue 1394 ? CGF.ConvertTypeForMem(ValueTy) 1395 : getAtomicAddress().getType()->getPointerElementType(); 1396 if (ValTy->isIntegerTy()) { 1397 assert(IntVal->getType() == ValTy && "Different integer types."); 1398 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1399 } else if (ValTy->isPointerTy()) 1400 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1401 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1402 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1403 } 1404 1405 // Create a temporary. This needs to be big enough to hold the 1406 // atomic integer. 1407 Address Temp = Address::invalid(); 1408 bool TempIsVolatile = false; 1409 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1410 assert(!ResultSlot.isIgnored()); 1411 Temp = ResultSlot.getAddress(); 1412 TempIsVolatile = ResultSlot.isVolatile(); 1413 } else { 1414 Temp = CreateTempAlloca(); 1415 } 1416 1417 // Slam the integer into the temporary. 1418 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1419 CGF.Builder.CreateStore(IntVal, CastTemp) 1420 ->setVolatile(TempIsVolatile); 1421 1422 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1423 } 1424 1425 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1426 llvm::AtomicOrdering AO, bool) { 1427 // void __atomic_load(size_t size, void *mem, void *return, int order); 1428 CallArgList Args; 1429 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1430 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1431 CGF.getContext().VoidPtrTy); 1432 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1433 CGF.getContext().VoidPtrTy); 1434 Args.add( 1435 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1436 CGF.getContext().IntTy); 1437 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1438 } 1439 1440 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1441 bool IsVolatile) { 1442 // Okay, we're doing this natively. 1443 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1444 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1445 Load->setAtomic(AO); 1446 1447 // Other decoration. 1448 if (IsVolatile) 1449 Load->setVolatile(true); 1450 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1451 return Load; 1452 } 1453 1454 /// An LValue is a candidate for having its loads and stores be made atomic if 1455 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1456 /// performing such an operation can be performed without a libcall. 1457 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1458 if (!CGM.getCodeGenOpts().MSVolatile) return false; 1459 AtomicInfo AI(*this, LV); 1460 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1461 // An atomic is inline if we don't need to use a libcall. 1462 bool AtomicIsInline = !AI.shouldUseLibcall(); 1463 // MSVC doesn't seem to do this for types wider than a pointer. 1464 if (getContext().getTypeSize(LV.getType()) > 1465 getContext().getTypeSize(getContext().getIntPtrType())) 1466 return false; 1467 return IsVolatile && AtomicIsInline; 1468 } 1469 1470 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1471 AggValueSlot Slot) { 1472 llvm::AtomicOrdering AO; 1473 bool IsVolatile = LV.isVolatileQualified(); 1474 if (LV.getType()->isAtomicType()) { 1475 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1476 } else { 1477 AO = llvm::AtomicOrdering::Acquire; 1478 IsVolatile = true; 1479 } 1480 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1481 } 1482 1483 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1484 bool AsValue, llvm::AtomicOrdering AO, 1485 bool IsVolatile) { 1486 // Check whether we should use a library call. 1487 if (shouldUseLibcall()) { 1488 Address TempAddr = Address::invalid(); 1489 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1490 assert(getEvaluationKind() == TEK_Aggregate); 1491 TempAddr = ResultSlot.getAddress(); 1492 } else 1493 TempAddr = CreateTempAlloca(); 1494 1495 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1496 1497 // Okay, turn that back into the original value or whole atomic (for 1498 // non-simple lvalues) type. 1499 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1500 } 1501 1502 // Okay, we're doing this natively. 1503 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1504 1505 // If we're ignoring an aggregate return, don't do anything. 1506 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1507 return RValue::getAggregate(Address::invalid(), false); 1508 1509 // Okay, turn that back into the original value or atomic (for non-simple 1510 // lvalues) type. 1511 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1512 } 1513 1514 /// Emit a load from an l-value of atomic type. Note that the r-value 1515 /// we produce is an r-value of the atomic *value* type. 1516 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1517 llvm::AtomicOrdering AO, bool IsVolatile, 1518 AggValueSlot resultSlot) { 1519 AtomicInfo Atomics(*this, src); 1520 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1521 IsVolatile); 1522 } 1523 1524 /// Copy an r-value into memory as part of storing to an atomic type. 1525 /// This needs to create a bit-pattern suitable for atomic operations. 1526 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1527 assert(LVal.isSimple()); 1528 // If we have an r-value, the rvalue should be of the atomic type, 1529 // which means that the caller is responsible for having zeroed 1530 // any padding. Just do an aggregate copy of that type. 1531 if (rvalue.isAggregate()) { 1532 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1533 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1534 getAtomicType()); 1535 bool IsVolatile = rvalue.isVolatileQualified() || 1536 LVal.isVolatileQualified(); 1537 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1538 AggValueSlot::DoesNotOverlap, IsVolatile); 1539 return; 1540 } 1541 1542 // Okay, otherwise we're copying stuff. 1543 1544 // Zero out the buffer if necessary. 1545 emitMemSetZeroIfNecessary(); 1546 1547 // Drill past the padding if present. 1548 LValue TempLVal = projectValue(); 1549 1550 // Okay, store the rvalue in. 1551 if (rvalue.isScalar()) { 1552 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1553 } else { 1554 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1555 } 1556 } 1557 1558 1559 /// Materialize an r-value into memory for the purposes of storing it 1560 /// to an atomic type. 1561 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1562 // Aggregate r-values are already in memory, and EmitAtomicStore 1563 // requires them to be values of the atomic type. 1564 if (rvalue.isAggregate()) 1565 return rvalue.getAggregateAddress(); 1566 1567 // Otherwise, make a temporary and materialize into it. 1568 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1569 AtomicInfo Atomics(CGF, TempLV); 1570 Atomics.emitCopyIntoMemory(rvalue); 1571 return TempLV.getAddress(); 1572 } 1573 1574 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1575 // If we've got a scalar value of the right size, try to avoid going 1576 // through memory. 1577 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1578 llvm::Value *Value = RVal.getScalarVal(); 1579 if (isa<llvm::IntegerType>(Value->getType())) 1580 return CGF.EmitToMemory(Value, ValueTy); 1581 else { 1582 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1583 CGF.getLLVMContext(), 1584 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1585 if (isa<llvm::PointerType>(Value->getType())) 1586 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1587 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1588 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1589 } 1590 } 1591 // Otherwise, we need to go through memory. 1592 // Put the r-value in memory. 1593 Address Addr = materializeRValue(RVal); 1594 1595 // Cast the temporary to the atomic int type and pull a value out. 1596 Addr = emitCastToAtomicIntPointer(Addr); 1597 return CGF.Builder.CreateLoad(Addr); 1598 } 1599 1600 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1601 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1602 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1603 // Do the atomic store. 1604 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1605 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1606 ExpectedVal, DesiredVal, 1607 Success, Failure); 1608 // Other decoration. 1609 Inst->setVolatile(LVal.isVolatileQualified()); 1610 Inst->setWeak(IsWeak); 1611 1612 // Okay, turn that back into the original value type. 1613 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1614 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1615 return std::make_pair(PreviousVal, SuccessFailureVal); 1616 } 1617 1618 llvm::Value * 1619 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1620 llvm::Value *DesiredAddr, 1621 llvm::AtomicOrdering Success, 1622 llvm::AtomicOrdering Failure) { 1623 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1624 // void *desired, int success, int failure); 1625 CallArgList Args; 1626 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1627 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1628 CGF.getContext().VoidPtrTy); 1629 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1630 CGF.getContext().VoidPtrTy); 1631 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1632 CGF.getContext().VoidPtrTy); 1633 Args.add(RValue::get( 1634 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1635 CGF.getContext().IntTy); 1636 Args.add(RValue::get( 1637 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1638 CGF.getContext().IntTy); 1639 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1640 CGF.getContext().BoolTy, Args); 1641 1642 return SuccessFailureRVal.getScalarVal(); 1643 } 1644 1645 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1646 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1647 llvm::AtomicOrdering Failure, bool IsWeak) { 1648 if (isStrongerThan(Failure, Success)) 1649 // Don't assert on undefined behavior "failure argument shall be no stronger 1650 // than the success argument". 1651 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 1652 1653 // Check whether we should use a library call. 1654 if (shouldUseLibcall()) { 1655 // Produce a source address. 1656 Address ExpectedAddr = materializeRValue(Expected); 1657 Address DesiredAddr = materializeRValue(Desired); 1658 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1659 DesiredAddr.getPointer(), 1660 Success, Failure); 1661 return std::make_pair( 1662 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1663 SourceLocation(), /*AsValue=*/false), 1664 Res); 1665 } 1666 1667 // If we've got a scalar value of the right size, try to avoid going 1668 // through memory. 1669 auto *ExpectedVal = convertRValueToInt(Expected); 1670 auto *DesiredVal = convertRValueToInt(Desired); 1671 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1672 Failure, IsWeak); 1673 return std::make_pair( 1674 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1675 SourceLocation(), /*AsValue=*/false), 1676 Res.second); 1677 } 1678 1679 static void 1680 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1681 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1682 Address DesiredAddr) { 1683 RValue UpRVal; 1684 LValue AtomicLVal = Atomics.getAtomicLValue(); 1685 LValue DesiredLVal; 1686 if (AtomicLVal.isSimple()) { 1687 UpRVal = OldRVal; 1688 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1689 } else { 1690 // Build new lvalue for temp address 1691 Address Ptr = Atomics.materializeRValue(OldRVal); 1692 LValue UpdateLVal; 1693 if (AtomicLVal.isBitField()) { 1694 UpdateLVal = 1695 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1696 AtomicLVal.getType(), 1697 AtomicLVal.getBaseInfo(), 1698 AtomicLVal.getTBAAInfo()); 1699 DesiredLVal = 1700 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1701 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1702 AtomicLVal.getTBAAInfo()); 1703 } else if (AtomicLVal.isVectorElt()) { 1704 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1705 AtomicLVal.getType(), 1706 AtomicLVal.getBaseInfo(), 1707 AtomicLVal.getTBAAInfo()); 1708 DesiredLVal = LValue::MakeVectorElt( 1709 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1710 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1711 } else { 1712 assert(AtomicLVal.isExtVectorElt()); 1713 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1714 AtomicLVal.getType(), 1715 AtomicLVal.getBaseInfo(), 1716 AtomicLVal.getTBAAInfo()); 1717 DesiredLVal = LValue::MakeExtVectorElt( 1718 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1719 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1720 } 1721 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1722 } 1723 // Store new value in the corresponding memory area 1724 RValue NewRVal = UpdateOp(UpRVal); 1725 if (NewRVal.isScalar()) { 1726 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1727 } else { 1728 assert(NewRVal.isComplex()); 1729 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1730 /*isInit=*/false); 1731 } 1732 } 1733 1734 void AtomicInfo::EmitAtomicUpdateLibcall( 1735 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1736 bool IsVolatile) { 1737 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1738 1739 Address ExpectedAddr = CreateTempAlloca(); 1740 1741 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1742 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1743 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1744 CGF.EmitBlock(ContBB); 1745 Address DesiredAddr = CreateTempAlloca(); 1746 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1747 requiresMemSetZero(getAtomicAddress().getElementType())) { 1748 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1749 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1750 } 1751 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1752 AggValueSlot::ignored(), 1753 SourceLocation(), /*AsValue=*/false); 1754 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1755 auto *Res = 1756 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1757 DesiredAddr.getPointer(), 1758 AO, Failure); 1759 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1760 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1761 } 1762 1763 void AtomicInfo::EmitAtomicUpdateOp( 1764 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1765 bool IsVolatile) { 1766 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1767 1768 // Do the atomic load. 1769 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1770 // For non-simple lvalues perform compare-and-swap procedure. 1771 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1772 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1773 auto *CurBB = CGF.Builder.GetInsertBlock(); 1774 CGF.EmitBlock(ContBB); 1775 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1776 /*NumReservedValues=*/2); 1777 PHI->addIncoming(OldVal, CurBB); 1778 Address NewAtomicAddr = CreateTempAlloca(); 1779 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1780 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1781 requiresMemSetZero(getAtomicAddress().getElementType())) { 1782 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1783 } 1784 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1785 SourceLocation(), /*AsValue=*/false); 1786 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1787 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1788 // Try to write new value using cmpxchg operation 1789 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1790 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1791 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1792 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1793 } 1794 1795 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1796 RValue UpdateRVal, Address DesiredAddr) { 1797 LValue AtomicLVal = Atomics.getAtomicLValue(); 1798 LValue DesiredLVal; 1799 // Build new lvalue for temp address 1800 if (AtomicLVal.isBitField()) { 1801 DesiredLVal = 1802 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1803 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1804 AtomicLVal.getTBAAInfo()); 1805 } else if (AtomicLVal.isVectorElt()) { 1806 DesiredLVal = 1807 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1808 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1809 AtomicLVal.getTBAAInfo()); 1810 } else { 1811 assert(AtomicLVal.isExtVectorElt()); 1812 DesiredLVal = LValue::MakeExtVectorElt( 1813 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1814 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1815 } 1816 // Store new value in the corresponding memory area 1817 assert(UpdateRVal.isScalar()); 1818 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1819 } 1820 1821 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1822 RValue UpdateRVal, bool IsVolatile) { 1823 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1824 1825 Address ExpectedAddr = CreateTempAlloca(); 1826 1827 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1828 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1829 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1830 CGF.EmitBlock(ContBB); 1831 Address DesiredAddr = CreateTempAlloca(); 1832 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1833 requiresMemSetZero(getAtomicAddress().getElementType())) { 1834 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1835 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1836 } 1837 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1838 auto *Res = 1839 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1840 DesiredAddr.getPointer(), 1841 AO, Failure); 1842 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1843 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1844 } 1845 1846 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1847 bool IsVolatile) { 1848 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1849 1850 // Do the atomic load. 1851 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1852 // For non-simple lvalues perform compare-and-swap procedure. 1853 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1854 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1855 auto *CurBB = CGF.Builder.GetInsertBlock(); 1856 CGF.EmitBlock(ContBB); 1857 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1858 /*NumReservedValues=*/2); 1859 PHI->addIncoming(OldVal, CurBB); 1860 Address NewAtomicAddr = CreateTempAlloca(); 1861 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1862 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1863 requiresMemSetZero(getAtomicAddress().getElementType())) { 1864 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1865 } 1866 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1867 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1868 // Try to write new value using cmpxchg operation 1869 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1870 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1871 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1872 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1873 } 1874 1875 void AtomicInfo::EmitAtomicUpdate( 1876 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1877 bool IsVolatile) { 1878 if (shouldUseLibcall()) { 1879 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1880 } else { 1881 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1882 } 1883 } 1884 1885 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1886 bool IsVolatile) { 1887 if (shouldUseLibcall()) { 1888 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1889 } else { 1890 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1891 } 1892 } 1893 1894 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1895 bool isInit) { 1896 bool IsVolatile = lvalue.isVolatileQualified(); 1897 llvm::AtomicOrdering AO; 1898 if (lvalue.getType()->isAtomicType()) { 1899 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1900 } else { 1901 AO = llvm::AtomicOrdering::Release; 1902 IsVolatile = true; 1903 } 1904 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1905 } 1906 1907 /// Emit a store to an l-value of atomic type. 1908 /// 1909 /// Note that the r-value is expected to be an r-value *of the atomic 1910 /// type*; this means that for aggregate r-values, it should include 1911 /// storage for any padding that was necessary. 1912 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1913 llvm::AtomicOrdering AO, bool IsVolatile, 1914 bool isInit) { 1915 // If this is an aggregate r-value, it should agree in type except 1916 // maybe for address-space qualification. 1917 assert(!rvalue.isAggregate() || 1918 rvalue.getAggregateAddress().getElementType() 1919 == dest.getAddress().getElementType()); 1920 1921 AtomicInfo atomics(*this, dest); 1922 LValue LVal = atomics.getAtomicLValue(); 1923 1924 // If this is an initialization, just put the value there normally. 1925 if (LVal.isSimple()) { 1926 if (isInit) { 1927 atomics.emitCopyIntoMemory(rvalue); 1928 return; 1929 } 1930 1931 // Check whether we should use a library call. 1932 if (atomics.shouldUseLibcall()) { 1933 // Produce a source address. 1934 Address srcAddr = atomics.materializeRValue(rvalue); 1935 1936 // void __atomic_store(size_t size, void *mem, void *val, int order) 1937 CallArgList args; 1938 args.add(RValue::get(atomics.getAtomicSizeValue()), 1939 getContext().getSizeType()); 1940 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 1941 getContext().VoidPtrTy); 1942 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 1943 getContext().VoidPtrTy); 1944 args.add( 1945 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 1946 getContext().IntTy); 1947 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 1948 return; 1949 } 1950 1951 // Okay, we're doing this natively. 1952 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 1953 1954 // Do the atomic store. 1955 Address addr = 1956 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 1957 intValue = Builder.CreateIntCast( 1958 intValue, addr.getElementType(), /*isSigned=*/false); 1959 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 1960 1961 // Initializations don't need to be atomic. 1962 if (!isInit) 1963 store->setAtomic(AO); 1964 1965 // Other decoration. 1966 if (IsVolatile) 1967 store->setVolatile(true); 1968 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 1969 return; 1970 } 1971 1972 // Emit simple atomic update operation. 1973 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 1974 } 1975 1976 /// Emit a compare-and-exchange op for atomic type. 1977 /// 1978 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 1979 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 1980 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 1981 AggValueSlot Slot) { 1982 // If this is an aggregate r-value, it should agree in type except 1983 // maybe for address-space qualification. 1984 assert(!Expected.isAggregate() || 1985 Expected.getAggregateAddress().getElementType() == 1986 Obj.getAddress().getElementType()); 1987 assert(!Desired.isAggregate() || 1988 Desired.getAggregateAddress().getElementType() == 1989 Obj.getAddress().getElementType()); 1990 AtomicInfo Atomics(*this, Obj); 1991 1992 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 1993 IsWeak); 1994 } 1995 1996 void CodeGenFunction::EmitAtomicUpdate( 1997 LValue LVal, llvm::AtomicOrdering AO, 1998 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 1999 AtomicInfo Atomics(*this, LVal); 2000 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 2001 } 2002 2003 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 2004 AtomicInfo atomics(*this, dest); 2005 2006 switch (atomics.getEvaluationKind()) { 2007 case TEK_Scalar: { 2008 llvm::Value *value = EmitScalarExpr(init); 2009 atomics.emitCopyIntoMemory(RValue::get(value)); 2010 return; 2011 } 2012 2013 case TEK_Complex: { 2014 ComplexPairTy value = EmitComplexExpr(init); 2015 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2016 return; 2017 } 2018 2019 case TEK_Aggregate: { 2020 // Fix up the destination if the initializer isn't an expression 2021 // of atomic type. 2022 bool Zeroed = false; 2023 if (!init->getType()->isAtomicType()) { 2024 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2025 dest = atomics.projectValue(); 2026 } 2027 2028 // Evaluate the expression directly into the destination. 2029 AggValueSlot slot = AggValueSlot::forLValue(dest, 2030 AggValueSlot::IsNotDestructed, 2031 AggValueSlot::DoesNotNeedGCBarriers, 2032 AggValueSlot::IsNotAliased, 2033 AggValueSlot::DoesNotOverlap, 2034 Zeroed ? AggValueSlot::IsZeroed : 2035 AggValueSlot::IsNotZeroed); 2036 2037 EmitAggExpr(init, slot); 2038 return; 2039 } 2040 } 2041 llvm_unreachable("bad evaluation kind"); 2042 } 2043