1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the code for emitting atomic operations. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCall.h" 14 #include "CGRecordLayout.h" 15 #include "CodeGenFunction.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "clang/Frontend/FrontendDiagnostic.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/IR/DataLayout.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/Operator.h" 25 26 using namespace clang; 27 using namespace CodeGen; 28 29 namespace { 30 class AtomicInfo { 31 CodeGenFunction &CGF; 32 QualType AtomicTy; 33 QualType ValueTy; 34 uint64_t AtomicSizeInBits; 35 uint64_t ValueSizeInBits; 36 CharUnits AtomicAlign; 37 CharUnits ValueAlign; 38 TypeEvaluationKind EvaluationKind; 39 bool UseLibcall; 40 LValue LVal; 41 CGBitFieldInfo BFI; 42 public: 43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 45 EvaluationKind(TEK_Scalar), UseLibcall(true) { 46 assert(!lvalue.isGlobalReg()); 47 ASTContext &C = CGF.getContext(); 48 if (lvalue.isSimple()) { 49 AtomicTy = lvalue.getType(); 50 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 51 ValueTy = ATy->getValueType(); 52 else 53 ValueTy = AtomicTy; 54 EvaluationKind = CGF.getEvaluationKind(ValueTy); 55 56 uint64_t ValueAlignInBits; 57 uint64_t AtomicAlignInBits; 58 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 59 ValueSizeInBits = ValueTI.Width; 60 ValueAlignInBits = ValueTI.Align; 61 62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 63 AtomicSizeInBits = AtomicTI.Width; 64 AtomicAlignInBits = AtomicTI.Align; 65 66 assert(ValueSizeInBits <= AtomicSizeInBits); 67 assert(ValueAlignInBits <= AtomicAlignInBits); 68 69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 71 if (lvalue.getAlignment().isZero()) 72 lvalue.setAlignment(AtomicAlign); 73 74 LVal = lvalue; 75 } else if (lvalue.isBitField()) { 76 ValueTy = lvalue.getType(); 77 ValueSizeInBits = C.getTypeSize(ValueTy); 78 auto &OrigBFI = lvalue.getBitFieldInfo(); 79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 80 AtomicSizeInBits = C.toBits( 81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 82 .alignTo(lvalue.getAlignment())); 83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 84 auto OffsetInChars = 85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 86 lvalue.getAlignment(); 87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 88 VoidPtrAddr, OffsetInChars.getQuantity()); 89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 90 VoidPtrAddr, 91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 92 "atomic_bitfield_base"); 93 BFI = OrigBFI; 94 BFI.Offset = Offset; 95 BFI.StorageSize = AtomicSizeInBits; 96 BFI.StorageOffset += OffsetInChars; 97 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 98 BFI, lvalue.getType(), lvalue.getBaseInfo(), 99 lvalue.getTBAAInfo()); 100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 101 if (AtomicTy.isNull()) { 102 llvm::APInt Size( 103 /*numBits=*/32, 104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 105 AtomicTy = 106 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal, 107 /*IndexTypeQuals=*/0); 108 } 109 AtomicAlign = ValueAlign = lvalue.getAlignment(); 110 } else if (lvalue.isVectorElt()) { 111 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType(); 112 ValueSizeInBits = C.getTypeSize(ValueTy); 113 AtomicTy = lvalue.getType(); 114 AtomicSizeInBits = C.getTypeSize(AtomicTy); 115 AtomicAlign = ValueAlign = lvalue.getAlignment(); 116 LVal = lvalue; 117 } else { 118 assert(lvalue.isExtVectorElt()); 119 ValueTy = lvalue.getType(); 120 ValueSizeInBits = C.getTypeSize(ValueTy); 121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 122 lvalue.getType(), cast<llvm::FixedVectorType>( 123 lvalue.getExtVectorAddress().getElementType()) 124 ->getNumElements()); 125 AtomicSizeInBits = C.getTypeSize(AtomicTy); 126 AtomicAlign = ValueAlign = lvalue.getAlignment(); 127 LVal = lvalue; 128 } 129 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 130 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 131 } 132 133 QualType getAtomicType() const { return AtomicTy; } 134 QualType getValueType() const { return ValueTy; } 135 CharUnits getAtomicAlignment() const { return AtomicAlign; } 136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 139 bool shouldUseLibcall() const { return UseLibcall; } 140 const LValue &getAtomicLValue() const { return LVal; } 141 llvm::Value *getAtomicPointer() const { 142 if (LVal.isSimple()) 143 return LVal.getPointer(CGF); 144 else if (LVal.isBitField()) 145 return LVal.getBitFieldPointer(); 146 else if (LVal.isVectorElt()) 147 return LVal.getVectorPointer(); 148 assert(LVal.isExtVectorElt()); 149 return LVal.getExtVectorPointer(); 150 } 151 Address getAtomicAddress() const { 152 return Address(getAtomicPointer(), getAtomicAlignment()); 153 } 154 155 Address getAtomicAddressAsAtomicIntPointer() const { 156 return emitCastToAtomicIntPointer(getAtomicAddress()); 157 } 158 159 /// Is the atomic size larger than the underlying value type? 160 /// 161 /// Note that the absence of padding does not mean that atomic 162 /// objects are completely interchangeable with non-atomic 163 /// objects: we might have promoted the alignment of a type 164 /// without making it bigger. 165 bool hasPadding() const { 166 return (ValueSizeInBits != AtomicSizeInBits); 167 } 168 169 bool emitMemSetZeroIfNecessary() const; 170 171 llvm::Value *getAtomicSizeValue() const { 172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 173 return CGF.CGM.getSize(size); 174 } 175 176 /// Cast the given pointer to an integer pointer suitable for atomic 177 /// operations if the source. 178 Address emitCastToAtomicIntPointer(Address Addr) const; 179 180 /// If Addr is compatible with the iN that will be used for an atomic 181 /// operation, bitcast it. Otherwise, create a temporary that is suitable 182 /// and copy the value across. 183 Address convertToAtomicIntPointer(Address Addr) const; 184 185 /// Turn an atomic-layout object into an r-value. 186 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 187 SourceLocation loc, bool AsValue) const; 188 189 /// Converts a rvalue to integer value. 190 llvm::Value *convertRValueToInt(RValue RVal) const; 191 192 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 193 AggValueSlot ResultSlot, 194 SourceLocation Loc, bool AsValue) const; 195 196 /// Copy an atomic r-value into atomic-layout memory. 197 void emitCopyIntoMemory(RValue rvalue) const; 198 199 /// Project an l-value down to the value field. 200 LValue projectValue() const { 201 assert(LVal.isSimple()); 202 Address addr = getAtomicAddress(); 203 if (hasPadding()) 204 addr = CGF.Builder.CreateStructGEP(addr, 0); 205 206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 207 LVal.getBaseInfo(), LVal.getTBAAInfo()); 208 } 209 210 /// Emits atomic load. 211 /// \returns Loaded value. 212 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 213 bool AsValue, llvm::AtomicOrdering AO, 214 bool IsVolatile); 215 216 /// Emits atomic compare-and-exchange sequence. 217 /// \param Expected Expected value. 218 /// \param Desired Desired value. 219 /// \param Success Atomic ordering for success operation. 220 /// \param Failure Atomic ordering for failed operation. 221 /// \param IsWeak true if atomic operation is weak, false otherwise. 222 /// \returns Pair of values: previous value from storage (value type) and 223 /// boolean flag (i1 type) with true if success and false otherwise. 224 std::pair<RValue, llvm::Value *> 225 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 226 llvm::AtomicOrdering Success = 227 llvm::AtomicOrdering::SequentiallyConsistent, 228 llvm::AtomicOrdering Failure = 229 llvm::AtomicOrdering::SequentiallyConsistent, 230 bool IsWeak = false); 231 232 /// Emits atomic update. 233 /// \param AO Atomic ordering. 234 /// \param UpdateOp Update operation for the current lvalue. 235 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 236 const llvm::function_ref<RValue(RValue)> &UpdateOp, 237 bool IsVolatile); 238 /// Emits atomic update. 239 /// \param AO Atomic ordering. 240 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 241 bool IsVolatile); 242 243 /// Materialize an atomic r-value in atomic-layout memory. 244 Address materializeRValue(RValue rvalue) const; 245 246 /// Creates temp alloca for intermediate operations on atomic value. 247 Address CreateTempAlloca() const; 248 private: 249 bool requiresMemSetZero(llvm::Type *type) const; 250 251 252 /// Emits atomic load as a libcall. 253 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 254 llvm::AtomicOrdering AO, bool IsVolatile); 255 /// Emits atomic load as LLVM instruction. 256 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 257 /// Emits atomic compare-and-exchange op as a libcall. 258 llvm::Value *EmitAtomicCompareExchangeLibcall( 259 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 260 llvm::AtomicOrdering Success = 261 llvm::AtomicOrdering::SequentiallyConsistent, 262 llvm::AtomicOrdering Failure = 263 llvm::AtomicOrdering::SequentiallyConsistent); 264 /// Emits atomic compare-and-exchange op as LLVM instruction. 265 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 266 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 267 llvm::AtomicOrdering Success = 268 llvm::AtomicOrdering::SequentiallyConsistent, 269 llvm::AtomicOrdering Failure = 270 llvm::AtomicOrdering::SequentiallyConsistent, 271 bool IsWeak = false); 272 /// Emit atomic update as libcalls. 273 void 274 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 275 const llvm::function_ref<RValue(RValue)> &UpdateOp, 276 bool IsVolatile); 277 /// Emit atomic update as LLVM instructions. 278 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 279 const llvm::function_ref<RValue(RValue)> &UpdateOp, 280 bool IsVolatile); 281 /// Emit atomic update as libcalls. 282 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 283 bool IsVolatile); 284 /// Emit atomic update as LLVM instructions. 285 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 286 bool IsVolatile); 287 }; 288 } 289 290 Address AtomicInfo::CreateTempAlloca() const { 291 Address TempAlloca = CGF.CreateMemTemp( 292 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 293 : AtomicTy, 294 getAtomicAlignment(), 295 "atomic-temp"); 296 // Cast to pointer to value type for bitfields. 297 if (LVal.isBitField()) 298 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 299 TempAlloca, getAtomicAddress().getType()); 300 return TempAlloca; 301 } 302 303 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 304 StringRef fnName, 305 QualType resultType, 306 CallArgList &args) { 307 const CGFunctionInfo &fnInfo = 308 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 309 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 310 llvm::AttrBuilder fnAttrB; 311 fnAttrB.addAttribute(llvm::Attribute::NoUnwind); 312 fnAttrB.addAttribute(llvm::Attribute::WillReturn); 313 llvm::AttributeList fnAttrs = llvm::AttributeList::get( 314 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB); 315 316 llvm::FunctionCallee fn = 317 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs); 318 auto callee = CGCallee::forDirect(fn); 319 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 320 } 321 322 /// Does a store of the given IR type modify the full expected width? 323 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 324 uint64_t expectedSize) { 325 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 326 } 327 328 /// Does the atomic type require memsetting to zero before initialization? 329 /// 330 /// The IR type is provided as a way of making certain queries faster. 331 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 332 // If the atomic type has size padding, we definitely need a memset. 333 if (hasPadding()) return true; 334 335 // Otherwise, do some simple heuristics to try to avoid it: 336 switch (getEvaluationKind()) { 337 // For scalars and complexes, check whether the store size of the 338 // type uses the full size. 339 case TEK_Scalar: 340 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 341 case TEK_Complex: 342 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 343 AtomicSizeInBits / 2); 344 345 // Padding in structs has an undefined bit pattern. User beware. 346 case TEK_Aggregate: 347 return false; 348 } 349 llvm_unreachable("bad evaluation kind"); 350 } 351 352 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 353 assert(LVal.isSimple()); 354 llvm::Value *addr = LVal.getPointer(CGF); 355 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 356 return false; 357 358 CGF.Builder.CreateMemSet( 359 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 360 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 361 LVal.getAlignment().getAsAlign()); 362 return true; 363 } 364 365 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 366 Address Dest, Address Ptr, 367 Address Val1, Address Val2, 368 uint64_t Size, 369 llvm::AtomicOrdering SuccessOrder, 370 llvm::AtomicOrdering FailureOrder, 371 llvm::SyncScope::ID Scope) { 372 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 373 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 374 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 375 376 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 377 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 378 Scope); 379 Pair->setVolatile(E->isVolatile()); 380 Pair->setWeak(IsWeak); 381 382 // Cmp holds the result of the compare-exchange operation: true on success, 383 // false on failure. 384 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 385 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 386 387 // This basic block is used to hold the store instruction if the operation 388 // failed. 389 llvm::BasicBlock *StoreExpectedBB = 390 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 391 392 // This basic block is the exit point of the operation, we should end up 393 // here regardless of whether or not the operation succeeded. 394 llvm::BasicBlock *ContinueBB = 395 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 396 397 // Update Expected if Expected isn't equal to Old, otherwise branch to the 398 // exit point. 399 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 400 401 CGF.Builder.SetInsertPoint(StoreExpectedBB); 402 // Update the memory at Expected with Old's value. 403 CGF.Builder.CreateStore(Old, Val1); 404 // Finally, branch to the exit point. 405 CGF.Builder.CreateBr(ContinueBB); 406 407 CGF.Builder.SetInsertPoint(ContinueBB); 408 // Update the memory at Dest with Cmp's value. 409 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 410 } 411 412 /// Given an ordering required on success, emit all possible cmpxchg 413 /// instructions to cope with the provided (but possibly only dynamically known) 414 /// FailureOrder. 415 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 416 bool IsWeak, Address Dest, Address Ptr, 417 Address Val1, Address Val2, 418 llvm::Value *FailureOrderVal, 419 uint64_t Size, 420 llvm::AtomicOrdering SuccessOrder, 421 llvm::SyncScope::ID Scope) { 422 llvm::AtomicOrdering FailureOrder; 423 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 424 auto FOS = FO->getSExtValue(); 425 if (!llvm::isValidAtomicOrderingCABI(FOS)) 426 FailureOrder = llvm::AtomicOrdering::Monotonic; 427 else 428 switch ((llvm::AtomicOrderingCABI)FOS) { 429 case llvm::AtomicOrderingCABI::relaxed: 430 // 31.7.2.18: "The failure argument shall not be memory_order_release 431 // nor memory_order_acq_rel". Fallback to monotonic. 432 case llvm::AtomicOrderingCABI::release: 433 case llvm::AtomicOrderingCABI::acq_rel: 434 FailureOrder = llvm::AtomicOrdering::Monotonic; 435 break; 436 case llvm::AtomicOrderingCABI::consume: 437 case llvm::AtomicOrderingCABI::acquire: 438 FailureOrder = llvm::AtomicOrdering::Acquire; 439 break; 440 case llvm::AtomicOrderingCABI::seq_cst: 441 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 442 break; 443 } 444 // Prior to c++17, "the failure argument shall be no stronger than the 445 // success argument". This condition has been lifted and the only 446 // precondition is 31.7.2.18. Effectively treat this as a DR and skip 447 // language version checks. 448 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 449 FailureOrder, Scope); 450 return; 451 } 452 453 // Create all the relevant BB's 454 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 455 *SeqCstBB = nullptr; 456 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 457 if (SuccessOrder != llvm::AtomicOrdering::Monotonic) 458 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 459 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 460 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 461 462 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 463 464 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 465 466 // Emit all the different atomics 467 468 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 469 // doesn't matter unless someone is crazy enough to use something that 470 // doesn't fold to a constant for the ordering. 471 CGF.Builder.SetInsertPoint(MonotonicBB); 472 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 473 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 474 CGF.Builder.CreateBr(ContBB); 475 476 if (AcquireBB) { 477 CGF.Builder.SetInsertPoint(AcquireBB); 478 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 479 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope); 480 CGF.Builder.CreateBr(ContBB); 481 if (SuccessOrder != llvm::AtomicOrdering::Release) 482 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 483 AcquireBB); 484 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 485 AcquireBB); 486 } 487 if (SeqCstBB) { 488 CGF.Builder.SetInsertPoint(SeqCstBB); 489 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 490 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 491 CGF.Builder.CreateBr(ContBB); 492 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 493 SeqCstBB); 494 } 495 496 CGF.Builder.SetInsertPoint(ContBB); 497 } 498 499 /// Duplicate the atomic min/max operation in conventional IR for the builtin 500 /// variants that return the new rather than the original value. 501 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, 502 AtomicExpr::AtomicOp Op, 503 bool IsSigned, 504 llvm::Value *OldVal, 505 llvm::Value *RHS) { 506 llvm::CmpInst::Predicate Pred; 507 switch (Op) { 508 default: 509 llvm_unreachable("Unexpected min/max operation"); 510 case AtomicExpr::AO__atomic_max_fetch: 511 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; 512 break; 513 case AtomicExpr::AO__atomic_min_fetch: 514 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; 515 break; 516 } 517 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst"); 518 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval"); 519 } 520 521 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 522 Address Ptr, Address Val1, Address Val2, 523 llvm::Value *IsWeak, llvm::Value *FailureOrder, 524 uint64_t Size, llvm::AtomicOrdering Order, 525 llvm::SyncScope::ID Scope) { 526 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 527 bool PostOpMinMax = false; 528 unsigned PostOp = 0; 529 530 switch (E->getOp()) { 531 case AtomicExpr::AO__c11_atomic_init: 532 case AtomicExpr::AO__opencl_atomic_init: 533 llvm_unreachable("Already handled!"); 534 535 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 536 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 537 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 538 FailureOrder, Size, Order, Scope); 539 return; 540 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 541 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 542 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 543 FailureOrder, Size, Order, Scope); 544 return; 545 case AtomicExpr::AO__atomic_compare_exchange: 546 case AtomicExpr::AO__atomic_compare_exchange_n: { 547 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 548 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 549 Val1, Val2, FailureOrder, Size, Order, Scope); 550 } else { 551 // Create all the relevant BB's 552 llvm::BasicBlock *StrongBB = 553 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 554 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 555 llvm::BasicBlock *ContBB = 556 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 557 558 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 559 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 560 561 CGF.Builder.SetInsertPoint(StrongBB); 562 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 563 FailureOrder, Size, Order, Scope); 564 CGF.Builder.CreateBr(ContBB); 565 566 CGF.Builder.SetInsertPoint(WeakBB); 567 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 568 FailureOrder, Size, Order, Scope); 569 CGF.Builder.CreateBr(ContBB); 570 571 CGF.Builder.SetInsertPoint(ContBB); 572 } 573 return; 574 } 575 case AtomicExpr::AO__c11_atomic_load: 576 case AtomicExpr::AO__opencl_atomic_load: 577 case AtomicExpr::AO__atomic_load_n: 578 case AtomicExpr::AO__atomic_load: { 579 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 580 Load->setAtomic(Order, Scope); 581 Load->setVolatile(E->isVolatile()); 582 CGF.Builder.CreateStore(Load, Dest); 583 return; 584 } 585 586 case AtomicExpr::AO__c11_atomic_store: 587 case AtomicExpr::AO__opencl_atomic_store: 588 case AtomicExpr::AO__atomic_store: 589 case AtomicExpr::AO__atomic_store_n: { 590 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 591 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 592 Store->setAtomic(Order, Scope); 593 Store->setVolatile(E->isVolatile()); 594 return; 595 } 596 597 case AtomicExpr::AO__c11_atomic_exchange: 598 case AtomicExpr::AO__opencl_atomic_exchange: 599 case AtomicExpr::AO__atomic_exchange_n: 600 case AtomicExpr::AO__atomic_exchange: 601 Op = llvm::AtomicRMWInst::Xchg; 602 break; 603 604 case AtomicExpr::AO__atomic_add_fetch: 605 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd 606 : llvm::Instruction::Add; 607 LLVM_FALLTHROUGH; 608 case AtomicExpr::AO__c11_atomic_fetch_add: 609 case AtomicExpr::AO__opencl_atomic_fetch_add: 610 case AtomicExpr::AO__atomic_fetch_add: 611 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd 612 : llvm::AtomicRMWInst::Add; 613 break; 614 615 case AtomicExpr::AO__atomic_sub_fetch: 616 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub 617 : llvm::Instruction::Sub; 618 LLVM_FALLTHROUGH; 619 case AtomicExpr::AO__c11_atomic_fetch_sub: 620 case AtomicExpr::AO__opencl_atomic_fetch_sub: 621 case AtomicExpr::AO__atomic_fetch_sub: 622 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub 623 : llvm::AtomicRMWInst::Sub; 624 break; 625 626 case AtomicExpr::AO__atomic_min_fetch: 627 PostOpMinMax = true; 628 LLVM_FALLTHROUGH; 629 case AtomicExpr::AO__c11_atomic_fetch_min: 630 case AtomicExpr::AO__opencl_atomic_fetch_min: 631 case AtomicExpr::AO__atomic_fetch_min: 632 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min 633 : llvm::AtomicRMWInst::UMin; 634 break; 635 636 case AtomicExpr::AO__atomic_max_fetch: 637 PostOpMinMax = true; 638 LLVM_FALLTHROUGH; 639 case AtomicExpr::AO__c11_atomic_fetch_max: 640 case AtomicExpr::AO__opencl_atomic_fetch_max: 641 case AtomicExpr::AO__atomic_fetch_max: 642 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max 643 : llvm::AtomicRMWInst::UMax; 644 break; 645 646 case AtomicExpr::AO__atomic_and_fetch: 647 PostOp = llvm::Instruction::And; 648 LLVM_FALLTHROUGH; 649 case AtomicExpr::AO__c11_atomic_fetch_and: 650 case AtomicExpr::AO__opencl_atomic_fetch_and: 651 case AtomicExpr::AO__atomic_fetch_and: 652 Op = llvm::AtomicRMWInst::And; 653 break; 654 655 case AtomicExpr::AO__atomic_or_fetch: 656 PostOp = llvm::Instruction::Or; 657 LLVM_FALLTHROUGH; 658 case AtomicExpr::AO__c11_atomic_fetch_or: 659 case AtomicExpr::AO__opencl_atomic_fetch_or: 660 case AtomicExpr::AO__atomic_fetch_or: 661 Op = llvm::AtomicRMWInst::Or; 662 break; 663 664 case AtomicExpr::AO__atomic_xor_fetch: 665 PostOp = llvm::Instruction::Xor; 666 LLVM_FALLTHROUGH; 667 case AtomicExpr::AO__c11_atomic_fetch_xor: 668 case AtomicExpr::AO__opencl_atomic_fetch_xor: 669 case AtomicExpr::AO__atomic_fetch_xor: 670 Op = llvm::AtomicRMWInst::Xor; 671 break; 672 673 case AtomicExpr::AO__atomic_nand_fetch: 674 PostOp = llvm::Instruction::And; // the NOT is special cased below 675 LLVM_FALLTHROUGH; 676 case AtomicExpr::AO__atomic_fetch_nand: 677 Op = llvm::AtomicRMWInst::Nand; 678 break; 679 } 680 681 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 682 llvm::AtomicRMWInst *RMWI = 683 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 684 RMWI->setVolatile(E->isVolatile()); 685 686 // For __atomic_*_fetch operations, perform the operation again to 687 // determine the value which was written. 688 llvm::Value *Result = RMWI; 689 if (PostOpMinMax) 690 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), 691 E->getValueType()->isSignedIntegerType(), 692 RMWI, LoadVal1); 693 else if (PostOp) 694 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, 695 LoadVal1); 696 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 697 Result = CGF.Builder.CreateNot(Result); 698 CGF.Builder.CreateStore(Result, Dest); 699 } 700 701 // This function emits any expression (scalar, complex, or aggregate) 702 // into a temporary alloca. 703 static Address 704 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 705 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 706 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 707 /*Init*/ true); 708 return DeclPtr; 709 } 710 711 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 712 Address Ptr, Address Val1, Address Val2, 713 llvm::Value *IsWeak, llvm::Value *FailureOrder, 714 uint64_t Size, llvm::AtomicOrdering Order, 715 llvm::Value *Scope) { 716 auto ScopeModel = Expr->getScopeModel(); 717 718 // LLVM atomic instructions always have synch scope. If clang atomic 719 // expression has no scope operand, use default LLVM synch scope. 720 if (!ScopeModel) { 721 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 722 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 723 return; 724 } 725 726 // Handle constant scope. 727 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 728 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 729 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), 730 Order, CGF.CGM.getLLVMContext()); 731 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 732 Order, SCID); 733 return; 734 } 735 736 // Handle non-constant scope. 737 auto &Builder = CGF.Builder; 738 auto Scopes = ScopeModel->getRuntimeValues(); 739 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 740 for (auto S : Scopes) 741 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 742 743 llvm::BasicBlock *ContBB = 744 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 745 746 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 747 // If unsupported synch scope is encountered at run time, assume a fallback 748 // synch scope value. 749 auto FallBack = ScopeModel->getFallBackValue(); 750 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 751 for (auto S : Scopes) { 752 auto *B = BB[S]; 753 if (S != FallBack) 754 SI->addCase(Builder.getInt32(S), B); 755 756 Builder.SetInsertPoint(B); 757 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 758 Order, 759 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), 760 ScopeModel->map(S), 761 Order, 762 CGF.getLLVMContext())); 763 Builder.CreateBr(ContBB); 764 } 765 766 Builder.SetInsertPoint(ContBB); 767 } 768 769 static void 770 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 771 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 772 SourceLocation Loc, CharUnits SizeInChars) { 773 if (UseOptimizedLibcall) { 774 // Load value and pass it to the function directly. 775 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 776 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 777 ValTy = 778 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 779 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 780 SizeInBits)->getPointerTo(); 781 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 782 Val = CGF.EmitLoadOfScalar(Ptr, false, 783 CGF.getContext().getPointerType(ValTy), 784 Loc); 785 // Coerce the value into an appropriately sized integer type. 786 Args.add(RValue::get(Val), ValTy); 787 } else { 788 // Non-optimized functions always take a reference. 789 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 790 CGF.getContext().VoidPtrTy); 791 } 792 } 793 794 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 795 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 796 QualType MemTy = AtomicTy; 797 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 798 MemTy = AT->getValueType(); 799 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 800 801 Address Val1 = Address::invalid(); 802 Address Val2 = Address::invalid(); 803 Address Dest = Address::invalid(); 804 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 805 806 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 807 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 808 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 809 EmitAtomicInit(E->getVal1(), lvalue); 810 return RValue::get(nullptr); 811 } 812 813 auto TInfo = getContext().getTypeInfoInChars(AtomicTy); 814 uint64_t Size = TInfo.Width.getQuantity(); 815 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 816 817 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; 818 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; 819 bool UseLibcall = Misaligned | Oversized; 820 bool ShouldCastToIntPtrTy = true; 821 822 CharUnits MaxInlineWidth = 823 getContext().toCharUnitsFromBits(MaxInlineWidthInBits); 824 825 DiagnosticsEngine &Diags = CGM.getDiags(); 826 827 if (Misaligned) { 828 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) 829 << (int)TInfo.Width.getQuantity() 830 << (int)Ptr.getAlignment().getQuantity(); 831 } 832 833 if (Oversized) { 834 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized) 835 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity(); 836 } 837 838 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 839 llvm::Value *Scope = 840 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 841 842 switch (E->getOp()) { 843 case AtomicExpr::AO__c11_atomic_init: 844 case AtomicExpr::AO__opencl_atomic_init: 845 llvm_unreachable("Already handled above with EmitAtomicInit!"); 846 847 case AtomicExpr::AO__c11_atomic_load: 848 case AtomicExpr::AO__opencl_atomic_load: 849 case AtomicExpr::AO__atomic_load_n: 850 break; 851 852 case AtomicExpr::AO__atomic_load: 853 Dest = EmitPointerWithAlignment(E->getVal1()); 854 break; 855 856 case AtomicExpr::AO__atomic_store: 857 Val1 = EmitPointerWithAlignment(E->getVal1()); 858 break; 859 860 case AtomicExpr::AO__atomic_exchange: 861 Val1 = EmitPointerWithAlignment(E->getVal1()); 862 Dest = EmitPointerWithAlignment(E->getVal2()); 863 break; 864 865 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 866 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 867 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 868 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 869 case AtomicExpr::AO__atomic_compare_exchange_n: 870 case AtomicExpr::AO__atomic_compare_exchange: 871 Val1 = EmitPointerWithAlignment(E->getVal1()); 872 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 873 Val2 = EmitPointerWithAlignment(E->getVal2()); 874 else 875 Val2 = EmitValToTemp(*this, E->getVal2()); 876 OrderFail = EmitScalarExpr(E->getOrderFail()); 877 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 878 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 879 IsWeak = EmitScalarExpr(E->getWeak()); 880 break; 881 882 case AtomicExpr::AO__c11_atomic_fetch_add: 883 case AtomicExpr::AO__c11_atomic_fetch_sub: 884 case AtomicExpr::AO__opencl_atomic_fetch_add: 885 case AtomicExpr::AO__opencl_atomic_fetch_sub: 886 if (MemTy->isPointerType()) { 887 // For pointer arithmetic, we're required to do a bit of math: 888 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 889 // ... but only for the C11 builtins. The GNU builtins expect the 890 // user to multiply by sizeof(T). 891 QualType Val1Ty = E->getVal1()->getType(); 892 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 893 CharUnits PointeeIncAmt = 894 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 895 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 896 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 897 Val1 = Temp; 898 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 899 break; 900 } 901 LLVM_FALLTHROUGH; 902 case AtomicExpr::AO__atomic_fetch_add: 903 case AtomicExpr::AO__atomic_fetch_sub: 904 case AtomicExpr::AO__atomic_add_fetch: 905 case AtomicExpr::AO__atomic_sub_fetch: 906 ShouldCastToIntPtrTy = !MemTy->isFloatingType(); 907 LLVM_FALLTHROUGH; 908 909 case AtomicExpr::AO__c11_atomic_store: 910 case AtomicExpr::AO__c11_atomic_exchange: 911 case AtomicExpr::AO__opencl_atomic_store: 912 case AtomicExpr::AO__opencl_atomic_exchange: 913 case AtomicExpr::AO__atomic_store_n: 914 case AtomicExpr::AO__atomic_exchange_n: 915 case AtomicExpr::AO__c11_atomic_fetch_and: 916 case AtomicExpr::AO__c11_atomic_fetch_or: 917 case AtomicExpr::AO__c11_atomic_fetch_xor: 918 case AtomicExpr::AO__c11_atomic_fetch_max: 919 case AtomicExpr::AO__c11_atomic_fetch_min: 920 case AtomicExpr::AO__opencl_atomic_fetch_and: 921 case AtomicExpr::AO__opencl_atomic_fetch_or: 922 case AtomicExpr::AO__opencl_atomic_fetch_xor: 923 case AtomicExpr::AO__opencl_atomic_fetch_min: 924 case AtomicExpr::AO__opencl_atomic_fetch_max: 925 case AtomicExpr::AO__atomic_fetch_and: 926 case AtomicExpr::AO__atomic_fetch_or: 927 case AtomicExpr::AO__atomic_fetch_xor: 928 case AtomicExpr::AO__atomic_fetch_nand: 929 case AtomicExpr::AO__atomic_and_fetch: 930 case AtomicExpr::AO__atomic_or_fetch: 931 case AtomicExpr::AO__atomic_xor_fetch: 932 case AtomicExpr::AO__atomic_nand_fetch: 933 case AtomicExpr::AO__atomic_max_fetch: 934 case AtomicExpr::AO__atomic_min_fetch: 935 case AtomicExpr::AO__atomic_fetch_max: 936 case AtomicExpr::AO__atomic_fetch_min: 937 Val1 = EmitValToTemp(*this, E->getVal1()); 938 break; 939 } 940 941 QualType RValTy = E->getType().getUnqualifiedType(); 942 943 // The inlined atomics only function on iN types, where N is a power of 2. We 944 // need to make sure (via temporaries if necessary) that all incoming values 945 // are compatible. 946 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 947 AtomicInfo Atomics(*this, AtomicVal); 948 949 if (ShouldCastToIntPtrTy) { 950 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 951 if (Val1.isValid()) 952 Val1 = Atomics.convertToAtomicIntPointer(Val1); 953 if (Val2.isValid()) 954 Val2 = Atomics.convertToAtomicIntPointer(Val2); 955 } 956 if (Dest.isValid()) { 957 if (ShouldCastToIntPtrTy) 958 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 959 } else if (E->isCmpXChg()) 960 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 961 else if (!RValTy->isVoidType()) { 962 Dest = Atomics.CreateTempAlloca(); 963 if (ShouldCastToIntPtrTy) 964 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 965 } 966 967 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 968 if (UseLibcall) { 969 bool UseOptimizedLibcall = false; 970 switch (E->getOp()) { 971 case AtomicExpr::AO__c11_atomic_init: 972 case AtomicExpr::AO__opencl_atomic_init: 973 llvm_unreachable("Already handled above with EmitAtomicInit!"); 974 975 case AtomicExpr::AO__c11_atomic_fetch_add: 976 case AtomicExpr::AO__opencl_atomic_fetch_add: 977 case AtomicExpr::AO__atomic_fetch_add: 978 case AtomicExpr::AO__c11_atomic_fetch_and: 979 case AtomicExpr::AO__opencl_atomic_fetch_and: 980 case AtomicExpr::AO__atomic_fetch_and: 981 case AtomicExpr::AO__c11_atomic_fetch_or: 982 case AtomicExpr::AO__opencl_atomic_fetch_or: 983 case AtomicExpr::AO__atomic_fetch_or: 984 case AtomicExpr::AO__atomic_fetch_nand: 985 case AtomicExpr::AO__c11_atomic_fetch_sub: 986 case AtomicExpr::AO__opencl_atomic_fetch_sub: 987 case AtomicExpr::AO__atomic_fetch_sub: 988 case AtomicExpr::AO__c11_atomic_fetch_xor: 989 case AtomicExpr::AO__opencl_atomic_fetch_xor: 990 case AtomicExpr::AO__opencl_atomic_fetch_min: 991 case AtomicExpr::AO__opencl_atomic_fetch_max: 992 case AtomicExpr::AO__atomic_fetch_xor: 993 case AtomicExpr::AO__c11_atomic_fetch_max: 994 case AtomicExpr::AO__c11_atomic_fetch_min: 995 case AtomicExpr::AO__atomic_add_fetch: 996 case AtomicExpr::AO__atomic_and_fetch: 997 case AtomicExpr::AO__atomic_nand_fetch: 998 case AtomicExpr::AO__atomic_or_fetch: 999 case AtomicExpr::AO__atomic_sub_fetch: 1000 case AtomicExpr::AO__atomic_xor_fetch: 1001 case AtomicExpr::AO__atomic_fetch_max: 1002 case AtomicExpr::AO__atomic_fetch_min: 1003 case AtomicExpr::AO__atomic_max_fetch: 1004 case AtomicExpr::AO__atomic_min_fetch: 1005 // For these, only library calls for certain sizes exist. 1006 UseOptimizedLibcall = true; 1007 break; 1008 1009 case AtomicExpr::AO__atomic_load: 1010 case AtomicExpr::AO__atomic_store: 1011 case AtomicExpr::AO__atomic_exchange: 1012 case AtomicExpr::AO__atomic_compare_exchange: 1013 // Use the generic version if we don't know that the operand will be 1014 // suitably aligned for the optimized version. 1015 if (Misaligned) 1016 break; 1017 LLVM_FALLTHROUGH; 1018 case AtomicExpr::AO__c11_atomic_load: 1019 case AtomicExpr::AO__c11_atomic_store: 1020 case AtomicExpr::AO__c11_atomic_exchange: 1021 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1022 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1023 case AtomicExpr::AO__opencl_atomic_load: 1024 case AtomicExpr::AO__opencl_atomic_store: 1025 case AtomicExpr::AO__opencl_atomic_exchange: 1026 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1027 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1028 case AtomicExpr::AO__atomic_load_n: 1029 case AtomicExpr::AO__atomic_store_n: 1030 case AtomicExpr::AO__atomic_exchange_n: 1031 case AtomicExpr::AO__atomic_compare_exchange_n: 1032 // Only use optimized library calls for sizes for which they exist. 1033 // FIXME: Size == 16 optimized library functions exist too. 1034 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 1035 UseOptimizedLibcall = true; 1036 break; 1037 } 1038 1039 CallArgList Args; 1040 if (!UseOptimizedLibcall) { 1041 // For non-optimized library calls, the size is the first parameter 1042 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 1043 getContext().getSizeType()); 1044 } 1045 // Atomic address is the first or second parameter 1046 // The OpenCL atomic library functions only accept pointer arguments to 1047 // generic address space. 1048 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 1049 if (!E->isOpenCL()) 1050 return V; 1051 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace(); 1052 if (AS == LangAS::opencl_generic) 1053 return V; 1054 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 1055 auto T = V->getType(); 1056 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS); 1057 1058 return getTargetHooks().performAddrSpaceCast( 1059 *this, V, AS, LangAS::opencl_generic, DestType, false); 1060 }; 1061 1062 Args.add(RValue::get(CastToGenericAddrSpace( 1063 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), 1064 getContext().VoidPtrTy); 1065 1066 std::string LibCallName; 1067 QualType LoweredMemTy = 1068 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 1069 QualType RetTy; 1070 bool HaveRetTy = false; 1071 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 1072 bool PostOpMinMax = false; 1073 switch (E->getOp()) { 1074 case AtomicExpr::AO__c11_atomic_init: 1075 case AtomicExpr::AO__opencl_atomic_init: 1076 llvm_unreachable("Already handled!"); 1077 1078 // There is only one libcall for compare an exchange, because there is no 1079 // optimisation benefit possible from a libcall version of a weak compare 1080 // and exchange. 1081 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 1082 // void *desired, int success, int failure) 1083 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 1084 // int success, int failure) 1085 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1086 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1087 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1088 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1089 case AtomicExpr::AO__atomic_compare_exchange: 1090 case AtomicExpr::AO__atomic_compare_exchange_n: 1091 LibCallName = "__atomic_compare_exchange"; 1092 RetTy = getContext().BoolTy; 1093 HaveRetTy = true; 1094 Args.add( 1095 RValue::get(CastToGenericAddrSpace( 1096 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), 1097 getContext().VoidPtrTy); 1098 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 1099 MemTy, E->getExprLoc(), TInfo.Width); 1100 Args.add(RValue::get(Order), getContext().IntTy); 1101 Order = OrderFail; 1102 break; 1103 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1104 // int order) 1105 // T __atomic_exchange_N(T *mem, T val, int order) 1106 case AtomicExpr::AO__c11_atomic_exchange: 1107 case AtomicExpr::AO__opencl_atomic_exchange: 1108 case AtomicExpr::AO__atomic_exchange_n: 1109 case AtomicExpr::AO__atomic_exchange: 1110 LibCallName = "__atomic_exchange"; 1111 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1112 MemTy, E->getExprLoc(), TInfo.Width); 1113 break; 1114 // void __atomic_store(size_t size, void *mem, void *val, int order) 1115 // void __atomic_store_N(T *mem, T val, int order) 1116 case AtomicExpr::AO__c11_atomic_store: 1117 case AtomicExpr::AO__opencl_atomic_store: 1118 case AtomicExpr::AO__atomic_store: 1119 case AtomicExpr::AO__atomic_store_n: 1120 LibCallName = "__atomic_store"; 1121 RetTy = getContext().VoidTy; 1122 HaveRetTy = true; 1123 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1124 MemTy, E->getExprLoc(), TInfo.Width); 1125 break; 1126 // void __atomic_load(size_t size, void *mem, void *return, int order) 1127 // T __atomic_load_N(T *mem, int order) 1128 case AtomicExpr::AO__c11_atomic_load: 1129 case AtomicExpr::AO__opencl_atomic_load: 1130 case AtomicExpr::AO__atomic_load: 1131 case AtomicExpr::AO__atomic_load_n: 1132 LibCallName = "__atomic_load"; 1133 break; 1134 // T __atomic_add_fetch_N(T *mem, T val, int order) 1135 // T __atomic_fetch_add_N(T *mem, T val, int order) 1136 case AtomicExpr::AO__atomic_add_fetch: 1137 PostOp = llvm::Instruction::Add; 1138 LLVM_FALLTHROUGH; 1139 case AtomicExpr::AO__c11_atomic_fetch_add: 1140 case AtomicExpr::AO__opencl_atomic_fetch_add: 1141 case AtomicExpr::AO__atomic_fetch_add: 1142 LibCallName = "__atomic_fetch_add"; 1143 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1144 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1145 break; 1146 // T __atomic_and_fetch_N(T *mem, T val, int order) 1147 // T __atomic_fetch_and_N(T *mem, T val, int order) 1148 case AtomicExpr::AO__atomic_and_fetch: 1149 PostOp = llvm::Instruction::And; 1150 LLVM_FALLTHROUGH; 1151 case AtomicExpr::AO__c11_atomic_fetch_and: 1152 case AtomicExpr::AO__opencl_atomic_fetch_and: 1153 case AtomicExpr::AO__atomic_fetch_and: 1154 LibCallName = "__atomic_fetch_and"; 1155 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1156 MemTy, E->getExprLoc(), TInfo.Width); 1157 break; 1158 // T __atomic_or_fetch_N(T *mem, T val, int order) 1159 // T __atomic_fetch_or_N(T *mem, T val, int order) 1160 case AtomicExpr::AO__atomic_or_fetch: 1161 PostOp = llvm::Instruction::Or; 1162 LLVM_FALLTHROUGH; 1163 case AtomicExpr::AO__c11_atomic_fetch_or: 1164 case AtomicExpr::AO__opencl_atomic_fetch_or: 1165 case AtomicExpr::AO__atomic_fetch_or: 1166 LibCallName = "__atomic_fetch_or"; 1167 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1168 MemTy, E->getExprLoc(), TInfo.Width); 1169 break; 1170 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1171 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1172 case AtomicExpr::AO__atomic_sub_fetch: 1173 PostOp = llvm::Instruction::Sub; 1174 LLVM_FALLTHROUGH; 1175 case AtomicExpr::AO__c11_atomic_fetch_sub: 1176 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1177 case AtomicExpr::AO__atomic_fetch_sub: 1178 LibCallName = "__atomic_fetch_sub"; 1179 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1180 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1181 break; 1182 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1183 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1184 case AtomicExpr::AO__atomic_xor_fetch: 1185 PostOp = llvm::Instruction::Xor; 1186 LLVM_FALLTHROUGH; 1187 case AtomicExpr::AO__c11_atomic_fetch_xor: 1188 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1189 case AtomicExpr::AO__atomic_fetch_xor: 1190 LibCallName = "__atomic_fetch_xor"; 1191 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1192 MemTy, E->getExprLoc(), TInfo.Width); 1193 break; 1194 case AtomicExpr::AO__atomic_min_fetch: 1195 PostOpMinMax = true; 1196 LLVM_FALLTHROUGH; 1197 case AtomicExpr::AO__c11_atomic_fetch_min: 1198 case AtomicExpr::AO__atomic_fetch_min: 1199 case AtomicExpr::AO__opencl_atomic_fetch_min: 1200 LibCallName = E->getValueType()->isSignedIntegerType() 1201 ? "__atomic_fetch_min" 1202 : "__atomic_fetch_umin"; 1203 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1204 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1205 break; 1206 case AtomicExpr::AO__atomic_max_fetch: 1207 PostOpMinMax = true; 1208 LLVM_FALLTHROUGH; 1209 case AtomicExpr::AO__c11_atomic_fetch_max: 1210 case AtomicExpr::AO__atomic_fetch_max: 1211 case AtomicExpr::AO__opencl_atomic_fetch_max: 1212 LibCallName = E->getValueType()->isSignedIntegerType() 1213 ? "__atomic_fetch_max" 1214 : "__atomic_fetch_umax"; 1215 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1216 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1217 break; 1218 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1219 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1220 case AtomicExpr::AO__atomic_nand_fetch: 1221 PostOp = llvm::Instruction::And; // the NOT is special cased below 1222 LLVM_FALLTHROUGH; 1223 case AtomicExpr::AO__atomic_fetch_nand: 1224 LibCallName = "__atomic_fetch_nand"; 1225 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1226 MemTy, E->getExprLoc(), TInfo.Width); 1227 break; 1228 } 1229 1230 if (E->isOpenCL()) { 1231 LibCallName = std::string("__opencl") + 1232 StringRef(LibCallName).drop_front(1).str(); 1233 1234 } 1235 // Optimized functions have the size in their name. 1236 if (UseOptimizedLibcall) 1237 LibCallName += "_" + llvm::utostr(Size); 1238 // By default, assume we return a value of the atomic type. 1239 if (!HaveRetTy) { 1240 if (UseOptimizedLibcall) { 1241 // Value is returned directly. 1242 // The function returns an appropriately sized integer type. 1243 RetTy = getContext().getIntTypeForBitwidth( 1244 getContext().toBits(TInfo.Width), /*Signed=*/false); 1245 } else { 1246 // Value is returned through parameter before the order. 1247 RetTy = getContext().VoidTy; 1248 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 1249 getContext().VoidPtrTy); 1250 } 1251 } 1252 // order is always the last parameter 1253 Args.add(RValue::get(Order), 1254 getContext().IntTy); 1255 if (E->isOpenCL()) 1256 Args.add(RValue::get(Scope), getContext().IntTy); 1257 1258 // PostOp is only needed for the atomic_*_fetch operations, and 1259 // thus is only needed for and implemented in the 1260 // UseOptimizedLibcall codepath. 1261 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax)); 1262 1263 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1264 // The value is returned directly from the libcall. 1265 if (E->isCmpXChg()) 1266 return Res; 1267 1268 // The value is returned directly for optimized libcalls but the expr 1269 // provided an out-param. 1270 if (UseOptimizedLibcall && Res.getScalarVal()) { 1271 llvm::Value *ResVal = Res.getScalarVal(); 1272 if (PostOpMinMax) { 1273 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1274 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(), 1275 E->getValueType()->isSignedIntegerType(), 1276 ResVal, LoadVal1); 1277 } else if (PostOp) { 1278 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1279 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1280 } 1281 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1282 ResVal = Builder.CreateNot(ResVal); 1283 1284 Builder.CreateStore( 1285 ResVal, 1286 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 1287 } 1288 1289 if (RValTy->isVoidType()) 1290 return RValue::get(nullptr); 1291 1292 return convertTempToRValue( 1293 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1294 RValTy, E->getExprLoc()); 1295 } 1296 1297 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1298 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1299 E->getOp() == AtomicExpr::AO__atomic_store || 1300 E->getOp() == AtomicExpr::AO__atomic_store_n; 1301 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1302 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1303 E->getOp() == AtomicExpr::AO__atomic_load || 1304 E->getOp() == AtomicExpr::AO__atomic_load_n; 1305 1306 if (isa<llvm::ConstantInt>(Order)) { 1307 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1308 // We should not ever get to a case where the ordering isn't a valid C ABI 1309 // value, but it's hard to enforce that in general. 1310 if (llvm::isValidAtomicOrderingCABI(ord)) 1311 switch ((llvm::AtomicOrderingCABI)ord) { 1312 case llvm::AtomicOrderingCABI::relaxed: 1313 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1314 llvm::AtomicOrdering::Monotonic, Scope); 1315 break; 1316 case llvm::AtomicOrderingCABI::consume: 1317 case llvm::AtomicOrderingCABI::acquire: 1318 if (IsStore) 1319 break; // Avoid crashing on code with undefined behavior 1320 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1321 llvm::AtomicOrdering::Acquire, Scope); 1322 break; 1323 case llvm::AtomicOrderingCABI::release: 1324 if (IsLoad) 1325 break; // Avoid crashing on code with undefined behavior 1326 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1327 llvm::AtomicOrdering::Release, Scope); 1328 break; 1329 case llvm::AtomicOrderingCABI::acq_rel: 1330 if (IsLoad || IsStore) 1331 break; // Avoid crashing on code with undefined behavior 1332 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1333 llvm::AtomicOrdering::AcquireRelease, Scope); 1334 break; 1335 case llvm::AtomicOrderingCABI::seq_cst: 1336 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1337 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1338 break; 1339 } 1340 if (RValTy->isVoidType()) 1341 return RValue::get(nullptr); 1342 1343 return convertTempToRValue( 1344 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1345 Dest.getAddressSpace())), 1346 RValTy, E->getExprLoc()); 1347 } 1348 1349 // Long case, when Order isn't obviously constant. 1350 1351 // Create all the relevant BB's 1352 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1353 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1354 *SeqCstBB = nullptr; 1355 MonotonicBB = createBasicBlock("monotonic", CurFn); 1356 if (!IsStore) 1357 AcquireBB = createBasicBlock("acquire", CurFn); 1358 if (!IsLoad) 1359 ReleaseBB = createBasicBlock("release", CurFn); 1360 if (!IsLoad && !IsStore) 1361 AcqRelBB = createBasicBlock("acqrel", CurFn); 1362 SeqCstBB = createBasicBlock("seqcst", CurFn); 1363 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1364 1365 // Create the switch for the split 1366 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1367 // doesn't matter unless someone is crazy enough to use something that 1368 // doesn't fold to a constant for the ordering. 1369 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1370 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1371 1372 // Emit all the different atomics 1373 Builder.SetInsertPoint(MonotonicBB); 1374 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1375 llvm::AtomicOrdering::Monotonic, Scope); 1376 Builder.CreateBr(ContBB); 1377 if (!IsStore) { 1378 Builder.SetInsertPoint(AcquireBB); 1379 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1380 llvm::AtomicOrdering::Acquire, Scope); 1381 Builder.CreateBr(ContBB); 1382 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1383 AcquireBB); 1384 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1385 AcquireBB); 1386 } 1387 if (!IsLoad) { 1388 Builder.SetInsertPoint(ReleaseBB); 1389 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1390 llvm::AtomicOrdering::Release, Scope); 1391 Builder.CreateBr(ContBB); 1392 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1393 ReleaseBB); 1394 } 1395 if (!IsLoad && !IsStore) { 1396 Builder.SetInsertPoint(AcqRelBB); 1397 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1398 llvm::AtomicOrdering::AcquireRelease, Scope); 1399 Builder.CreateBr(ContBB); 1400 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1401 AcqRelBB); 1402 } 1403 Builder.SetInsertPoint(SeqCstBB); 1404 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1405 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1406 Builder.CreateBr(ContBB); 1407 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1408 SeqCstBB); 1409 1410 // Cleanup and return 1411 Builder.SetInsertPoint(ContBB); 1412 if (RValTy->isVoidType()) 1413 return RValue::get(nullptr); 1414 1415 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1416 return convertTempToRValue( 1417 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1418 Dest.getAddressSpace())), 1419 RValTy, E->getExprLoc()); 1420 } 1421 1422 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1423 unsigned addrspace = 1424 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 1425 llvm::IntegerType *ty = 1426 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1427 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 1428 } 1429 1430 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1431 llvm::Type *Ty = Addr.getElementType(); 1432 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1433 if (SourceSizeInBits != AtomicSizeInBits) { 1434 Address Tmp = CreateTempAlloca(); 1435 CGF.Builder.CreateMemCpy(Tmp, Addr, 1436 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1437 Addr = Tmp; 1438 } 1439 1440 return emitCastToAtomicIntPointer(Addr); 1441 } 1442 1443 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1444 AggValueSlot resultSlot, 1445 SourceLocation loc, 1446 bool asValue) const { 1447 if (LVal.isSimple()) { 1448 if (EvaluationKind == TEK_Aggregate) 1449 return resultSlot.asRValue(); 1450 1451 // Drill into the padding structure if we have one. 1452 if (hasPadding()) 1453 addr = CGF.Builder.CreateStructGEP(addr, 0); 1454 1455 // Otherwise, just convert the temporary to an r-value using the 1456 // normal conversion routine. 1457 return CGF.convertTempToRValue(addr, getValueType(), loc); 1458 } 1459 if (!asValue) 1460 // Get RValue from temp memory as atomic for non-simple lvalues 1461 return RValue::get(CGF.Builder.CreateLoad(addr)); 1462 if (LVal.isBitField()) 1463 return CGF.EmitLoadOfBitfieldLValue( 1464 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1465 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1466 if (LVal.isVectorElt()) 1467 return CGF.EmitLoadOfLValue( 1468 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1469 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1470 assert(LVal.isExtVectorElt()); 1471 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1472 addr, LVal.getExtVectorElts(), LVal.getType(), 1473 LVal.getBaseInfo(), TBAAAccessInfo())); 1474 } 1475 1476 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1477 AggValueSlot ResultSlot, 1478 SourceLocation Loc, 1479 bool AsValue) const { 1480 // Try not to in some easy cases. 1481 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1482 if (getEvaluationKind() == TEK_Scalar && 1483 (((!LVal.isBitField() || 1484 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1485 !hasPadding()) || 1486 !AsValue)) { 1487 auto *ValTy = AsValue 1488 ? CGF.ConvertTypeForMem(ValueTy) 1489 : getAtomicAddress().getType()->getPointerElementType(); 1490 if (ValTy->isIntegerTy()) { 1491 assert(IntVal->getType() == ValTy && "Different integer types."); 1492 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1493 } else if (ValTy->isPointerTy()) 1494 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1495 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1496 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1497 } 1498 1499 // Create a temporary. This needs to be big enough to hold the 1500 // atomic integer. 1501 Address Temp = Address::invalid(); 1502 bool TempIsVolatile = false; 1503 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1504 assert(!ResultSlot.isIgnored()); 1505 Temp = ResultSlot.getAddress(); 1506 TempIsVolatile = ResultSlot.isVolatile(); 1507 } else { 1508 Temp = CreateTempAlloca(); 1509 } 1510 1511 // Slam the integer into the temporary. 1512 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1513 CGF.Builder.CreateStore(IntVal, CastTemp) 1514 ->setVolatile(TempIsVolatile); 1515 1516 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1517 } 1518 1519 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1520 llvm::AtomicOrdering AO, bool) { 1521 // void __atomic_load(size_t size, void *mem, void *return, int order); 1522 CallArgList Args; 1523 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1524 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1525 CGF.getContext().VoidPtrTy); 1526 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1527 CGF.getContext().VoidPtrTy); 1528 Args.add( 1529 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1530 CGF.getContext().IntTy); 1531 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1532 } 1533 1534 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1535 bool IsVolatile) { 1536 // Okay, we're doing this natively. 1537 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1538 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1539 Load->setAtomic(AO); 1540 1541 // Other decoration. 1542 if (IsVolatile) 1543 Load->setVolatile(true); 1544 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1545 return Load; 1546 } 1547 1548 /// An LValue is a candidate for having its loads and stores be made atomic if 1549 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1550 /// performing such an operation can be performed without a libcall. 1551 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1552 if (!CGM.getCodeGenOpts().MSVolatile) return false; 1553 AtomicInfo AI(*this, LV); 1554 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1555 // An atomic is inline if we don't need to use a libcall. 1556 bool AtomicIsInline = !AI.shouldUseLibcall(); 1557 // MSVC doesn't seem to do this for types wider than a pointer. 1558 if (getContext().getTypeSize(LV.getType()) > 1559 getContext().getTypeSize(getContext().getIntPtrType())) 1560 return false; 1561 return IsVolatile && AtomicIsInline; 1562 } 1563 1564 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1565 AggValueSlot Slot) { 1566 llvm::AtomicOrdering AO; 1567 bool IsVolatile = LV.isVolatileQualified(); 1568 if (LV.getType()->isAtomicType()) { 1569 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1570 } else { 1571 AO = llvm::AtomicOrdering::Acquire; 1572 IsVolatile = true; 1573 } 1574 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1575 } 1576 1577 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1578 bool AsValue, llvm::AtomicOrdering AO, 1579 bool IsVolatile) { 1580 // Check whether we should use a library call. 1581 if (shouldUseLibcall()) { 1582 Address TempAddr = Address::invalid(); 1583 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1584 assert(getEvaluationKind() == TEK_Aggregate); 1585 TempAddr = ResultSlot.getAddress(); 1586 } else 1587 TempAddr = CreateTempAlloca(); 1588 1589 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1590 1591 // Okay, turn that back into the original value or whole atomic (for 1592 // non-simple lvalues) type. 1593 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1594 } 1595 1596 // Okay, we're doing this natively. 1597 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1598 1599 // If we're ignoring an aggregate return, don't do anything. 1600 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1601 return RValue::getAggregate(Address::invalid(), false); 1602 1603 // Okay, turn that back into the original value or atomic (for non-simple 1604 // lvalues) type. 1605 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1606 } 1607 1608 /// Emit a load from an l-value of atomic type. Note that the r-value 1609 /// we produce is an r-value of the atomic *value* type. 1610 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1611 llvm::AtomicOrdering AO, bool IsVolatile, 1612 AggValueSlot resultSlot) { 1613 AtomicInfo Atomics(*this, src); 1614 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1615 IsVolatile); 1616 } 1617 1618 /// Copy an r-value into memory as part of storing to an atomic type. 1619 /// This needs to create a bit-pattern suitable for atomic operations. 1620 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1621 assert(LVal.isSimple()); 1622 // If we have an r-value, the rvalue should be of the atomic type, 1623 // which means that the caller is responsible for having zeroed 1624 // any padding. Just do an aggregate copy of that type. 1625 if (rvalue.isAggregate()) { 1626 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1627 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1628 getAtomicType()); 1629 bool IsVolatile = rvalue.isVolatileQualified() || 1630 LVal.isVolatileQualified(); 1631 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1632 AggValueSlot::DoesNotOverlap, IsVolatile); 1633 return; 1634 } 1635 1636 // Okay, otherwise we're copying stuff. 1637 1638 // Zero out the buffer if necessary. 1639 emitMemSetZeroIfNecessary(); 1640 1641 // Drill past the padding if present. 1642 LValue TempLVal = projectValue(); 1643 1644 // Okay, store the rvalue in. 1645 if (rvalue.isScalar()) { 1646 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1647 } else { 1648 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1649 } 1650 } 1651 1652 1653 /// Materialize an r-value into memory for the purposes of storing it 1654 /// to an atomic type. 1655 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1656 // Aggregate r-values are already in memory, and EmitAtomicStore 1657 // requires them to be values of the atomic type. 1658 if (rvalue.isAggregate()) 1659 return rvalue.getAggregateAddress(); 1660 1661 // Otherwise, make a temporary and materialize into it. 1662 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1663 AtomicInfo Atomics(CGF, TempLV); 1664 Atomics.emitCopyIntoMemory(rvalue); 1665 return TempLV.getAddress(CGF); 1666 } 1667 1668 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1669 // If we've got a scalar value of the right size, try to avoid going 1670 // through memory. 1671 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1672 llvm::Value *Value = RVal.getScalarVal(); 1673 if (isa<llvm::IntegerType>(Value->getType())) 1674 return CGF.EmitToMemory(Value, ValueTy); 1675 else { 1676 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1677 CGF.getLLVMContext(), 1678 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1679 if (isa<llvm::PointerType>(Value->getType())) 1680 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1681 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1682 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1683 } 1684 } 1685 // Otherwise, we need to go through memory. 1686 // Put the r-value in memory. 1687 Address Addr = materializeRValue(RVal); 1688 1689 // Cast the temporary to the atomic int type and pull a value out. 1690 Addr = emitCastToAtomicIntPointer(Addr); 1691 return CGF.Builder.CreateLoad(Addr); 1692 } 1693 1694 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1695 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1696 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1697 // Do the atomic store. 1698 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1699 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1700 ExpectedVal, DesiredVal, 1701 Success, Failure); 1702 // Other decoration. 1703 Inst->setVolatile(LVal.isVolatileQualified()); 1704 Inst->setWeak(IsWeak); 1705 1706 // Okay, turn that back into the original value type. 1707 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1708 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1709 return std::make_pair(PreviousVal, SuccessFailureVal); 1710 } 1711 1712 llvm::Value * 1713 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1714 llvm::Value *DesiredAddr, 1715 llvm::AtomicOrdering Success, 1716 llvm::AtomicOrdering Failure) { 1717 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1718 // void *desired, int success, int failure); 1719 CallArgList Args; 1720 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1721 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1722 CGF.getContext().VoidPtrTy); 1723 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1724 CGF.getContext().VoidPtrTy); 1725 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1726 CGF.getContext().VoidPtrTy); 1727 Args.add(RValue::get( 1728 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1729 CGF.getContext().IntTy); 1730 Args.add(RValue::get( 1731 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1732 CGF.getContext().IntTy); 1733 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1734 CGF.getContext().BoolTy, Args); 1735 1736 return SuccessFailureRVal.getScalarVal(); 1737 } 1738 1739 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1740 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1741 llvm::AtomicOrdering Failure, bool IsWeak) { 1742 if (isStrongerThan(Failure, Success)) 1743 // Don't assert on undefined behavior "failure argument shall be no stronger 1744 // than the success argument". 1745 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 1746 1747 // Check whether we should use a library call. 1748 if (shouldUseLibcall()) { 1749 // Produce a source address. 1750 Address ExpectedAddr = materializeRValue(Expected); 1751 Address DesiredAddr = materializeRValue(Desired); 1752 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1753 DesiredAddr.getPointer(), 1754 Success, Failure); 1755 return std::make_pair( 1756 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1757 SourceLocation(), /*AsValue=*/false), 1758 Res); 1759 } 1760 1761 // If we've got a scalar value of the right size, try to avoid going 1762 // through memory. 1763 auto *ExpectedVal = convertRValueToInt(Expected); 1764 auto *DesiredVal = convertRValueToInt(Desired); 1765 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1766 Failure, IsWeak); 1767 return std::make_pair( 1768 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1769 SourceLocation(), /*AsValue=*/false), 1770 Res.second); 1771 } 1772 1773 static void 1774 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1775 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1776 Address DesiredAddr) { 1777 RValue UpRVal; 1778 LValue AtomicLVal = Atomics.getAtomicLValue(); 1779 LValue DesiredLVal; 1780 if (AtomicLVal.isSimple()) { 1781 UpRVal = OldRVal; 1782 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1783 } else { 1784 // Build new lvalue for temp address. 1785 Address Ptr = Atomics.materializeRValue(OldRVal); 1786 LValue UpdateLVal; 1787 if (AtomicLVal.isBitField()) { 1788 UpdateLVal = 1789 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1790 AtomicLVal.getType(), 1791 AtomicLVal.getBaseInfo(), 1792 AtomicLVal.getTBAAInfo()); 1793 DesiredLVal = 1794 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1795 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1796 AtomicLVal.getTBAAInfo()); 1797 } else if (AtomicLVal.isVectorElt()) { 1798 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1799 AtomicLVal.getType(), 1800 AtomicLVal.getBaseInfo(), 1801 AtomicLVal.getTBAAInfo()); 1802 DesiredLVal = LValue::MakeVectorElt( 1803 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1804 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1805 } else { 1806 assert(AtomicLVal.isExtVectorElt()); 1807 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1808 AtomicLVal.getType(), 1809 AtomicLVal.getBaseInfo(), 1810 AtomicLVal.getTBAAInfo()); 1811 DesiredLVal = LValue::MakeExtVectorElt( 1812 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1813 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1814 } 1815 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1816 } 1817 // Store new value in the corresponding memory area. 1818 RValue NewRVal = UpdateOp(UpRVal); 1819 if (NewRVal.isScalar()) { 1820 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1821 } else { 1822 assert(NewRVal.isComplex()); 1823 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1824 /*isInit=*/false); 1825 } 1826 } 1827 1828 void AtomicInfo::EmitAtomicUpdateLibcall( 1829 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1830 bool IsVolatile) { 1831 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1832 1833 Address ExpectedAddr = CreateTempAlloca(); 1834 1835 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1836 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1837 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1838 CGF.EmitBlock(ContBB); 1839 Address DesiredAddr = CreateTempAlloca(); 1840 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1841 requiresMemSetZero(getAtomicAddress().getElementType())) { 1842 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1843 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1844 } 1845 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1846 AggValueSlot::ignored(), 1847 SourceLocation(), /*AsValue=*/false); 1848 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1849 auto *Res = 1850 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1851 DesiredAddr.getPointer(), 1852 AO, Failure); 1853 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1854 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1855 } 1856 1857 void AtomicInfo::EmitAtomicUpdateOp( 1858 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1859 bool IsVolatile) { 1860 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1861 1862 // Do the atomic load. 1863 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); 1864 // For non-simple lvalues perform compare-and-swap procedure. 1865 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1866 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1867 auto *CurBB = CGF.Builder.GetInsertBlock(); 1868 CGF.EmitBlock(ContBB); 1869 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1870 /*NumReservedValues=*/2); 1871 PHI->addIncoming(OldVal, CurBB); 1872 Address NewAtomicAddr = CreateTempAlloca(); 1873 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1874 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1875 requiresMemSetZero(getAtomicAddress().getElementType())) { 1876 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1877 } 1878 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1879 SourceLocation(), /*AsValue=*/false); 1880 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1881 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1882 // Try to write new value using cmpxchg operation. 1883 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1884 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1885 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1886 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1887 } 1888 1889 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1890 RValue UpdateRVal, Address DesiredAddr) { 1891 LValue AtomicLVal = Atomics.getAtomicLValue(); 1892 LValue DesiredLVal; 1893 // Build new lvalue for temp address. 1894 if (AtomicLVal.isBitField()) { 1895 DesiredLVal = 1896 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1897 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1898 AtomicLVal.getTBAAInfo()); 1899 } else if (AtomicLVal.isVectorElt()) { 1900 DesiredLVal = 1901 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1902 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1903 AtomicLVal.getTBAAInfo()); 1904 } else { 1905 assert(AtomicLVal.isExtVectorElt()); 1906 DesiredLVal = LValue::MakeExtVectorElt( 1907 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1908 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1909 } 1910 // Store new value in the corresponding memory area. 1911 assert(UpdateRVal.isScalar()); 1912 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1913 } 1914 1915 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1916 RValue UpdateRVal, bool IsVolatile) { 1917 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1918 1919 Address ExpectedAddr = CreateTempAlloca(); 1920 1921 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1922 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1923 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1924 CGF.EmitBlock(ContBB); 1925 Address DesiredAddr = CreateTempAlloca(); 1926 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1927 requiresMemSetZero(getAtomicAddress().getElementType())) { 1928 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1929 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1930 } 1931 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1932 auto *Res = 1933 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1934 DesiredAddr.getPointer(), 1935 AO, Failure); 1936 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1937 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1938 } 1939 1940 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1941 bool IsVolatile) { 1942 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1943 1944 // Do the atomic load. 1945 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); 1946 // For non-simple lvalues perform compare-and-swap procedure. 1947 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1948 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1949 auto *CurBB = CGF.Builder.GetInsertBlock(); 1950 CGF.EmitBlock(ContBB); 1951 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1952 /*NumReservedValues=*/2); 1953 PHI->addIncoming(OldVal, CurBB); 1954 Address NewAtomicAddr = CreateTempAlloca(); 1955 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1956 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1957 requiresMemSetZero(getAtomicAddress().getElementType())) { 1958 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1959 } 1960 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1961 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1962 // Try to write new value using cmpxchg operation. 1963 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1964 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1965 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1966 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1967 } 1968 1969 void AtomicInfo::EmitAtomicUpdate( 1970 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1971 bool IsVolatile) { 1972 if (shouldUseLibcall()) { 1973 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1974 } else { 1975 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1976 } 1977 } 1978 1979 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1980 bool IsVolatile) { 1981 if (shouldUseLibcall()) { 1982 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1983 } else { 1984 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1985 } 1986 } 1987 1988 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1989 bool isInit) { 1990 bool IsVolatile = lvalue.isVolatileQualified(); 1991 llvm::AtomicOrdering AO; 1992 if (lvalue.getType()->isAtomicType()) { 1993 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1994 } else { 1995 AO = llvm::AtomicOrdering::Release; 1996 IsVolatile = true; 1997 } 1998 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1999 } 2000 2001 /// Emit a store to an l-value of atomic type. 2002 /// 2003 /// Note that the r-value is expected to be an r-value *of the atomic 2004 /// type*; this means that for aggregate r-values, it should include 2005 /// storage for any padding that was necessary. 2006 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 2007 llvm::AtomicOrdering AO, bool IsVolatile, 2008 bool isInit) { 2009 // If this is an aggregate r-value, it should agree in type except 2010 // maybe for address-space qualification. 2011 assert(!rvalue.isAggregate() || 2012 rvalue.getAggregateAddress().getElementType() == 2013 dest.getAddress(*this).getElementType()); 2014 2015 AtomicInfo atomics(*this, dest); 2016 LValue LVal = atomics.getAtomicLValue(); 2017 2018 // If this is an initialization, just put the value there normally. 2019 if (LVal.isSimple()) { 2020 if (isInit) { 2021 atomics.emitCopyIntoMemory(rvalue); 2022 return; 2023 } 2024 2025 // Check whether we should use a library call. 2026 if (atomics.shouldUseLibcall()) { 2027 // Produce a source address. 2028 Address srcAddr = atomics.materializeRValue(rvalue); 2029 2030 // void __atomic_store(size_t size, void *mem, void *val, int order) 2031 CallArgList args; 2032 args.add(RValue::get(atomics.getAtomicSizeValue()), 2033 getContext().getSizeType()); 2034 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 2035 getContext().VoidPtrTy); 2036 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 2037 getContext().VoidPtrTy); 2038 args.add( 2039 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 2040 getContext().IntTy); 2041 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 2042 return; 2043 } 2044 2045 // Okay, we're doing this natively. 2046 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 2047 2048 // Do the atomic store. 2049 Address addr = 2050 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 2051 intValue = Builder.CreateIntCast( 2052 intValue, addr.getElementType(), /*isSigned=*/false); 2053 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 2054 2055 if (AO == llvm::AtomicOrdering::Acquire) 2056 AO = llvm::AtomicOrdering::Monotonic; 2057 else if (AO == llvm::AtomicOrdering::AcquireRelease) 2058 AO = llvm::AtomicOrdering::Release; 2059 // Initializations don't need to be atomic. 2060 if (!isInit) 2061 store->setAtomic(AO); 2062 2063 // Other decoration. 2064 if (IsVolatile) 2065 store->setVolatile(true); 2066 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 2067 return; 2068 } 2069 2070 // Emit simple atomic update operation. 2071 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 2072 } 2073 2074 /// Emit a compare-and-exchange op for atomic type. 2075 /// 2076 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 2077 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 2078 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 2079 AggValueSlot Slot) { 2080 // If this is an aggregate r-value, it should agree in type except 2081 // maybe for address-space qualification. 2082 assert(!Expected.isAggregate() || 2083 Expected.getAggregateAddress().getElementType() == 2084 Obj.getAddress(*this).getElementType()); 2085 assert(!Desired.isAggregate() || 2086 Desired.getAggregateAddress().getElementType() == 2087 Obj.getAddress(*this).getElementType()); 2088 AtomicInfo Atomics(*this, Obj); 2089 2090 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 2091 IsWeak); 2092 } 2093 2094 void CodeGenFunction::EmitAtomicUpdate( 2095 LValue LVal, llvm::AtomicOrdering AO, 2096 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 2097 AtomicInfo Atomics(*this, LVal); 2098 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 2099 } 2100 2101 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 2102 AtomicInfo atomics(*this, dest); 2103 2104 switch (atomics.getEvaluationKind()) { 2105 case TEK_Scalar: { 2106 llvm::Value *value = EmitScalarExpr(init); 2107 atomics.emitCopyIntoMemory(RValue::get(value)); 2108 return; 2109 } 2110 2111 case TEK_Complex: { 2112 ComplexPairTy value = EmitComplexExpr(init); 2113 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2114 return; 2115 } 2116 2117 case TEK_Aggregate: { 2118 // Fix up the destination if the initializer isn't an expression 2119 // of atomic type. 2120 bool Zeroed = false; 2121 if (!init->getType()->isAtomicType()) { 2122 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2123 dest = atomics.projectValue(); 2124 } 2125 2126 // Evaluate the expression directly into the destination. 2127 AggValueSlot slot = AggValueSlot::forLValue( 2128 dest, *this, AggValueSlot::IsNotDestructed, 2129 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, 2130 AggValueSlot::DoesNotOverlap, 2131 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); 2132 2133 EmitAggExpr(init, slot); 2134 return; 2135 } 2136 } 2137 llvm_unreachable("bad evaluation kind"); 2138 } 2139