1 //===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Stmt nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGDebugInfo.h"
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/Attr.h"
19 #include "clang/AST/Expr.h"
20 #include "clang/AST/Stmt.h"
21 #include "clang/AST/StmtVisitor.h"
22 #include "clang/Basic/Builtins.h"
23 #include "clang/Basic/DiagnosticSema.h"
24 #include "clang/Basic/PrettyStackTrace.h"
25 #include "clang/Basic/SourceManager.h"
26 #include "clang/Basic/TargetInfo.h"
27 #include "llvm/ADT/SmallSet.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/IR/Assumptions.h"
30 #include "llvm/IR/DataLayout.h"
31 #include "llvm/IR/InlineAsm.h"
32 #include "llvm/IR/Intrinsics.h"
33 #include "llvm/IR/MDBuilder.h"
34 #include "llvm/Support/SaveAndRestore.h"
35 
36 using namespace clang;
37 using namespace CodeGen;
38 
39 //===----------------------------------------------------------------------===//
40 //                              Statement Emission
41 //===----------------------------------------------------------------------===//
42 
43 void CodeGenFunction::EmitStopPoint(const Stmt *S) {
44   if (CGDebugInfo *DI = getDebugInfo()) {
45     SourceLocation Loc;
46     Loc = S->getBeginLoc();
47     DI->EmitLocation(Builder, Loc);
48 
49     LastStopPoint = Loc;
50   }
51 }
52 
53 void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
54   assert(S && "Null statement?");
55   PGO.setCurrentStmt(S);
56 
57   // These statements have their own debug info handling.
58   if (EmitSimpleStmt(S, Attrs))
59     return;
60 
61   // Check if we are generating unreachable code.
62   if (!HaveInsertPoint()) {
63     // If so, and the statement doesn't contain a label, then we do not need to
64     // generate actual code. This is safe because (1) the current point is
65     // unreachable, so we don't need to execute the code, and (2) we've already
66     // handled the statements which update internal data structures (like the
67     // local variable map) which could be used by subsequent statements.
68     if (!ContainsLabel(S)) {
69       // Verify that any decl statements were handled as simple, they may be in
70       // scope of subsequent reachable statements.
71       assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
72       return;
73     }
74 
75     // Otherwise, make a new block to hold the code.
76     EnsureInsertPoint();
77   }
78 
79   // Generate a stoppoint if we are emitting debug info.
80   EmitStopPoint(S);
81 
82   // Ignore all OpenMP directives except for simd if OpenMP with Simd is
83   // enabled.
84   if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
85     if (const auto *D = dyn_cast<OMPExecutableDirective>(S)) {
86       EmitSimpleOMPExecutableDirective(*D);
87       return;
88     }
89   }
90 
91   switch (S->getStmtClass()) {
92   case Stmt::NoStmtClass:
93   case Stmt::CXXCatchStmtClass:
94   case Stmt::SEHExceptStmtClass:
95   case Stmt::SEHFinallyStmtClass:
96   case Stmt::MSDependentExistsStmtClass:
97     llvm_unreachable("invalid statement class to emit generically");
98   case Stmt::NullStmtClass:
99   case Stmt::CompoundStmtClass:
100   case Stmt::DeclStmtClass:
101   case Stmt::LabelStmtClass:
102   case Stmt::AttributedStmtClass:
103   case Stmt::GotoStmtClass:
104   case Stmt::BreakStmtClass:
105   case Stmt::ContinueStmtClass:
106   case Stmt::DefaultStmtClass:
107   case Stmt::CaseStmtClass:
108   case Stmt::SEHLeaveStmtClass:
109     llvm_unreachable("should have emitted these statements as simple");
110 
111 #define STMT(Type, Base)
112 #define ABSTRACT_STMT(Op)
113 #define EXPR(Type, Base) \
114   case Stmt::Type##Class:
115 #include "clang/AST/StmtNodes.inc"
116   {
117     // Remember the block we came in on.
118     llvm::BasicBlock *incoming = Builder.GetInsertBlock();
119     assert(incoming && "expression emission must have an insertion point");
120 
121     EmitIgnoredExpr(cast<Expr>(S));
122 
123     llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
124     assert(outgoing && "expression emission cleared block!");
125 
126     // The expression emitters assume (reasonably!) that the insertion
127     // point is always set.  To maintain that, the call-emission code
128     // for noreturn functions has to enter a new block with no
129     // predecessors.  We want to kill that block and mark the current
130     // insertion point unreachable in the common case of a call like
131     // "exit();".  Since expression emission doesn't otherwise create
132     // blocks with no predecessors, we can just test for that.
133     // However, we must be careful not to do this to our incoming
134     // block, because *statement* emission does sometimes create
135     // reachable blocks which will have no predecessors until later in
136     // the function.  This occurs with, e.g., labels that are not
137     // reachable by fallthrough.
138     if (incoming != outgoing && outgoing->use_empty()) {
139       outgoing->eraseFromParent();
140       Builder.ClearInsertionPoint();
141     }
142     break;
143   }
144 
145   case Stmt::IndirectGotoStmtClass:
146     EmitIndirectGotoStmt(cast<IndirectGotoStmt>(*S)); break;
147 
148   case Stmt::IfStmtClass:      EmitIfStmt(cast<IfStmt>(*S));              break;
149   case Stmt::WhileStmtClass:   EmitWhileStmt(cast<WhileStmt>(*S), Attrs); break;
150   case Stmt::DoStmtClass:      EmitDoStmt(cast<DoStmt>(*S), Attrs);       break;
151   case Stmt::ForStmtClass:     EmitForStmt(cast<ForStmt>(*S), Attrs);     break;
152 
153   case Stmt::ReturnStmtClass:  EmitReturnStmt(cast<ReturnStmt>(*S));      break;
154 
155   case Stmt::SwitchStmtClass:  EmitSwitchStmt(cast<SwitchStmt>(*S));      break;
156   case Stmt::GCCAsmStmtClass:  // Intentional fall-through.
157   case Stmt::MSAsmStmtClass:   EmitAsmStmt(cast<AsmStmt>(*S));            break;
158   case Stmt::CoroutineBodyStmtClass:
159     EmitCoroutineBody(cast<CoroutineBodyStmt>(*S));
160     break;
161   case Stmt::CoreturnStmtClass:
162     EmitCoreturnStmt(cast<CoreturnStmt>(*S));
163     break;
164   case Stmt::CapturedStmtClass: {
165     const CapturedStmt *CS = cast<CapturedStmt>(S);
166     EmitCapturedStmt(*CS, CS->getCapturedRegionKind());
167     }
168     break;
169   case Stmt::ObjCAtTryStmtClass:
170     EmitObjCAtTryStmt(cast<ObjCAtTryStmt>(*S));
171     break;
172   case Stmt::ObjCAtCatchStmtClass:
173     llvm_unreachable(
174                     "@catch statements should be handled by EmitObjCAtTryStmt");
175   case Stmt::ObjCAtFinallyStmtClass:
176     llvm_unreachable(
177                   "@finally statements should be handled by EmitObjCAtTryStmt");
178   case Stmt::ObjCAtThrowStmtClass:
179     EmitObjCAtThrowStmt(cast<ObjCAtThrowStmt>(*S));
180     break;
181   case Stmt::ObjCAtSynchronizedStmtClass:
182     EmitObjCAtSynchronizedStmt(cast<ObjCAtSynchronizedStmt>(*S));
183     break;
184   case Stmt::ObjCForCollectionStmtClass:
185     EmitObjCForCollectionStmt(cast<ObjCForCollectionStmt>(*S));
186     break;
187   case Stmt::ObjCAutoreleasePoolStmtClass:
188     EmitObjCAutoreleasePoolStmt(cast<ObjCAutoreleasePoolStmt>(*S));
189     break;
190 
191   case Stmt::CXXTryStmtClass:
192     EmitCXXTryStmt(cast<CXXTryStmt>(*S));
193     break;
194   case Stmt::CXXForRangeStmtClass:
195     EmitCXXForRangeStmt(cast<CXXForRangeStmt>(*S), Attrs);
196     break;
197   case Stmt::SEHTryStmtClass:
198     EmitSEHTryStmt(cast<SEHTryStmt>(*S));
199     break;
200   case Stmt::OMPMetaDirectiveClass:
201     EmitOMPMetaDirective(cast<OMPMetaDirective>(*S));
202     break;
203   case Stmt::OMPCanonicalLoopClass:
204     EmitOMPCanonicalLoop(cast<OMPCanonicalLoop>(S));
205     break;
206   case Stmt::OMPParallelDirectiveClass:
207     EmitOMPParallelDirective(cast<OMPParallelDirective>(*S));
208     break;
209   case Stmt::OMPSimdDirectiveClass:
210     EmitOMPSimdDirective(cast<OMPSimdDirective>(*S));
211     break;
212   case Stmt::OMPTileDirectiveClass:
213     EmitOMPTileDirective(cast<OMPTileDirective>(*S));
214     break;
215   case Stmt::OMPUnrollDirectiveClass:
216     EmitOMPUnrollDirective(cast<OMPUnrollDirective>(*S));
217     break;
218   case Stmt::OMPForDirectiveClass:
219     EmitOMPForDirective(cast<OMPForDirective>(*S));
220     break;
221   case Stmt::OMPForSimdDirectiveClass:
222     EmitOMPForSimdDirective(cast<OMPForSimdDirective>(*S));
223     break;
224   case Stmt::OMPSectionsDirectiveClass:
225     EmitOMPSectionsDirective(cast<OMPSectionsDirective>(*S));
226     break;
227   case Stmt::OMPSectionDirectiveClass:
228     EmitOMPSectionDirective(cast<OMPSectionDirective>(*S));
229     break;
230   case Stmt::OMPSingleDirectiveClass:
231     EmitOMPSingleDirective(cast<OMPSingleDirective>(*S));
232     break;
233   case Stmt::OMPMasterDirectiveClass:
234     EmitOMPMasterDirective(cast<OMPMasterDirective>(*S));
235     break;
236   case Stmt::OMPCriticalDirectiveClass:
237     EmitOMPCriticalDirective(cast<OMPCriticalDirective>(*S));
238     break;
239   case Stmt::OMPParallelForDirectiveClass:
240     EmitOMPParallelForDirective(cast<OMPParallelForDirective>(*S));
241     break;
242   case Stmt::OMPParallelForSimdDirectiveClass:
243     EmitOMPParallelForSimdDirective(cast<OMPParallelForSimdDirective>(*S));
244     break;
245   case Stmt::OMPParallelMasterDirectiveClass:
246     EmitOMPParallelMasterDirective(cast<OMPParallelMasterDirective>(*S));
247     break;
248   case Stmt::OMPParallelSectionsDirectiveClass:
249     EmitOMPParallelSectionsDirective(cast<OMPParallelSectionsDirective>(*S));
250     break;
251   case Stmt::OMPTaskDirectiveClass:
252     EmitOMPTaskDirective(cast<OMPTaskDirective>(*S));
253     break;
254   case Stmt::OMPTaskyieldDirectiveClass:
255     EmitOMPTaskyieldDirective(cast<OMPTaskyieldDirective>(*S));
256     break;
257   case Stmt::OMPBarrierDirectiveClass:
258     EmitOMPBarrierDirective(cast<OMPBarrierDirective>(*S));
259     break;
260   case Stmt::OMPTaskwaitDirectiveClass:
261     EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));
262     break;
263   case Stmt::OMPTaskgroupDirectiveClass:
264     EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S));
265     break;
266   case Stmt::OMPFlushDirectiveClass:
267     EmitOMPFlushDirective(cast<OMPFlushDirective>(*S));
268     break;
269   case Stmt::OMPDepobjDirectiveClass:
270     EmitOMPDepobjDirective(cast<OMPDepobjDirective>(*S));
271     break;
272   case Stmt::OMPScanDirectiveClass:
273     EmitOMPScanDirective(cast<OMPScanDirective>(*S));
274     break;
275   case Stmt::OMPOrderedDirectiveClass:
276     EmitOMPOrderedDirective(cast<OMPOrderedDirective>(*S));
277     break;
278   case Stmt::OMPAtomicDirectiveClass:
279     EmitOMPAtomicDirective(cast<OMPAtomicDirective>(*S));
280     break;
281   case Stmt::OMPTargetDirectiveClass:
282     EmitOMPTargetDirective(cast<OMPTargetDirective>(*S));
283     break;
284   case Stmt::OMPTeamsDirectiveClass:
285     EmitOMPTeamsDirective(cast<OMPTeamsDirective>(*S));
286     break;
287   case Stmt::OMPCancellationPointDirectiveClass:
288     EmitOMPCancellationPointDirective(cast<OMPCancellationPointDirective>(*S));
289     break;
290   case Stmt::OMPCancelDirectiveClass:
291     EmitOMPCancelDirective(cast<OMPCancelDirective>(*S));
292     break;
293   case Stmt::OMPTargetDataDirectiveClass:
294     EmitOMPTargetDataDirective(cast<OMPTargetDataDirective>(*S));
295     break;
296   case Stmt::OMPTargetEnterDataDirectiveClass:
297     EmitOMPTargetEnterDataDirective(cast<OMPTargetEnterDataDirective>(*S));
298     break;
299   case Stmt::OMPTargetExitDataDirectiveClass:
300     EmitOMPTargetExitDataDirective(cast<OMPTargetExitDataDirective>(*S));
301     break;
302   case Stmt::OMPTargetParallelDirectiveClass:
303     EmitOMPTargetParallelDirective(cast<OMPTargetParallelDirective>(*S));
304     break;
305   case Stmt::OMPTargetParallelForDirectiveClass:
306     EmitOMPTargetParallelForDirective(cast<OMPTargetParallelForDirective>(*S));
307     break;
308   case Stmt::OMPTaskLoopDirectiveClass:
309     EmitOMPTaskLoopDirective(cast<OMPTaskLoopDirective>(*S));
310     break;
311   case Stmt::OMPTaskLoopSimdDirectiveClass:
312     EmitOMPTaskLoopSimdDirective(cast<OMPTaskLoopSimdDirective>(*S));
313     break;
314   case Stmt::OMPMasterTaskLoopDirectiveClass:
315     EmitOMPMasterTaskLoopDirective(cast<OMPMasterTaskLoopDirective>(*S));
316     break;
317   case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
318     EmitOMPMasterTaskLoopSimdDirective(
319         cast<OMPMasterTaskLoopSimdDirective>(*S));
320     break;
321   case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
322     EmitOMPParallelMasterTaskLoopDirective(
323         cast<OMPParallelMasterTaskLoopDirective>(*S));
324     break;
325   case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
326     EmitOMPParallelMasterTaskLoopSimdDirective(
327         cast<OMPParallelMasterTaskLoopSimdDirective>(*S));
328     break;
329   case Stmt::OMPDistributeDirectiveClass:
330     EmitOMPDistributeDirective(cast<OMPDistributeDirective>(*S));
331     break;
332   case Stmt::OMPTargetUpdateDirectiveClass:
333     EmitOMPTargetUpdateDirective(cast<OMPTargetUpdateDirective>(*S));
334     break;
335   case Stmt::OMPDistributeParallelForDirectiveClass:
336     EmitOMPDistributeParallelForDirective(
337         cast<OMPDistributeParallelForDirective>(*S));
338     break;
339   case Stmt::OMPDistributeParallelForSimdDirectiveClass:
340     EmitOMPDistributeParallelForSimdDirective(
341         cast<OMPDistributeParallelForSimdDirective>(*S));
342     break;
343   case Stmt::OMPDistributeSimdDirectiveClass:
344     EmitOMPDistributeSimdDirective(cast<OMPDistributeSimdDirective>(*S));
345     break;
346   case Stmt::OMPTargetParallelForSimdDirectiveClass:
347     EmitOMPTargetParallelForSimdDirective(
348         cast<OMPTargetParallelForSimdDirective>(*S));
349     break;
350   case Stmt::OMPTargetSimdDirectiveClass:
351     EmitOMPTargetSimdDirective(cast<OMPTargetSimdDirective>(*S));
352     break;
353   case Stmt::OMPTeamsDistributeDirectiveClass:
354     EmitOMPTeamsDistributeDirective(cast<OMPTeamsDistributeDirective>(*S));
355     break;
356   case Stmt::OMPTeamsDistributeSimdDirectiveClass:
357     EmitOMPTeamsDistributeSimdDirective(
358         cast<OMPTeamsDistributeSimdDirective>(*S));
359     break;
360   case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
361     EmitOMPTeamsDistributeParallelForSimdDirective(
362         cast<OMPTeamsDistributeParallelForSimdDirective>(*S));
363     break;
364   case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
365     EmitOMPTeamsDistributeParallelForDirective(
366         cast<OMPTeamsDistributeParallelForDirective>(*S));
367     break;
368   case Stmt::OMPTargetTeamsDirectiveClass:
369     EmitOMPTargetTeamsDirective(cast<OMPTargetTeamsDirective>(*S));
370     break;
371   case Stmt::OMPTargetTeamsDistributeDirectiveClass:
372     EmitOMPTargetTeamsDistributeDirective(
373         cast<OMPTargetTeamsDistributeDirective>(*S));
374     break;
375   case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
376     EmitOMPTargetTeamsDistributeParallelForDirective(
377         cast<OMPTargetTeamsDistributeParallelForDirective>(*S));
378     break;
379   case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
380     EmitOMPTargetTeamsDistributeParallelForSimdDirective(
381         cast<OMPTargetTeamsDistributeParallelForSimdDirective>(*S));
382     break;
383   case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
384     EmitOMPTargetTeamsDistributeSimdDirective(
385         cast<OMPTargetTeamsDistributeSimdDirective>(*S));
386     break;
387   case Stmt::OMPInteropDirectiveClass:
388     EmitOMPInteropDirective(cast<OMPInteropDirective>(*S));
389     break;
390   case Stmt::OMPDispatchDirectiveClass:
391     llvm_unreachable("Dispatch directive not supported yet.");
392     break;
393   case Stmt::OMPMaskedDirectiveClass:
394     EmitOMPMaskedDirective(cast<OMPMaskedDirective>(*S));
395     break;
396   case Stmt::OMPGenericLoopDirectiveClass:
397     EmitOMPGenericLoopDirective(cast<OMPGenericLoopDirective>(*S));
398     break;
399   case Stmt::OMPTeamsGenericLoopDirectiveClass:
400     llvm_unreachable("teams loop directive not supported yet.");
401     break;
402   case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
403     llvm_unreachable("target teams loop directive not supported yet.");
404     break;
405   case Stmt::OMPParallelGenericLoopDirectiveClass:
406     llvm_unreachable("parallel loop directive not supported yet.");
407     break;
408   case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
409     llvm_unreachable("target parallel loop directive not supported yet.");
410     break;
411   }
412 }
413 
414 bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
415                                      ArrayRef<const Attr *> Attrs) {
416   switch (S->getStmtClass()) {
417   default:
418     return false;
419   case Stmt::NullStmtClass:
420     break;
421   case Stmt::CompoundStmtClass:
422     EmitCompoundStmt(cast<CompoundStmt>(*S));
423     break;
424   case Stmt::DeclStmtClass:
425     EmitDeclStmt(cast<DeclStmt>(*S));
426     break;
427   case Stmt::LabelStmtClass:
428     EmitLabelStmt(cast<LabelStmt>(*S));
429     break;
430   case Stmt::AttributedStmtClass:
431     EmitAttributedStmt(cast<AttributedStmt>(*S));
432     break;
433   case Stmt::GotoStmtClass:
434     EmitGotoStmt(cast<GotoStmt>(*S));
435     break;
436   case Stmt::BreakStmtClass:
437     EmitBreakStmt(cast<BreakStmt>(*S));
438     break;
439   case Stmt::ContinueStmtClass:
440     EmitContinueStmt(cast<ContinueStmt>(*S));
441     break;
442   case Stmt::DefaultStmtClass:
443     EmitDefaultStmt(cast<DefaultStmt>(*S), Attrs);
444     break;
445   case Stmt::CaseStmtClass:
446     EmitCaseStmt(cast<CaseStmt>(*S), Attrs);
447     break;
448   case Stmt::SEHLeaveStmtClass:
449     EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S));
450     break;
451   }
452   return true;
453 }
454 
455 /// EmitCompoundStmt - Emit a compound statement {..} node.  If GetLast is true,
456 /// this captures the expression result of the last sub-statement and returns it
457 /// (for use by the statement expression extension).
458 Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
459                                           AggValueSlot AggSlot) {
460   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
461                              "LLVM IR generation of compound statement ('{}')");
462 
463   // Keep track of the current cleanup stack depth, including debug scopes.
464   LexicalScope Scope(*this, S.getSourceRange());
465 
466   return EmitCompoundStmtWithoutScope(S, GetLast, AggSlot);
467 }
468 
469 Address
470 CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
471                                               bool GetLast,
472                                               AggValueSlot AggSlot) {
473 
474   const Stmt *ExprResult = S.getStmtExprResult();
475   assert((!GetLast || (GetLast && ExprResult)) &&
476          "If GetLast is true then the CompoundStmt must have a StmtExprResult");
477 
478   Address RetAlloca = Address::invalid();
479 
480   for (auto *CurStmt : S.body()) {
481     if (GetLast && ExprResult == CurStmt) {
482       // We have to special case labels here.  They are statements, but when put
483       // at the end of a statement expression, they yield the value of their
484       // subexpression.  Handle this by walking through all labels we encounter,
485       // emitting them before we evaluate the subexpr.
486       // Similar issues arise for attributed statements.
487       while (!isa<Expr>(ExprResult)) {
488         if (const auto *LS = dyn_cast<LabelStmt>(ExprResult)) {
489           EmitLabel(LS->getDecl());
490           ExprResult = LS->getSubStmt();
491         } else if (const auto *AS = dyn_cast<AttributedStmt>(ExprResult)) {
492           // FIXME: Update this if we ever have attributes that affect the
493           // semantics of an expression.
494           ExprResult = AS->getSubStmt();
495         } else {
496           llvm_unreachable("unknown value statement");
497         }
498       }
499 
500       EnsureInsertPoint();
501 
502       const Expr *E = cast<Expr>(ExprResult);
503       QualType ExprTy = E->getType();
504       if (hasAggregateEvaluationKind(ExprTy)) {
505         EmitAggExpr(E, AggSlot);
506       } else {
507         // We can't return an RValue here because there might be cleanups at
508         // the end of the StmtExpr.  Because of that, we have to emit the result
509         // here into a temporary alloca.
510         RetAlloca = CreateMemTemp(ExprTy);
511         EmitAnyExprToMem(E, RetAlloca, Qualifiers(),
512                          /*IsInit*/ false);
513       }
514     } else {
515       EmitStmt(CurStmt);
516     }
517   }
518 
519   return RetAlloca;
520 }
521 
522 void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
523   llvm::BranchInst *BI = dyn_cast<llvm::BranchInst>(BB->getTerminator());
524 
525   // If there is a cleanup stack, then we it isn't worth trying to
526   // simplify this block (we would need to remove it from the scope map
527   // and cleanup entry).
528   if (!EHStack.empty())
529     return;
530 
531   // Can only simplify direct branches.
532   if (!BI || !BI->isUnconditional())
533     return;
534 
535   // Can only simplify empty blocks.
536   if (BI->getIterator() != BB->begin())
537     return;
538 
539   BB->replaceAllUsesWith(BI->getSuccessor(0));
540   BI->eraseFromParent();
541   BB->eraseFromParent();
542 }
543 
544 void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
545   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
546 
547   // Fall out of the current block (if necessary).
548   EmitBranch(BB);
549 
550   if (IsFinished && BB->use_empty()) {
551     delete BB;
552     return;
553   }
554 
555   // Place the block after the current block, if possible, or else at
556   // the end of the function.
557   if (CurBB && CurBB->getParent())
558     CurFn->getBasicBlockList().insertAfter(CurBB->getIterator(), BB);
559   else
560     CurFn->getBasicBlockList().push_back(BB);
561   Builder.SetInsertPoint(BB);
562 }
563 
564 void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
565   // Emit a branch from the current block to the target one if this
566   // was a real block.  If this was just a fall-through block after a
567   // terminator, don't emit it.
568   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
569 
570   if (!CurBB || CurBB->getTerminator()) {
571     // If there is no insert point or the previous block is already
572     // terminated, don't touch it.
573   } else {
574     // Otherwise, create a fall-through branch.
575     Builder.CreateBr(Target);
576   }
577 
578   Builder.ClearInsertionPoint();
579 }
580 
581 void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
582   bool inserted = false;
583   for (llvm::User *u : block->users()) {
584     if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(u)) {
585       CurFn->getBasicBlockList().insertAfter(insn->getParent()->getIterator(),
586                                              block);
587       inserted = true;
588       break;
589     }
590   }
591 
592   if (!inserted)
593     CurFn->getBasicBlockList().push_back(block);
594 
595   Builder.SetInsertPoint(block);
596 }
597 
598 CodeGenFunction::JumpDest
599 CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
600   JumpDest &Dest = LabelMap[D];
601   if (Dest.isValid()) return Dest;
602 
603   // Create, but don't insert, the new block.
604   Dest = JumpDest(createBasicBlock(D->getName()),
605                   EHScopeStack::stable_iterator::invalid(),
606                   NextCleanupDestIndex++);
607   return Dest;
608 }
609 
610 void CodeGenFunction::EmitLabel(const LabelDecl *D) {
611   // Add this label to the current lexical scope if we're within any
612   // normal cleanups.  Jumps "in" to this label --- when permitted by
613   // the language --- may need to be routed around such cleanups.
614   if (EHStack.hasNormalCleanups() && CurLexicalScope)
615     CurLexicalScope->addLabel(D);
616 
617   JumpDest &Dest = LabelMap[D];
618 
619   // If we didn't need a forward reference to this label, just go
620   // ahead and create a destination at the current scope.
621   if (!Dest.isValid()) {
622     Dest = getJumpDestInCurrentScope(D->getName());
623 
624   // Otherwise, we need to give this label a target depth and remove
625   // it from the branch-fixups list.
626   } else {
627     assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
628     Dest.setScopeDepth(EHStack.stable_begin());
629     ResolveBranchFixups(Dest.getBlock());
630   }
631 
632   EmitBlock(Dest.getBlock());
633 
634   // Emit debug info for labels.
635   if (CGDebugInfo *DI = getDebugInfo()) {
636     if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
637       DI->setLocation(D->getLocation());
638       DI->EmitLabel(D, Builder);
639     }
640   }
641 
642   incrementProfileCounter(D->getStmt());
643 }
644 
645 /// Change the cleanup scope of the labels in this lexical scope to
646 /// match the scope of the enclosing context.
647 void CodeGenFunction::LexicalScope::rescopeLabels() {
648   assert(!Labels.empty());
649   EHScopeStack::stable_iterator innermostScope
650     = CGF.EHStack.getInnermostNormalCleanup();
651 
652   // Change the scope depth of all the labels.
653   for (SmallVectorImpl<const LabelDecl*>::const_iterator
654          i = Labels.begin(), e = Labels.end(); i != e; ++i) {
655     assert(CGF.LabelMap.count(*i));
656     JumpDest &dest = CGF.LabelMap.find(*i)->second;
657     assert(dest.getScopeDepth().isValid());
658     assert(innermostScope.encloses(dest.getScopeDepth()));
659     dest.setScopeDepth(innermostScope);
660   }
661 
662   // Reparent the labels if the new scope also has cleanups.
663   if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
664     ParentScope->Labels.append(Labels.begin(), Labels.end());
665   }
666 }
667 
668 
669 void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
670   EmitLabel(S.getDecl());
671 
672   // IsEHa - emit eha.scope.begin if it's a side entry of a scope
673   if (getLangOpts().EHAsynch && S.isSideEntry())
674     EmitSehCppScopeBegin();
675 
676   EmitStmt(S.getSubStmt());
677 }
678 
679 void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
680   bool nomerge = false;
681   bool noinline = false;
682   bool alwaysinline = false;
683   const CallExpr *musttail = nullptr;
684 
685   for (const auto *A : S.getAttrs()) {
686     switch (A->getKind()) {
687     default:
688       break;
689     case attr::NoMerge:
690       nomerge = true;
691       break;
692     case attr::NoInline:
693       noinline = true;
694       break;
695     case attr::AlwaysInline:
696       alwaysinline = true;
697       break;
698     case attr::MustTail:
699       const Stmt *Sub = S.getSubStmt();
700       const ReturnStmt *R = cast<ReturnStmt>(Sub);
701       musttail = cast<CallExpr>(R->getRetValue()->IgnoreParens());
702       break;
703     }
704   }
705   SaveAndRestore<bool> save_nomerge(InNoMergeAttributedStmt, nomerge);
706   SaveAndRestore<bool> save_noinline(InNoInlineAttributedStmt, noinline);
707   SaveAndRestore<bool> save_alwaysinline(InAlwaysInlineAttributedStmt,
708                                          alwaysinline);
709   SaveAndRestore<const CallExpr *> save_musttail(MustTailCall, musttail);
710   EmitStmt(S.getSubStmt(), S.getAttrs());
711 }
712 
713 void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
714   // If this code is reachable then emit a stop point (if generating
715   // debug info). We have to do this ourselves because we are on the
716   // "simple" statement path.
717   if (HaveInsertPoint())
718     EmitStopPoint(&S);
719 
720   EmitBranchThroughCleanup(getJumpDestForLabel(S.getLabel()));
721 }
722 
723 
724 void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
725   if (const LabelDecl *Target = S.getConstantTarget()) {
726     EmitBranchThroughCleanup(getJumpDestForLabel(Target));
727     return;
728   }
729 
730   // Ensure that we have an i8* for our PHI node.
731   llvm::Value *V = Builder.CreateBitCast(EmitScalarExpr(S.getTarget()),
732                                          Int8PtrTy, "addr");
733   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
734 
735   // Get the basic block for the indirect goto.
736   llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
737 
738   // The first instruction in the block has to be the PHI for the switch dest,
739   // add an entry for this branch.
740   cast<llvm::PHINode>(IndGotoBB->begin())->addIncoming(V, CurBB);
741 
742   EmitBranch(IndGotoBB);
743 }
744 
745 void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
746   // The else branch of a consteval if statement is always the only branch that
747   // can be runtime evaluated.
748   if (S.isConsteval()) {
749     const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : S.getElse();
750     if (Executed) {
751       RunCleanupsScope ExecutedScope(*this);
752       EmitStmt(Executed);
753     }
754     return;
755   }
756 
757   // C99 6.8.4.1: The first substatement is executed if the expression compares
758   // unequal to 0.  The condition must be a scalar type.
759   LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
760 
761   if (S.getInit())
762     EmitStmt(S.getInit());
763 
764   if (S.getConditionVariable())
765     EmitDecl(*S.getConditionVariable());
766 
767   // If the condition constant folds and can be elided, try to avoid emitting
768   // the condition and the dead arm of the if/else.
769   bool CondConstant;
770   if (ConstantFoldsToSimpleInteger(S.getCond(), CondConstant,
771                                    S.isConstexpr())) {
772     // Figure out which block (then or else) is executed.
773     const Stmt *Executed = S.getThen();
774     const Stmt *Skipped  = S.getElse();
775     if (!CondConstant)  // Condition false?
776       std::swap(Executed, Skipped);
777 
778     // If the skipped block has no labels in it, just emit the executed block.
779     // This avoids emitting dead code and simplifies the CFG substantially.
780     if (S.isConstexpr() || !ContainsLabel(Skipped)) {
781       if (CondConstant)
782         incrementProfileCounter(&S);
783       if (Executed) {
784         RunCleanupsScope ExecutedScope(*this);
785         EmitStmt(Executed);
786       }
787       return;
788     }
789   }
790 
791   // Otherwise, the condition did not fold, or we couldn't elide it.  Just emit
792   // the conditional branch.
793   llvm::BasicBlock *ThenBlock = createBasicBlock("if.then");
794   llvm::BasicBlock *ContBlock = createBasicBlock("if.end");
795   llvm::BasicBlock *ElseBlock = ContBlock;
796   if (S.getElse())
797     ElseBlock = createBasicBlock("if.else");
798 
799   // Prefer the PGO based weights over the likelihood attribute.
800   // When the build isn't optimized the metadata isn't used, so don't generate
801   // it.
802   Stmt::Likelihood LH = Stmt::LH_None;
803   uint64_t Count = getProfileCount(S.getThen());
804   if (!Count && CGM.getCodeGenOpts().OptimizationLevel)
805     LH = Stmt::getLikelihood(S.getThen(), S.getElse());
806   EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, Count, LH);
807 
808   // Emit the 'then' code.
809   EmitBlock(ThenBlock);
810   incrementProfileCounter(&S);
811   {
812     RunCleanupsScope ThenScope(*this);
813     EmitStmt(S.getThen());
814   }
815   EmitBranch(ContBlock);
816 
817   // Emit the 'else' code if present.
818   if (const Stmt *Else = S.getElse()) {
819     {
820       // There is no need to emit line number for an unconditional branch.
821       auto NL = ApplyDebugLocation::CreateEmpty(*this);
822       EmitBlock(ElseBlock);
823     }
824     {
825       RunCleanupsScope ElseScope(*this);
826       EmitStmt(Else);
827     }
828     {
829       // There is no need to emit line number for an unconditional branch.
830       auto NL = ApplyDebugLocation::CreateEmpty(*this);
831       EmitBranch(ContBlock);
832     }
833   }
834 
835   // Emit the continuation block for code after the if.
836   EmitBlock(ContBlock, true);
837 }
838 
839 void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
840                                     ArrayRef<const Attr *> WhileAttrs) {
841   // Emit the header for the loop, which will also become
842   // the continue target.
843   JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
844   EmitBlock(LoopHeader.getBlock());
845 
846   // Create an exit block for when the condition fails, which will
847   // also become the break target.
848   JumpDest LoopExit = getJumpDestInCurrentScope("while.end");
849 
850   // Store the blocks to use for break and continue.
851   BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader));
852 
853   // C++ [stmt.while]p2:
854   //   When the condition of a while statement is a declaration, the
855   //   scope of the variable that is declared extends from its point
856   //   of declaration (3.3.2) to the end of the while statement.
857   //   [...]
858   //   The object created in a condition is destroyed and created
859   //   with each iteration of the loop.
860   RunCleanupsScope ConditionScope(*this);
861 
862   if (S.getConditionVariable())
863     EmitDecl(*S.getConditionVariable());
864 
865   // Evaluate the conditional in the while header.  C99 6.8.5.1: The
866   // evaluation of the controlling expression takes place before each
867   // execution of the loop body.
868   llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
869 
870   // while(1) is common, avoid extra exit blocks.  Be sure
871   // to correctly handle break/continue though.
872   llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal);
873   bool CondIsConstInt = C != nullptr;
874   bool EmitBoolCondBranch = !CondIsConstInt || !C->isOne();
875   const SourceRange &R = S.getSourceRange();
876   LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), CGM.getCodeGenOpts(),
877                  WhileAttrs, SourceLocToDebugLoc(R.getBegin()),
878                  SourceLocToDebugLoc(R.getEnd()),
879                  checkIfLoopMustProgress(CondIsConstInt));
880 
881   // As long as the condition is true, go to the loop body.
882   llvm::BasicBlock *LoopBody = createBasicBlock("while.body");
883   if (EmitBoolCondBranch) {
884     llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
885     if (ConditionScope.requiresCleanups())
886       ExitBlock = createBasicBlock("while.exit");
887     llvm::MDNode *Weights =
888         createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
889     if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
890       BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
891           BoolCondVal, Stmt::getLikelihood(S.getBody()));
892     Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights);
893 
894     if (ExitBlock != LoopExit.getBlock()) {
895       EmitBlock(ExitBlock);
896       EmitBranchThroughCleanup(LoopExit);
897     }
898   } else if (const Attr *A = Stmt::getLikelihoodAttr(S.getBody())) {
899     CGM.getDiags().Report(A->getLocation(),
900                           diag::warn_attribute_has_no_effect_on_infinite_loop)
901         << A << A->getRange();
902     CGM.getDiags().Report(
903         S.getWhileLoc(),
904         diag::note_attribute_has_no_effect_on_infinite_loop_here)
905         << SourceRange(S.getWhileLoc(), S.getRParenLoc());
906   }
907 
908   // Emit the loop body.  We have to emit this in a cleanup scope
909   // because it might be a singleton DeclStmt.
910   {
911     RunCleanupsScope BodyScope(*this);
912     EmitBlock(LoopBody);
913     incrementProfileCounter(&S);
914     EmitStmt(S.getBody());
915   }
916 
917   BreakContinueStack.pop_back();
918 
919   // Immediately force cleanup.
920   ConditionScope.ForceCleanup();
921 
922   EmitStopPoint(&S);
923   // Branch to the loop header again.
924   EmitBranch(LoopHeader.getBlock());
925 
926   LoopStack.pop();
927 
928   // Emit the exit block.
929   EmitBlock(LoopExit.getBlock(), true);
930 
931   // The LoopHeader typically is just a branch if we skipped emitting
932   // a branch, try to erase it.
933   if (!EmitBoolCondBranch)
934     SimplifyForwardingBlocks(LoopHeader.getBlock());
935 }
936 
937 void CodeGenFunction::EmitDoStmt(const DoStmt &S,
938                                  ArrayRef<const Attr *> DoAttrs) {
939   JumpDest LoopExit = getJumpDestInCurrentScope("do.end");
940   JumpDest LoopCond = getJumpDestInCurrentScope("do.cond");
941 
942   uint64_t ParentCount = getCurrentProfileCount();
943 
944   // Store the blocks to use for break and continue.
945   BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond));
946 
947   // Emit the body of the loop.
948   llvm::BasicBlock *LoopBody = createBasicBlock("do.body");
949 
950   EmitBlockWithFallThrough(LoopBody, &S);
951   {
952     RunCleanupsScope BodyScope(*this);
953     EmitStmt(S.getBody());
954   }
955 
956   EmitBlock(LoopCond.getBlock());
957 
958   // C99 6.8.5.2: "The evaluation of the controlling expression takes place
959   // after each execution of the loop body."
960 
961   // Evaluate the conditional in the while header.
962   // C99 6.8.5p2/p4: The first substatement is executed if the expression
963   // compares unequal to 0.  The condition must be a scalar type.
964   llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
965 
966   BreakContinueStack.pop_back();
967 
968   // "do {} while (0)" is common in macros, avoid extra blocks.  Be sure
969   // to correctly handle break/continue though.
970   llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal);
971   bool CondIsConstInt = C;
972   bool EmitBoolCondBranch = !C || !C->isZero();
973 
974   const SourceRange &R = S.getSourceRange();
975   LoopStack.push(LoopBody, CGM.getContext(), CGM.getCodeGenOpts(), DoAttrs,
976                  SourceLocToDebugLoc(R.getBegin()),
977                  SourceLocToDebugLoc(R.getEnd()),
978                  checkIfLoopMustProgress(CondIsConstInt));
979 
980   // As long as the condition is true, iterate the loop.
981   if (EmitBoolCondBranch) {
982     uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount;
983     Builder.CreateCondBr(
984         BoolCondVal, LoopBody, LoopExit.getBlock(),
985         createProfileWeightsForLoop(S.getCond(), BackedgeCount));
986   }
987 
988   LoopStack.pop();
989 
990   // Emit the exit block.
991   EmitBlock(LoopExit.getBlock());
992 
993   // The DoCond block typically is just a branch if we skipped
994   // emitting a branch, try to erase it.
995   if (!EmitBoolCondBranch)
996     SimplifyForwardingBlocks(LoopCond.getBlock());
997 }
998 
999 void CodeGenFunction::EmitForStmt(const ForStmt &S,
1000                                   ArrayRef<const Attr *> ForAttrs) {
1001   JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
1002 
1003   LexicalScope ForScope(*this, S.getSourceRange());
1004 
1005   // Evaluate the first part before the loop.
1006   if (S.getInit())
1007     EmitStmt(S.getInit());
1008 
1009   // Start the loop with a block that tests the condition.
1010   // If there's an increment, the continue scope will be overwritten
1011   // later.
1012   JumpDest CondDest = getJumpDestInCurrentScope("for.cond");
1013   llvm::BasicBlock *CondBlock = CondDest.getBlock();
1014   EmitBlock(CondBlock);
1015 
1016   Expr::EvalResult Result;
1017   bool CondIsConstInt =
1018       !S.getCond() || S.getCond()->EvaluateAsInt(Result, getContext());
1019 
1020   const SourceRange &R = S.getSourceRange();
1021   LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,
1022                  SourceLocToDebugLoc(R.getBegin()),
1023                  SourceLocToDebugLoc(R.getEnd()),
1024                  checkIfLoopMustProgress(CondIsConstInt));
1025 
1026   // Create a cleanup scope for the condition variable cleanups.
1027   LexicalScope ConditionScope(*this, S.getSourceRange());
1028 
1029   // If the for loop doesn't have an increment we can just use the condition as
1030   // the continue block. Otherwise, if there is no condition variable, we can
1031   // form the continue block now. If there is a condition variable, we can't
1032   // form the continue block until after we've emitted the condition, because
1033   // the condition is in scope in the increment, but Sema's jump diagnostics
1034   // ensure that there are no continues from the condition variable that jump
1035   // to the loop increment.
1036   JumpDest Continue;
1037   if (!S.getInc())
1038     Continue = CondDest;
1039   else if (!S.getConditionVariable())
1040     Continue = getJumpDestInCurrentScope("for.inc");
1041   BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1042 
1043   if (S.getCond()) {
1044     // If the for statement has a condition scope, emit the local variable
1045     // declaration.
1046     if (S.getConditionVariable()) {
1047       EmitDecl(*S.getConditionVariable());
1048 
1049       // We have entered the condition variable's scope, so we're now able to
1050       // jump to the continue block.
1051       Continue = S.getInc() ? getJumpDestInCurrentScope("for.inc") : CondDest;
1052       BreakContinueStack.back().ContinueBlock = Continue;
1053     }
1054 
1055     llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1056     // If there are any cleanups between here and the loop-exit scope,
1057     // create a block to stage a loop exit along.
1058     if (ForScope.requiresCleanups())
1059       ExitBlock = createBasicBlock("for.cond.cleanup");
1060 
1061     // As long as the condition is true, iterate the loop.
1062     llvm::BasicBlock *ForBody = createBasicBlock("for.body");
1063 
1064     // C99 6.8.5p2/p4: The first substatement is executed if the expression
1065     // compares unequal to 0.  The condition must be a scalar type.
1066     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
1067     llvm::MDNode *Weights =
1068         createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
1069     if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1070       BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1071           BoolCondVal, Stmt::getLikelihood(S.getBody()));
1072 
1073     Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
1074 
1075     if (ExitBlock != LoopExit.getBlock()) {
1076       EmitBlock(ExitBlock);
1077       EmitBranchThroughCleanup(LoopExit);
1078     }
1079 
1080     EmitBlock(ForBody);
1081   } else {
1082     // Treat it as a non-zero constant.  Don't even create a new block for the
1083     // body, just fall into it.
1084   }
1085   incrementProfileCounter(&S);
1086 
1087   {
1088     // Create a separate cleanup scope for the body, in case it is not
1089     // a compound statement.
1090     RunCleanupsScope BodyScope(*this);
1091     EmitStmt(S.getBody());
1092   }
1093 
1094   // If there is an increment, emit it next.
1095   if (S.getInc()) {
1096     EmitBlock(Continue.getBlock());
1097     EmitStmt(S.getInc());
1098   }
1099 
1100   BreakContinueStack.pop_back();
1101 
1102   ConditionScope.ForceCleanup();
1103 
1104   EmitStopPoint(&S);
1105   EmitBranch(CondBlock);
1106 
1107   ForScope.ForceCleanup();
1108 
1109   LoopStack.pop();
1110 
1111   // Emit the fall-through block.
1112   EmitBlock(LoopExit.getBlock(), true);
1113 }
1114 
1115 void
1116 CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1117                                      ArrayRef<const Attr *> ForAttrs) {
1118   JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
1119 
1120   LexicalScope ForScope(*this, S.getSourceRange());
1121 
1122   // Evaluate the first pieces before the loop.
1123   if (S.getInit())
1124     EmitStmt(S.getInit());
1125   EmitStmt(S.getRangeStmt());
1126   EmitStmt(S.getBeginStmt());
1127   EmitStmt(S.getEndStmt());
1128 
1129   // Start the loop with a block that tests the condition.
1130   // If there's an increment, the continue scope will be overwritten
1131   // later.
1132   llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
1133   EmitBlock(CondBlock);
1134 
1135   const SourceRange &R = S.getSourceRange();
1136   LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,
1137                  SourceLocToDebugLoc(R.getBegin()),
1138                  SourceLocToDebugLoc(R.getEnd()));
1139 
1140   // If there are any cleanups between here and the loop-exit scope,
1141   // create a block to stage a loop exit along.
1142   llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1143   if (ForScope.requiresCleanups())
1144     ExitBlock = createBasicBlock("for.cond.cleanup");
1145 
1146   // The loop body, consisting of the specified body and the loop variable.
1147   llvm::BasicBlock *ForBody = createBasicBlock("for.body");
1148 
1149   // The body is executed if the expression, contextually converted
1150   // to bool, is true.
1151   llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
1152   llvm::MDNode *Weights =
1153       createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
1154   if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1155     BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1156         BoolCondVal, Stmt::getLikelihood(S.getBody()));
1157   Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
1158 
1159   if (ExitBlock != LoopExit.getBlock()) {
1160     EmitBlock(ExitBlock);
1161     EmitBranchThroughCleanup(LoopExit);
1162   }
1163 
1164   EmitBlock(ForBody);
1165   incrementProfileCounter(&S);
1166 
1167   // Create a block for the increment. In case of a 'continue', we jump there.
1168   JumpDest Continue = getJumpDestInCurrentScope("for.inc");
1169 
1170   // Store the blocks to use for break and continue.
1171   BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1172 
1173   {
1174     // Create a separate cleanup scope for the loop variable and body.
1175     LexicalScope BodyScope(*this, S.getSourceRange());
1176     EmitStmt(S.getLoopVarStmt());
1177     EmitStmt(S.getBody());
1178   }
1179 
1180   EmitStopPoint(&S);
1181   // If there is an increment, emit it next.
1182   EmitBlock(Continue.getBlock());
1183   EmitStmt(S.getInc());
1184 
1185   BreakContinueStack.pop_back();
1186 
1187   EmitBranch(CondBlock);
1188 
1189   ForScope.ForceCleanup();
1190 
1191   LoopStack.pop();
1192 
1193   // Emit the fall-through block.
1194   EmitBlock(LoopExit.getBlock(), true);
1195 }
1196 
1197 void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1198   if (RV.isScalar()) {
1199     Builder.CreateStore(RV.getScalarVal(), ReturnValue);
1200   } else if (RV.isAggregate()) {
1201     LValue Dest = MakeAddrLValue(ReturnValue, Ty);
1202     LValue Src = MakeAddrLValue(RV.getAggregateAddress(), Ty);
1203     EmitAggregateCopy(Dest, Src, Ty, getOverlapForReturnValue());
1204   } else {
1205     EmitStoreOfComplex(RV.getComplexVal(), MakeAddrLValue(ReturnValue, Ty),
1206                        /*init*/ true);
1207   }
1208   EmitBranchThroughCleanup(ReturnBlock);
1209 }
1210 
1211 namespace {
1212 // RAII struct used to save and restore a return statment's result expression.
1213 struct SaveRetExprRAII {
1214   SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1215       : OldRetExpr(CGF.RetExpr), CGF(CGF) {
1216     CGF.RetExpr = RetExpr;
1217   }
1218   ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1219   const Expr *OldRetExpr;
1220   CodeGenFunction &CGF;
1221 };
1222 } // namespace
1223 
1224 /// If we have 'return f(...);', where both caller and callee are SwiftAsync,
1225 /// codegen it as 'tail call ...; ret void;'.
1226 static void makeTailCallIfSwiftAsync(const CallExpr *CE, CGBuilderTy &Builder,
1227                                      const CGFunctionInfo *CurFnInfo) {
1228   auto calleeQualType = CE->getCallee()->getType();
1229   const FunctionType *calleeType = nullptr;
1230   if (calleeQualType->isFunctionPointerType() ||
1231       calleeQualType->isFunctionReferenceType() ||
1232       calleeQualType->isBlockPointerType() ||
1233       calleeQualType->isMemberFunctionPointerType()) {
1234     calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();
1235   } else if (auto *ty = dyn_cast<FunctionType>(calleeQualType)) {
1236     calleeType = ty;
1237   } else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(CE)) {
1238     if (auto methodDecl = CMCE->getMethodDecl()) {
1239       // getMethodDecl() doesn't handle member pointers at the moment.
1240       calleeType = methodDecl->getType()->castAs<FunctionType>();
1241     } else {
1242       return;
1243     }
1244   } else {
1245     return;
1246   }
1247   if (calleeType->getCallConv() == CallingConv::CC_SwiftAsync &&
1248       (CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync)) {
1249     auto CI = cast<llvm::CallInst>(&Builder.GetInsertBlock()->back());
1250     CI->setTailCallKind(llvm::CallInst::TCK_MustTail);
1251     Builder.CreateRetVoid();
1252     Builder.ClearInsertionPoint();
1253   }
1254 }
1255 
1256 /// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1257 /// if the function returns void, or may be missing one if the function returns
1258 /// non-void.  Fun stuff :).
1259 void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1260   if (requiresReturnValueCheck()) {
1261     llvm::Constant *SLoc = EmitCheckSourceLocation(S.getBeginLoc());
1262     auto *SLocPtr =
1263         new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1264                                  llvm::GlobalVariable::PrivateLinkage, SLoc);
1265     SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1266     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(SLocPtr);
1267     assert(ReturnLocation.isValid() && "No valid return location");
1268     Builder.CreateStore(Builder.CreateBitCast(SLocPtr, Int8PtrTy),
1269                         ReturnLocation);
1270   }
1271 
1272   // Returning from an outlined SEH helper is UB, and we already warn on it.
1273   if (IsOutlinedSEHHelper) {
1274     Builder.CreateUnreachable();
1275     Builder.ClearInsertionPoint();
1276   }
1277 
1278   // Emit the result value, even if unused, to evaluate the side effects.
1279   const Expr *RV = S.getRetValue();
1280 
1281   // Record the result expression of the return statement. The recorded
1282   // expression is used to determine whether a block capture's lifetime should
1283   // end at the end of the full expression as opposed to the end of the scope
1284   // enclosing the block expression.
1285   //
1286   // This permits a small, easily-implemented exception to our over-conservative
1287   // rules about not jumping to statements following block literals with
1288   // non-trivial cleanups.
1289   SaveRetExprRAII SaveRetExpr(RV, *this);
1290 
1291   RunCleanupsScope cleanupScope(*this);
1292   if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(RV))
1293     RV = EWC->getSubExpr();
1294   // FIXME: Clean this up by using an LValue for ReturnTemp,
1295   // EmitStoreThroughLValue, and EmitAnyExpr.
1296   // Check if the NRVO candidate was not globalized in OpenMP mode.
1297   if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&
1298       S.getNRVOCandidate()->isNRVOVariable() &&
1299       (!getLangOpts().OpenMP ||
1300        !CGM.getOpenMPRuntime()
1301             .getAddressOfLocalVariable(*this, S.getNRVOCandidate())
1302             .isValid())) {
1303     // Apply the named return value optimization for this return statement,
1304     // which means doing nothing: the appropriate result has already been
1305     // constructed into the NRVO variable.
1306 
1307     // If there is an NRVO flag for this variable, set it to 1 into indicate
1308     // that the cleanup code should not destroy the variable.
1309     if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1310       Builder.CreateFlagStore(Builder.getTrue(), NRVOFlag);
1311   } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1312     // Make sure not to return anything, but evaluate the expression
1313     // for side effects.
1314     if (RV) {
1315       EmitAnyExpr(RV);
1316       if (auto *CE = dyn_cast<CallExpr>(RV))
1317         makeTailCallIfSwiftAsync(CE, Builder, CurFnInfo);
1318     }
1319   } else if (!RV) {
1320     // Do nothing (return value is left uninitialized)
1321   } else if (FnRetTy->isReferenceType()) {
1322     // If this function returns a reference, take the address of the expression
1323     // rather than the value.
1324     RValue Result = EmitReferenceBindingToExpr(RV);
1325     Builder.CreateStore(Result.getScalarVal(), ReturnValue);
1326   } else {
1327     switch (getEvaluationKind(RV->getType())) {
1328     case TEK_Scalar:
1329       Builder.CreateStore(EmitScalarExpr(RV), ReturnValue);
1330       break;
1331     case TEK_Complex:
1332       EmitComplexExprIntoLValue(RV, MakeAddrLValue(ReturnValue, RV->getType()),
1333                                 /*isInit*/ true);
1334       break;
1335     case TEK_Aggregate:
1336       EmitAggExpr(RV, AggValueSlot::forAddr(
1337                           ReturnValue, Qualifiers(),
1338                           AggValueSlot::IsDestructed,
1339                           AggValueSlot::DoesNotNeedGCBarriers,
1340                           AggValueSlot::IsNotAliased,
1341                           getOverlapForReturnValue()));
1342       break;
1343     }
1344   }
1345 
1346   ++NumReturnExprs;
1347   if (!RV || RV->isEvaluatable(getContext()))
1348     ++NumSimpleReturnExprs;
1349 
1350   cleanupScope.ForceCleanup();
1351   EmitBranchThroughCleanup(ReturnBlock);
1352 }
1353 
1354 void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1355   // As long as debug info is modeled with instructions, we have to ensure we
1356   // have a place to insert here and write the stop point here.
1357   if (HaveInsertPoint())
1358     EmitStopPoint(&S);
1359 
1360   for (const auto *I : S.decls())
1361     EmitDecl(*I);
1362 }
1363 
1364 void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1365   assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1366 
1367   // If this code is reachable then emit a stop point (if generating
1368   // debug info). We have to do this ourselves because we are on the
1369   // "simple" statement path.
1370   if (HaveInsertPoint())
1371     EmitStopPoint(&S);
1372 
1373   EmitBranchThroughCleanup(BreakContinueStack.back().BreakBlock);
1374 }
1375 
1376 void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1377   assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1378 
1379   // If this code is reachable then emit a stop point (if generating
1380   // debug info). We have to do this ourselves because we are on the
1381   // "simple" statement path.
1382   if (HaveInsertPoint())
1383     EmitStopPoint(&S);
1384 
1385   EmitBranchThroughCleanup(BreakContinueStack.back().ContinueBlock);
1386 }
1387 
1388 /// EmitCaseStmtRange - If case statement range is not too big then
1389 /// add multiple cases to switch instruction, one for each value within
1390 /// the range. If range is too big then emit "if" condition check.
1391 void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1392                                         ArrayRef<const Attr *> Attrs) {
1393   assert(S.getRHS() && "Expected RHS value in CaseStmt");
1394 
1395   llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext());
1396   llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext());
1397 
1398   // Emit the code for this case. We do this first to make sure it is
1399   // properly chained from our predecessor before generating the
1400   // switch machinery to enter this block.
1401   llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
1402   EmitBlockWithFallThrough(CaseDest, &S);
1403   EmitStmt(S.getSubStmt());
1404 
1405   // If range is empty, do nothing.
1406   if (LHS.isSigned() ? RHS.slt(LHS) : RHS.ult(LHS))
1407     return;
1408 
1409   Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1410   llvm::APInt Range = RHS - LHS;
1411   // FIXME: parameters such as this should not be hardcoded.
1412   if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {
1413     // Range is small enough to add multiple switch instruction cases.
1414     uint64_t Total = getProfileCount(&S);
1415     unsigned NCases = Range.getZExtValue() + 1;
1416     // We only have one region counter for the entire set of cases here, so we
1417     // need to divide the weights evenly between the generated cases, ensuring
1418     // that the total weight is preserved. E.g., a weight of 5 over three cases
1419     // will be distributed as weights of 2, 2, and 1.
1420     uint64_t Weight = Total / NCases, Rem = Total % NCases;
1421     for (unsigned I = 0; I != NCases; ++I) {
1422       if (SwitchWeights)
1423         SwitchWeights->push_back(Weight + (Rem ? 1 : 0));
1424       else if (SwitchLikelihood)
1425         SwitchLikelihood->push_back(LH);
1426 
1427       if (Rem)
1428         Rem--;
1429       SwitchInsn->addCase(Builder.getInt(LHS), CaseDest);
1430       ++LHS;
1431     }
1432     return;
1433   }
1434 
1435   // The range is too big. Emit "if" condition into a new block,
1436   // making sure to save and restore the current insertion point.
1437   llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1438 
1439   // Push this test onto the chain of range checks (which terminates
1440   // in the default basic block). The switch's default will be changed
1441   // to the top of this chain after switch emission is complete.
1442   llvm::BasicBlock *FalseDest = CaseRangeBlock;
1443   CaseRangeBlock = createBasicBlock("sw.caserange");
1444 
1445   CurFn->getBasicBlockList().push_back(CaseRangeBlock);
1446   Builder.SetInsertPoint(CaseRangeBlock);
1447 
1448   // Emit range check.
1449   llvm::Value *Diff =
1450     Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS));
1451   llvm::Value *Cond =
1452     Builder.CreateICmpULE(Diff, Builder.getInt(Range), "inbounds");
1453 
1454   llvm::MDNode *Weights = nullptr;
1455   if (SwitchWeights) {
1456     uint64_t ThisCount = getProfileCount(&S);
1457     uint64_t DefaultCount = (*SwitchWeights)[0];
1458     Weights = createProfileWeights(ThisCount, DefaultCount);
1459 
1460     // Since we're chaining the switch default through each large case range, we
1461     // need to update the weight for the default, ie, the first case, to include
1462     // this case.
1463     (*SwitchWeights)[0] += ThisCount;
1464   } else if (SwitchLikelihood)
1465     Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
1466 
1467   Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
1468 
1469   // Restore the appropriate insertion point.
1470   if (RestoreBB)
1471     Builder.SetInsertPoint(RestoreBB);
1472   else
1473     Builder.ClearInsertionPoint();
1474 }
1475 
1476 void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1477                                    ArrayRef<const Attr *> Attrs) {
1478   // If there is no enclosing switch instance that we're aware of, then this
1479   // case statement and its block can be elided.  This situation only happens
1480   // when we've constant-folded the switch, are emitting the constant case,
1481   // and part of the constant case includes another case statement.  For
1482   // instance: switch (4) { case 4: do { case 5: } while (1); }
1483   if (!SwitchInsn) {
1484     EmitStmt(S.getSubStmt());
1485     return;
1486   }
1487 
1488   // Handle case ranges.
1489   if (S.getRHS()) {
1490     EmitCaseStmtRange(S, Attrs);
1491     return;
1492   }
1493 
1494   llvm::ConstantInt *CaseVal =
1495     Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext()));
1496   if (SwitchLikelihood)
1497     SwitchLikelihood->push_back(Stmt::getLikelihood(Attrs));
1498 
1499   // If the body of the case is just a 'break', try to not emit an empty block.
1500   // If we're profiling or we're not optimizing, leave the block in for better
1501   // debug and coverage analysis.
1502   if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1503       CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1504       isa<BreakStmt>(S.getSubStmt())) {
1505     JumpDest Block = BreakContinueStack.back().BreakBlock;
1506 
1507     // Only do this optimization if there are no cleanups that need emitting.
1508     if (isObviouslyBranchWithoutCleanups(Block)) {
1509       if (SwitchWeights)
1510         SwitchWeights->push_back(getProfileCount(&S));
1511       SwitchInsn->addCase(CaseVal, Block.getBlock());
1512 
1513       // If there was a fallthrough into this case, make sure to redirect it to
1514       // the end of the switch as well.
1515       if (Builder.GetInsertBlock()) {
1516         Builder.CreateBr(Block.getBlock());
1517         Builder.ClearInsertionPoint();
1518       }
1519       return;
1520     }
1521   }
1522 
1523   llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
1524   EmitBlockWithFallThrough(CaseDest, &S);
1525   if (SwitchWeights)
1526     SwitchWeights->push_back(getProfileCount(&S));
1527   SwitchInsn->addCase(CaseVal, CaseDest);
1528 
1529   // Recursively emitting the statement is acceptable, but is not wonderful for
1530   // code where we have many case statements nested together, i.e.:
1531   //  case 1:
1532   //    case 2:
1533   //      case 3: etc.
1534   // Handling this recursively will create a new block for each case statement
1535   // that falls through to the next case which is IR intensive.  It also causes
1536   // deep recursion which can run into stack depth limitations.  Handle
1537   // sequential non-range case statements specially.
1538   //
1539   // TODO When the next case has a likelihood attribute the code returns to the
1540   // recursive algorithm. Maybe improve this case if it becomes common practice
1541   // to use a lot of attributes.
1542   const CaseStmt *CurCase = &S;
1543   const CaseStmt *NextCase = dyn_cast<CaseStmt>(S.getSubStmt());
1544 
1545   // Otherwise, iteratively add consecutive cases to this switch stmt.
1546   while (NextCase && NextCase->getRHS() == nullptr) {
1547     CurCase = NextCase;
1548     llvm::ConstantInt *CaseVal =
1549       Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext()));
1550 
1551     if (SwitchWeights)
1552       SwitchWeights->push_back(getProfileCount(NextCase));
1553     if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1554       CaseDest = createBasicBlock("sw.bb");
1555       EmitBlockWithFallThrough(CaseDest, CurCase);
1556     }
1557     // Since this loop is only executed when the CaseStmt has no attributes
1558     // use a hard-coded value.
1559     if (SwitchLikelihood)
1560       SwitchLikelihood->push_back(Stmt::LH_None);
1561 
1562     SwitchInsn->addCase(CaseVal, CaseDest);
1563     NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());
1564   }
1565 
1566   // Generate a stop point for debug info if the case statement is
1567   // followed by a default statement. A fallthrough case before a
1568   // default case gets its own branch target.
1569   if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)
1570     EmitStopPoint(CurCase);
1571 
1572   // Normal default recursion for non-cases.
1573   EmitStmt(CurCase->getSubStmt());
1574 }
1575 
1576 void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1577                                       ArrayRef<const Attr *> Attrs) {
1578   // If there is no enclosing switch instance that we're aware of, then this
1579   // default statement can be elided. This situation only happens when we've
1580   // constant-folded the switch.
1581   if (!SwitchInsn) {
1582     EmitStmt(S.getSubStmt());
1583     return;
1584   }
1585 
1586   llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1587   assert(DefaultBlock->empty() &&
1588          "EmitDefaultStmt: Default block already defined?");
1589 
1590   if (SwitchLikelihood)
1591     SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
1592 
1593   EmitBlockWithFallThrough(DefaultBlock, &S);
1594 
1595   EmitStmt(S.getSubStmt());
1596 }
1597 
1598 /// CollectStatementsForCase - Given the body of a 'switch' statement and a
1599 /// constant value that is being switched on, see if we can dead code eliminate
1600 /// the body of the switch to a simple series of statements to emit.  Basically,
1601 /// on a switch (5) we want to find these statements:
1602 ///    case 5:
1603 ///      printf(...);    <--
1604 ///      ++i;            <--
1605 ///      break;
1606 ///
1607 /// and add them to the ResultStmts vector.  If it is unsafe to do this
1608 /// transformation (for example, one of the elided statements contains a label
1609 /// that might be jumped to), return CSFC_Failure.  If we handled it and 'S'
1610 /// should include statements after it (e.g. the printf() line is a substmt of
1611 /// the case) then return CSFC_FallThrough.  If we handled it and found a break
1612 /// statement, then return CSFC_Success.
1613 ///
1614 /// If Case is non-null, then we are looking for the specified case, checking
1615 /// that nothing we jump over contains labels.  If Case is null, then we found
1616 /// the case and are looking for the break.
1617 ///
1618 /// If the recursive walk actually finds our Case, then we set FoundCase to
1619 /// true.
1620 ///
1621 enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
1622 static CSFC_Result CollectStatementsForCase(const Stmt *S,
1623                                             const SwitchCase *Case,
1624                                             bool &FoundCase,
1625                               SmallVectorImpl<const Stmt*> &ResultStmts) {
1626   // If this is a null statement, just succeed.
1627   if (!S)
1628     return Case ? CSFC_Success : CSFC_FallThrough;
1629 
1630   // If this is the switchcase (case 4: or default) that we're looking for, then
1631   // we're in business.  Just add the substatement.
1632   if (const SwitchCase *SC = dyn_cast<SwitchCase>(S)) {
1633     if (S == Case) {
1634       FoundCase = true;
1635       return CollectStatementsForCase(SC->getSubStmt(), nullptr, FoundCase,
1636                                       ResultStmts);
1637     }
1638 
1639     // Otherwise, this is some other case or default statement, just ignore it.
1640     return CollectStatementsForCase(SC->getSubStmt(), Case, FoundCase,
1641                                     ResultStmts);
1642   }
1643 
1644   // If we are in the live part of the code and we found our break statement,
1645   // return a success!
1646   if (!Case && isa<BreakStmt>(S))
1647     return CSFC_Success;
1648 
1649   // If this is a switch statement, then it might contain the SwitchCase, the
1650   // break, or neither.
1651   if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(S)) {
1652     // Handle this as two cases: we might be looking for the SwitchCase (if so
1653     // the skipped statements must be skippable) or we might already have it.
1654     CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
1655     bool StartedInLiveCode = FoundCase;
1656     unsigned StartSize = ResultStmts.size();
1657 
1658     // If we've not found the case yet, scan through looking for it.
1659     if (Case) {
1660       // Keep track of whether we see a skipped declaration.  The code could be
1661       // using the declaration even if it is skipped, so we can't optimize out
1662       // the decl if the kept statements might refer to it.
1663       bool HadSkippedDecl = false;
1664 
1665       // If we're looking for the case, just see if we can skip each of the
1666       // substatements.
1667       for (; Case && I != E; ++I) {
1668         HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(*I);
1669 
1670         switch (CollectStatementsForCase(*I, Case, FoundCase, ResultStmts)) {
1671         case CSFC_Failure: return CSFC_Failure;
1672         case CSFC_Success:
1673           // A successful result means that either 1) that the statement doesn't
1674           // have the case and is skippable, or 2) does contain the case value
1675           // and also contains the break to exit the switch.  In the later case,
1676           // we just verify the rest of the statements are elidable.
1677           if (FoundCase) {
1678             // If we found the case and skipped declarations, we can't do the
1679             // optimization.
1680             if (HadSkippedDecl)
1681               return CSFC_Failure;
1682 
1683             for (++I; I != E; ++I)
1684               if (CodeGenFunction::ContainsLabel(*I, true))
1685                 return CSFC_Failure;
1686             return CSFC_Success;
1687           }
1688           break;
1689         case CSFC_FallThrough:
1690           // If we have a fallthrough condition, then we must have found the
1691           // case started to include statements.  Consider the rest of the
1692           // statements in the compound statement as candidates for inclusion.
1693           assert(FoundCase && "Didn't find case but returned fallthrough?");
1694           // We recursively found Case, so we're not looking for it anymore.
1695           Case = nullptr;
1696 
1697           // If we found the case and skipped declarations, we can't do the
1698           // optimization.
1699           if (HadSkippedDecl)
1700             return CSFC_Failure;
1701           break;
1702         }
1703       }
1704 
1705       if (!FoundCase)
1706         return CSFC_Success;
1707 
1708       assert(!HadSkippedDecl && "fallthrough after skipping decl");
1709     }
1710 
1711     // If we have statements in our range, then we know that the statements are
1712     // live and need to be added to the set of statements we're tracking.
1713     bool AnyDecls = false;
1714     for (; I != E; ++I) {
1715       AnyDecls |= CodeGenFunction::mightAddDeclToScope(*I);
1716 
1717       switch (CollectStatementsForCase(*I, nullptr, FoundCase, ResultStmts)) {
1718       case CSFC_Failure: return CSFC_Failure;
1719       case CSFC_FallThrough:
1720         // A fallthrough result means that the statement was simple and just
1721         // included in ResultStmt, keep adding them afterwards.
1722         break;
1723       case CSFC_Success:
1724         // A successful result means that we found the break statement and
1725         // stopped statement inclusion.  We just ensure that any leftover stmts
1726         // are skippable and return success ourselves.
1727         for (++I; I != E; ++I)
1728           if (CodeGenFunction::ContainsLabel(*I, true))
1729             return CSFC_Failure;
1730         return CSFC_Success;
1731       }
1732     }
1733 
1734     // If we're about to fall out of a scope without hitting a 'break;', we
1735     // can't perform the optimization if there were any decls in that scope
1736     // (we'd lose their end-of-lifetime).
1737     if (AnyDecls) {
1738       // If the entire compound statement was live, there's one more thing we
1739       // can try before giving up: emit the whole thing as a single statement.
1740       // We can do that unless the statement contains a 'break;'.
1741       // FIXME: Such a break must be at the end of a construct within this one.
1742       // We could emit this by just ignoring the BreakStmts entirely.
1743       if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
1744         ResultStmts.resize(StartSize);
1745         ResultStmts.push_back(S);
1746       } else {
1747         return CSFC_Failure;
1748       }
1749     }
1750 
1751     return CSFC_FallThrough;
1752   }
1753 
1754   // Okay, this is some other statement that we don't handle explicitly, like a
1755   // for statement or increment etc.  If we are skipping over this statement,
1756   // just verify it doesn't have labels, which would make it invalid to elide.
1757   if (Case) {
1758     if (CodeGenFunction::ContainsLabel(S, true))
1759       return CSFC_Failure;
1760     return CSFC_Success;
1761   }
1762 
1763   // Otherwise, we want to include this statement.  Everything is cool with that
1764   // so long as it doesn't contain a break out of the switch we're in.
1765   if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
1766 
1767   // Otherwise, everything is great.  Include the statement and tell the caller
1768   // that we fall through and include the next statement as well.
1769   ResultStmts.push_back(S);
1770   return CSFC_FallThrough;
1771 }
1772 
1773 /// FindCaseStatementsForValue - Find the case statement being jumped to and
1774 /// then invoke CollectStatementsForCase to find the list of statements to emit
1775 /// for a switch on constant.  See the comment above CollectStatementsForCase
1776 /// for more details.
1777 static bool FindCaseStatementsForValue(const SwitchStmt &S,
1778                                        const llvm::APSInt &ConstantCondValue,
1779                                 SmallVectorImpl<const Stmt*> &ResultStmts,
1780                                        ASTContext &C,
1781                                        const SwitchCase *&ResultCase) {
1782   // First step, find the switch case that is being branched to.  We can do this
1783   // efficiently by scanning the SwitchCase list.
1784   const SwitchCase *Case = S.getSwitchCaseList();
1785   const DefaultStmt *DefaultCase = nullptr;
1786 
1787   for (; Case; Case = Case->getNextSwitchCase()) {
1788     // It's either a default or case.  Just remember the default statement in
1789     // case we're not jumping to any numbered cases.
1790     if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Case)) {
1791       DefaultCase = DS;
1792       continue;
1793     }
1794 
1795     // Check to see if this case is the one we're looking for.
1796     const CaseStmt *CS = cast<CaseStmt>(Case);
1797     // Don't handle case ranges yet.
1798     if (CS->getRHS()) return false;
1799 
1800     // If we found our case, remember it as 'case'.
1801     if (CS->getLHS()->EvaluateKnownConstInt(C) == ConstantCondValue)
1802       break;
1803   }
1804 
1805   // If we didn't find a matching case, we use a default if it exists, or we
1806   // elide the whole switch body!
1807   if (!Case) {
1808     // It is safe to elide the body of the switch if it doesn't contain labels
1809     // etc.  If it is safe, return successfully with an empty ResultStmts list.
1810     if (!DefaultCase)
1811       return !CodeGenFunction::ContainsLabel(&S);
1812     Case = DefaultCase;
1813   }
1814 
1815   // Ok, we know which case is being jumped to, try to collect all the
1816   // statements that follow it.  This can fail for a variety of reasons.  Also,
1817   // check to see that the recursive walk actually found our case statement.
1818   // Insane cases like this can fail to find it in the recursive walk since we
1819   // don't handle every stmt kind:
1820   // switch (4) {
1821   //   while (1) {
1822   //     case 4: ...
1823   bool FoundCase = false;
1824   ResultCase = Case;
1825   return CollectStatementsForCase(S.getBody(), Case, FoundCase,
1826                                   ResultStmts) != CSFC_Failure &&
1827          FoundCase;
1828 }
1829 
1830 static Optional<SmallVector<uint64_t, 16>>
1831 getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
1832   // Are there enough branches to weight them?
1833   if (Likelihoods.size() <= 1)
1834     return None;
1835 
1836   uint64_t NumUnlikely = 0;
1837   uint64_t NumNone = 0;
1838   uint64_t NumLikely = 0;
1839   for (const auto LH : Likelihoods) {
1840     switch (LH) {
1841     case Stmt::LH_Unlikely:
1842       ++NumUnlikely;
1843       break;
1844     case Stmt::LH_None:
1845       ++NumNone;
1846       break;
1847     case Stmt::LH_Likely:
1848       ++NumLikely;
1849       break;
1850     }
1851   }
1852 
1853   // Is there a likelihood attribute used?
1854   if (NumUnlikely == 0 && NumLikely == 0)
1855     return None;
1856 
1857   // When multiple cases share the same code they can be combined during
1858   // optimization. In that case the weights of the branch will be the sum of
1859   // the individual weights. Make sure the combined sum of all neutral cases
1860   // doesn't exceed the value of a single likely attribute.
1861   // The additions both avoid divisions by 0 and make sure the weights of None
1862   // don't exceed the weight of Likely.
1863   const uint64_t Likely = INT32_MAX / (NumLikely + 2);
1864   const uint64_t None = Likely / (NumNone + 1);
1865   const uint64_t Unlikely = 0;
1866 
1867   SmallVector<uint64_t, 16> Result;
1868   Result.reserve(Likelihoods.size());
1869   for (const auto LH : Likelihoods) {
1870     switch (LH) {
1871     case Stmt::LH_Unlikely:
1872       Result.push_back(Unlikely);
1873       break;
1874     case Stmt::LH_None:
1875       Result.push_back(None);
1876       break;
1877     case Stmt::LH_Likely:
1878       Result.push_back(Likely);
1879       break;
1880     }
1881   }
1882 
1883   return Result;
1884 }
1885 
1886 void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
1887   // Handle nested switch statements.
1888   llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
1889   SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
1890   SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
1891   llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
1892 
1893   // See if we can constant fold the condition of the switch and therefore only
1894   // emit the live case statement (if any) of the switch.
1895   llvm::APSInt ConstantCondValue;
1896   if (ConstantFoldsToSimpleInteger(S.getCond(), ConstantCondValue)) {
1897     SmallVector<const Stmt*, 4> CaseStmts;
1898     const SwitchCase *Case = nullptr;
1899     if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts,
1900                                    getContext(), Case)) {
1901       if (Case)
1902         incrementProfileCounter(Case);
1903       RunCleanupsScope ExecutedScope(*this);
1904 
1905       if (S.getInit())
1906         EmitStmt(S.getInit());
1907 
1908       // Emit the condition variable if needed inside the entire cleanup scope
1909       // used by this special case for constant folded switches.
1910       if (S.getConditionVariable())
1911         EmitDecl(*S.getConditionVariable());
1912 
1913       // At this point, we are no longer "within" a switch instance, so
1914       // we can temporarily enforce this to ensure that any embedded case
1915       // statements are not emitted.
1916       SwitchInsn = nullptr;
1917 
1918       // Okay, we can dead code eliminate everything except this case.  Emit the
1919       // specified series of statements and we're good.
1920       for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i)
1921         EmitStmt(CaseStmts[i]);
1922       incrementProfileCounter(&S);
1923 
1924       // Now we want to restore the saved switch instance so that nested
1925       // switches continue to function properly
1926       SwitchInsn = SavedSwitchInsn;
1927 
1928       return;
1929     }
1930   }
1931 
1932   JumpDest SwitchExit = getJumpDestInCurrentScope("sw.epilog");
1933 
1934   RunCleanupsScope ConditionScope(*this);
1935 
1936   if (S.getInit())
1937     EmitStmt(S.getInit());
1938 
1939   if (S.getConditionVariable())
1940     EmitDecl(*S.getConditionVariable());
1941   llvm::Value *CondV = EmitScalarExpr(S.getCond());
1942 
1943   // Create basic block to hold stuff that comes after switch
1944   // statement. We also need to create a default block now so that
1945   // explicit case ranges tests can have a place to jump to on
1946   // failure.
1947   llvm::BasicBlock *DefaultBlock = createBasicBlock("sw.default");
1948   SwitchInsn = Builder.CreateSwitch(CondV, DefaultBlock);
1949   if (PGO.haveRegionCounts()) {
1950     // Walk the SwitchCase list to find how many there are.
1951     uint64_t DefaultCount = 0;
1952     unsigned NumCases = 0;
1953     for (const SwitchCase *Case = S.getSwitchCaseList();
1954          Case;
1955          Case = Case->getNextSwitchCase()) {
1956       if (isa<DefaultStmt>(Case))
1957         DefaultCount = getProfileCount(Case);
1958       NumCases += 1;
1959     }
1960     SwitchWeights = new SmallVector<uint64_t, 16>();
1961     SwitchWeights->reserve(NumCases);
1962     // The default needs to be first. We store the edge count, so we already
1963     // know the right weight.
1964     SwitchWeights->push_back(DefaultCount);
1965   } else if (CGM.getCodeGenOpts().OptimizationLevel) {
1966     SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
1967     // Initialize the default case.
1968     SwitchLikelihood->push_back(Stmt::LH_None);
1969   }
1970 
1971   CaseRangeBlock = DefaultBlock;
1972 
1973   // Clear the insertion point to indicate we are in unreachable code.
1974   Builder.ClearInsertionPoint();
1975 
1976   // All break statements jump to NextBlock. If BreakContinueStack is non-empty
1977   // then reuse last ContinueBlock.
1978   JumpDest OuterContinue;
1979   if (!BreakContinueStack.empty())
1980     OuterContinue = BreakContinueStack.back().ContinueBlock;
1981 
1982   BreakContinueStack.push_back(BreakContinue(SwitchExit, OuterContinue));
1983 
1984   // Emit switch body.
1985   EmitStmt(S.getBody());
1986 
1987   BreakContinueStack.pop_back();
1988 
1989   // Update the default block in case explicit case range tests have
1990   // been chained on top.
1991   SwitchInsn->setDefaultDest(CaseRangeBlock);
1992 
1993   // If a default was never emitted:
1994   if (!DefaultBlock->getParent()) {
1995     // If we have cleanups, emit the default block so that there's a
1996     // place to jump through the cleanups from.
1997     if (ConditionScope.requiresCleanups()) {
1998       EmitBlock(DefaultBlock);
1999 
2000     // Otherwise, just forward the default block to the switch end.
2001     } else {
2002       DefaultBlock->replaceAllUsesWith(SwitchExit.getBlock());
2003       delete DefaultBlock;
2004     }
2005   }
2006 
2007   ConditionScope.ForceCleanup();
2008 
2009   // Emit continuation.
2010   EmitBlock(SwitchExit.getBlock(), true);
2011   incrementProfileCounter(&S);
2012 
2013   // If the switch has a condition wrapped by __builtin_unpredictable,
2014   // create metadata that specifies that the switch is unpredictable.
2015   // Don't bother if not optimizing because that metadata would not be used.
2016   auto *Call = dyn_cast<CallExpr>(S.getCond());
2017   if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2018     auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
2019     if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2020       llvm::MDBuilder MDHelper(getLLVMContext());
2021       SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable,
2022                               MDHelper.createUnpredictable());
2023     }
2024   }
2025 
2026   if (SwitchWeights) {
2027     assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
2028            "switch weights do not match switch cases");
2029     // If there's only one jump destination there's no sense weighting it.
2030     if (SwitchWeights->size() > 1)
2031       SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,
2032                               createProfileWeights(*SwitchWeights));
2033     delete SwitchWeights;
2034   } else if (SwitchLikelihood) {
2035     assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
2036            "switch likelihoods do not match switch cases");
2037     Optional<SmallVector<uint64_t, 16>> LHW =
2038         getLikelihoodWeights(*SwitchLikelihood);
2039     if (LHW) {
2040       llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2041       SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,
2042                               createProfileWeights(*LHW));
2043     }
2044     delete SwitchLikelihood;
2045   }
2046   SwitchInsn = SavedSwitchInsn;
2047   SwitchWeights = SavedSwitchWeights;
2048   SwitchLikelihood = SavedSwitchLikelihood;
2049   CaseRangeBlock = SavedCRBlock;
2050 }
2051 
2052 static std::string
2053 SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
2054                  SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=nullptr) {
2055   std::string Result;
2056 
2057   while (*Constraint) {
2058     switch (*Constraint) {
2059     default:
2060       Result += Target.convertConstraint(Constraint);
2061       break;
2062     // Ignore these
2063     case '*':
2064     case '?':
2065     case '!':
2066     case '=': // Will see this and the following in mult-alt constraints.
2067     case '+':
2068       break;
2069     case '#': // Ignore the rest of the constraint alternative.
2070       while (Constraint[1] && Constraint[1] != ',')
2071         Constraint++;
2072       break;
2073     case '&':
2074     case '%':
2075       Result += *Constraint;
2076       while (Constraint[1] && Constraint[1] == *Constraint)
2077         Constraint++;
2078       break;
2079     case ',':
2080       Result += "|";
2081       break;
2082     case 'g':
2083       Result += "imr";
2084       break;
2085     case '[': {
2086       assert(OutCons &&
2087              "Must pass output names to constraints with a symbolic name");
2088       unsigned Index;
2089       bool result = Target.resolveSymbolicName(Constraint, *OutCons, Index);
2090       assert(result && "Could not resolve symbolic name"); (void)result;
2091       Result += llvm::utostr(Index);
2092       break;
2093     }
2094     }
2095 
2096     Constraint++;
2097   }
2098 
2099   return Result;
2100 }
2101 
2102 /// AddVariableConstraints - Look at AsmExpr and if it is a variable declared
2103 /// as using a particular register add that as a constraint that will be used
2104 /// in this asm stmt.
2105 static std::string
2106 AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,
2107                        const TargetInfo &Target, CodeGenModule &CGM,
2108                        const AsmStmt &Stmt, const bool EarlyClobber,
2109                        std::string *GCCReg = nullptr) {
2110   const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(&AsmExpr);
2111   if (!AsmDeclRef)
2112     return Constraint;
2113   const ValueDecl &Value = *AsmDeclRef->getDecl();
2114   const VarDecl *Variable = dyn_cast<VarDecl>(&Value);
2115   if (!Variable)
2116     return Constraint;
2117   if (Variable->getStorageClass() != SC_Register)
2118     return Constraint;
2119   AsmLabelAttr *Attr = Variable->getAttr<AsmLabelAttr>();
2120   if (!Attr)
2121     return Constraint;
2122   StringRef Register = Attr->getLabel();
2123   assert(Target.isValidGCCRegisterName(Register));
2124   // We're using validateOutputConstraint here because we only care if
2125   // this is a register constraint.
2126   TargetInfo::ConstraintInfo Info(Constraint, "");
2127   if (Target.validateOutputConstraint(Info) &&
2128       !Info.allowsRegister()) {
2129     CGM.ErrorUnsupported(&Stmt, "__asm__");
2130     return Constraint;
2131   }
2132   // Canonicalize the register here before returning it.
2133   Register = Target.getNormalizedGCCRegisterName(Register);
2134   if (GCCReg != nullptr)
2135     *GCCReg = Register.str();
2136   return (EarlyClobber ? "&{" : "{") + Register.str() + "}";
2137 }
2138 
2139 std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
2140     const TargetInfo::ConstraintInfo &Info, LValue InputValue,
2141     QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {
2142   if (Info.allowsRegister() || !Info.allowsMemory()) {
2143     if (CodeGenFunction::hasScalarEvaluationKind(InputType))
2144       return {EmitLoadOfLValue(InputValue, Loc).getScalarVal(), nullptr};
2145 
2146     llvm::Type *Ty = ConvertType(InputType);
2147     uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2148     if ((Size <= 64 && llvm::isPowerOf2_64(Size)) ||
2149         getTargetHooks().isScalarizableAsmOperand(*this, Ty)) {
2150       Ty = llvm::IntegerType::get(getLLVMContext(), Size);
2151 
2152       return {Builder.CreateLoad(Builder.CreateElementBitCast(
2153                   InputValue.getAddress(*this), Ty)),
2154               nullptr};
2155     }
2156   }
2157 
2158   Address Addr = InputValue.getAddress(*this);
2159   ConstraintStr += '*';
2160   return {Addr.getPointer(), Addr.getElementType()};
2161 }
2162 
2163 std::pair<llvm::Value *, llvm::Type *>
2164 CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
2165                               const Expr *InputExpr,
2166                               std::string &ConstraintStr) {
2167   // If this can't be a register or memory, i.e., has to be a constant
2168   // (immediate or symbolic), try to emit it as such.
2169   if (!Info.allowsRegister() && !Info.allowsMemory()) {
2170     if (Info.requiresImmediateConstant()) {
2171       Expr::EvalResult EVResult;
2172       InputExpr->EvaluateAsRValue(EVResult, getContext(), true);
2173 
2174       llvm::APSInt IntResult;
2175       if (EVResult.Val.toIntegralConstant(IntResult, InputExpr->getType(),
2176                                           getContext()))
2177         return {llvm::ConstantInt::get(getLLVMContext(), IntResult), nullptr};
2178     }
2179 
2180     Expr::EvalResult Result;
2181     if (InputExpr->EvaluateAsInt(Result, getContext()))
2182       return {llvm::ConstantInt::get(getLLVMContext(), Result.Val.getInt()),
2183               nullptr};
2184   }
2185 
2186   if (Info.allowsRegister() || !Info.allowsMemory())
2187     if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType()))
2188       return {EmitScalarExpr(InputExpr), nullptr};
2189   if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2190     return {EmitScalarExpr(InputExpr), nullptr};
2191   InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
2192   LValue Dest = EmitLValue(InputExpr);
2193   return EmitAsmInputLValue(Info, Dest, InputExpr->getType(), ConstraintStr,
2194                             InputExpr->getExprLoc());
2195 }
2196 
2197 /// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2198 /// asm call instruction.  The !srcloc MDNode contains a list of constant
2199 /// integers which are the source locations of the start of each line in the
2200 /// asm.
2201 static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2202                                       CodeGenFunction &CGF) {
2203   SmallVector<llvm::Metadata *, 8> Locs;
2204   // Add the location of the first line to the MDNode.
2205   Locs.push_back(llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
2206       CGF.Int64Ty, Str->getBeginLoc().getRawEncoding())));
2207   StringRef StrVal = Str->getString();
2208   if (!StrVal.empty()) {
2209     const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2210     const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2211     unsigned StartToken = 0;
2212     unsigned ByteOffset = 0;
2213 
2214     // Add the location of the start of each subsequent line of the asm to the
2215     // MDNode.
2216     for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
2217       if (StrVal[i] != '\n') continue;
2218       SourceLocation LineLoc = Str->getLocationOfByte(
2219           i + 1, SM, LangOpts, CGF.getTarget(), &StartToken, &ByteOffset);
2220       Locs.push_back(llvm::ConstantAsMetadata::get(
2221           llvm::ConstantInt::get(CGF.Int64Ty, LineLoc.getRawEncoding())));
2222     }
2223   }
2224 
2225   return llvm::MDNode::get(CGF.getLLVMContext(), Locs);
2226 }
2227 
2228 static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2229                               bool HasUnwindClobber, bool ReadOnly,
2230                               bool ReadNone, bool NoMerge, const AsmStmt &S,
2231                               const std::vector<llvm::Type *> &ResultRegTypes,
2232                               const std::vector<llvm::Type *> &ArgElemTypes,
2233                               CodeGenFunction &CGF,
2234                               std::vector<llvm::Value *> &RegResults) {
2235   if (!HasUnwindClobber)
2236     Result.addFnAttr(llvm::Attribute::NoUnwind);
2237 
2238   if (NoMerge)
2239     Result.addFnAttr(llvm::Attribute::NoMerge);
2240   // Attach readnone and readonly attributes.
2241   if (!HasSideEffect) {
2242     if (ReadNone)
2243       Result.addFnAttr(llvm::Attribute::ReadNone);
2244     else if (ReadOnly)
2245       Result.addFnAttr(llvm::Attribute::ReadOnly);
2246   }
2247 
2248   // Add elementtype attribute for indirect constraints.
2249   for (auto Pair : llvm::enumerate(ArgElemTypes)) {
2250     if (Pair.value()) {
2251       auto Attr = llvm::Attribute::get(
2252           CGF.getLLVMContext(), llvm::Attribute::ElementType, Pair.value());
2253       Result.addParamAttr(Pair.index(), Attr);
2254     }
2255   }
2256 
2257   // Slap the source location of the inline asm into a !srcloc metadata on the
2258   // call.
2259   if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(&S))
2260     Result.setMetadata("srcloc",
2261                        getAsmSrcLocInfo(gccAsmStmt->getAsmString(), CGF));
2262   else {
2263     // At least put the line number on MS inline asm blobs.
2264     llvm::Constant *Loc =
2265         llvm::ConstantInt::get(CGF.Int64Ty, S.getAsmLoc().getRawEncoding());
2266     Result.setMetadata("srcloc",
2267                        llvm::MDNode::get(CGF.getLLVMContext(),
2268                                          llvm::ConstantAsMetadata::get(Loc)));
2269   }
2270 
2271   if (CGF.getLangOpts().assumeFunctionsAreConvergent())
2272     // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
2273     // convergent (meaning, they may call an intrinsically convergent op, such
2274     // as bar.sync, and so can't have certain optimizations applied around
2275     // them).
2276     Result.addFnAttr(llvm::Attribute::Convergent);
2277   // Extract all of the register value results from the asm.
2278   if (ResultRegTypes.size() == 1) {
2279     RegResults.push_back(&Result);
2280   } else {
2281     for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
2282       llvm::Value *Tmp = CGF.Builder.CreateExtractValue(&Result, i, "asmresult");
2283       RegResults.push_back(Tmp);
2284     }
2285   }
2286 }
2287 
2288 void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2289   // Assemble the final asm string.
2290   std::string AsmString = S.generateAsmString(getContext());
2291 
2292   // Get all the output and input constraints together.
2293   SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2294   SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2295 
2296   for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
2297     StringRef Name;
2298     if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
2299       Name = GAS->getOutputName(i);
2300     TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);
2301     bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;
2302     assert(IsValid && "Failed to parse output constraint");
2303     OutputConstraintInfos.push_back(Info);
2304   }
2305 
2306   for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
2307     StringRef Name;
2308     if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
2309       Name = GAS->getInputName(i);
2310     TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);
2311     bool IsValid =
2312       getTarget().validateInputConstraint(OutputConstraintInfos, Info);
2313     assert(IsValid && "Failed to parse input constraint"); (void)IsValid;
2314     InputConstraintInfos.push_back(Info);
2315   }
2316 
2317   std::string Constraints;
2318 
2319   std::vector<LValue> ResultRegDests;
2320   std::vector<QualType> ResultRegQualTys;
2321   std::vector<llvm::Type *> ResultRegTypes;
2322   std::vector<llvm::Type *> ResultTruncRegTypes;
2323   std::vector<llvm::Type *> ArgTypes;
2324   std::vector<llvm::Type *> ArgElemTypes;
2325   std::vector<llvm::Value*> Args;
2326   llvm::BitVector ResultTypeRequiresCast;
2327 
2328   // Keep track of inout constraints.
2329   std::string InOutConstraints;
2330   std::vector<llvm::Value*> InOutArgs;
2331   std::vector<llvm::Type*> InOutArgTypes;
2332   std::vector<llvm::Type*> InOutArgElemTypes;
2333 
2334   // Keep track of out constraints for tied input operand.
2335   std::vector<std::string> OutputConstraints;
2336 
2337   // Keep track of defined physregs.
2338   llvm::SmallSet<std::string, 8> PhysRegOutputs;
2339 
2340   // An inline asm can be marked readonly if it meets the following conditions:
2341   //  - it doesn't have any sideeffects
2342   //  - it doesn't clobber memory
2343   //  - it doesn't return a value by-reference
2344   // It can be marked readnone if it doesn't have any input memory constraints
2345   // in addition to meeting the conditions listed above.
2346   bool ReadOnly = true, ReadNone = true;
2347 
2348   for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
2349     TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];
2350 
2351     // Simplify the output constraint.
2352     std::string OutputConstraint(S.getOutputConstraint(i));
2353     OutputConstraint = SimplifyConstraint(OutputConstraint.c_str() + 1,
2354                                           getTarget(), &OutputConstraintInfos);
2355 
2356     const Expr *OutExpr = S.getOutputExpr(i);
2357     OutExpr = OutExpr->IgnoreParenNoopCasts(getContext());
2358 
2359     std::string GCCReg;
2360     OutputConstraint = AddVariableConstraints(OutputConstraint, *OutExpr,
2361                                               getTarget(), CGM, S,
2362                                               Info.earlyClobber(),
2363                                               &GCCReg);
2364     // Give an error on multiple outputs to same physreg.
2365     if (!GCCReg.empty() && !PhysRegOutputs.insert(GCCReg).second)
2366       CGM.Error(S.getAsmLoc(), "multiple outputs to hard register: " + GCCReg);
2367 
2368     OutputConstraints.push_back(OutputConstraint);
2369     LValue Dest = EmitLValue(OutExpr);
2370     if (!Constraints.empty())
2371       Constraints += ',';
2372 
2373     // If this is a register output, then make the inline asm return it
2374     // by-value.  If this is a memory result, return the value by-reference.
2375     QualType QTy = OutExpr->getType();
2376     const bool IsScalarOrAggregate = hasScalarEvaluationKind(QTy) ||
2377                                      hasAggregateEvaluationKind(QTy);
2378     if (!Info.allowsMemory() && IsScalarOrAggregate) {
2379 
2380       Constraints += "=" + OutputConstraint;
2381       ResultRegQualTys.push_back(QTy);
2382       ResultRegDests.push_back(Dest);
2383 
2384       llvm::Type *Ty = ConvertTypeForMem(QTy);
2385       const bool RequiresCast = Info.allowsRegister() &&
2386           (getTargetHooks().isScalarizableAsmOperand(*this, Ty) ||
2387            Ty->isAggregateType());
2388 
2389       ResultTruncRegTypes.push_back(Ty);
2390       ResultTypeRequiresCast.push_back(RequiresCast);
2391 
2392       if (RequiresCast) {
2393         unsigned Size = getContext().getTypeSize(QTy);
2394         Ty = llvm::IntegerType::get(getLLVMContext(), Size);
2395       }
2396       ResultRegTypes.push_back(Ty);
2397       // If this output is tied to an input, and if the input is larger, then
2398       // we need to set the actual result type of the inline asm node to be the
2399       // same as the input type.
2400       if (Info.hasMatchingInput()) {
2401         unsigned InputNo;
2402         for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
2403           TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2404           if (Input.hasTiedOperand() && Input.getTiedOperand() == i)
2405             break;
2406         }
2407         assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
2408 
2409         QualType InputTy = S.getInputExpr(InputNo)->getType();
2410         QualType OutputType = OutExpr->getType();
2411 
2412         uint64_t InputSize = getContext().getTypeSize(InputTy);
2413         if (getContext().getTypeSize(OutputType) < InputSize) {
2414           // Form the asm to return the value as a larger integer or fp type.
2415           ResultRegTypes.back() = ConvertType(InputTy);
2416         }
2417       }
2418       if (llvm::Type* AdjTy =
2419             getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
2420                                                  ResultRegTypes.back()))
2421         ResultRegTypes.back() = AdjTy;
2422       else {
2423         CGM.getDiags().Report(S.getAsmLoc(),
2424                               diag::err_asm_invalid_type_in_input)
2425             << OutExpr->getType() << OutputConstraint;
2426       }
2427 
2428       // Update largest vector width for any vector types.
2429       if (auto *VT = dyn_cast<llvm::VectorType>(ResultRegTypes.back()))
2430         LargestVectorWidth =
2431             std::max((uint64_t)LargestVectorWidth,
2432                      VT->getPrimitiveSizeInBits().getKnownMinSize());
2433     } else {
2434       Address DestAddr = Dest.getAddress(*this);
2435       // Matrix types in memory are represented by arrays, but accessed through
2436       // vector pointers, with the alignment specified on the access operation.
2437       // For inline assembly, update pointer arguments to use vector pointers.
2438       // Otherwise there will be a mis-match if the matrix is also an
2439       // input-argument which is represented as vector.
2440       if (isa<MatrixType>(OutExpr->getType().getCanonicalType()))
2441         DestAddr = Builder.CreateElementBitCast(
2442             DestAddr, ConvertType(OutExpr->getType()));
2443 
2444       ArgTypes.push_back(DestAddr.getType());
2445       ArgElemTypes.push_back(DestAddr.getElementType());
2446       Args.push_back(DestAddr.getPointer());
2447       Constraints += "=*";
2448       Constraints += OutputConstraint;
2449       ReadOnly = ReadNone = false;
2450     }
2451 
2452     if (Info.isReadWrite()) {
2453       InOutConstraints += ',';
2454 
2455       const Expr *InputExpr = S.getOutputExpr(i);
2456       llvm::Value *Arg;
2457       llvm::Type *ArgElemType;
2458       std::tie(Arg, ArgElemType) = EmitAsmInputLValue(
2459           Info, Dest, InputExpr->getType(), InOutConstraints,
2460           InputExpr->getExprLoc());
2461 
2462       if (llvm::Type* AdjTy =
2463           getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
2464                                                Arg->getType()))
2465         Arg = Builder.CreateBitCast(Arg, AdjTy);
2466 
2467       // Update largest vector width for any vector types.
2468       if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))
2469         LargestVectorWidth =
2470             std::max((uint64_t)LargestVectorWidth,
2471                      VT->getPrimitiveSizeInBits().getKnownMinSize());
2472       // Only tie earlyclobber physregs.
2473       if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))
2474         InOutConstraints += llvm::utostr(i);
2475       else
2476         InOutConstraints += OutputConstraint;
2477 
2478       InOutArgTypes.push_back(Arg->getType());
2479       InOutArgElemTypes.push_back(ArgElemType);
2480       InOutArgs.push_back(Arg);
2481     }
2482   }
2483 
2484   // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2485   // to the return value slot. Only do this when returning in registers.
2486   if (isa<MSAsmStmt>(&S)) {
2487     const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo();
2488     if (RetAI.isDirect() || RetAI.isExtend()) {
2489       // Make a fake lvalue for the return value slot.
2490       LValue ReturnSlot = MakeAddrLValueWithoutTBAA(ReturnValue, FnRetTy);
2491       CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
2492           *this, ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
2493           ResultRegDests, AsmString, S.getNumOutputs());
2494       SawAsmBlock = true;
2495     }
2496   }
2497 
2498   for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
2499     const Expr *InputExpr = S.getInputExpr(i);
2500 
2501     TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
2502 
2503     if (Info.allowsMemory())
2504       ReadNone = false;
2505 
2506     if (!Constraints.empty())
2507       Constraints += ',';
2508 
2509     // Simplify the input constraint.
2510     std::string InputConstraint(S.getInputConstraint(i));
2511     InputConstraint = SimplifyConstraint(InputConstraint.c_str(), getTarget(),
2512                                          &OutputConstraintInfos);
2513 
2514     InputConstraint = AddVariableConstraints(
2515         InputConstraint, *InputExpr->IgnoreParenNoopCasts(getContext()),
2516         getTarget(), CGM, S, false /* No EarlyClobber */);
2517 
2518     std::string ReplaceConstraint (InputConstraint);
2519     llvm::Value *Arg;
2520     llvm::Type *ArgElemType;
2521     std::tie(Arg, ArgElemType) = EmitAsmInput(Info, InputExpr, Constraints);
2522 
2523     // If this input argument is tied to a larger output result, extend the
2524     // input to be the same size as the output.  The LLVM backend wants to see
2525     // the input and output of a matching constraint be the same size.  Note
2526     // that GCC does not define what the top bits are here.  We use zext because
2527     // that is usually cheaper, but LLVM IR should really get an anyext someday.
2528     if (Info.hasTiedOperand()) {
2529       unsigned Output = Info.getTiedOperand();
2530       QualType OutputType = S.getOutputExpr(Output)->getType();
2531       QualType InputTy = InputExpr->getType();
2532 
2533       if (getContext().getTypeSize(OutputType) >
2534           getContext().getTypeSize(InputTy)) {
2535         // Use ptrtoint as appropriate so that we can do our extension.
2536         if (isa<llvm::PointerType>(Arg->getType()))
2537           Arg = Builder.CreatePtrToInt(Arg, IntPtrTy);
2538         llvm::Type *OutputTy = ConvertType(OutputType);
2539         if (isa<llvm::IntegerType>(OutputTy))
2540           Arg = Builder.CreateZExt(Arg, OutputTy);
2541         else if (isa<llvm::PointerType>(OutputTy))
2542           Arg = Builder.CreateZExt(Arg, IntPtrTy);
2543         else if (OutputTy->isFloatingPointTy())
2544           Arg = Builder.CreateFPExt(Arg, OutputTy);
2545       }
2546       // Deal with the tied operands' constraint code in adjustInlineAsmType.
2547       ReplaceConstraint = OutputConstraints[Output];
2548     }
2549     if (llvm::Type* AdjTy =
2550           getTargetHooks().adjustInlineAsmType(*this, ReplaceConstraint,
2551                                                    Arg->getType()))
2552       Arg = Builder.CreateBitCast(Arg, AdjTy);
2553     else
2554       CGM.getDiags().Report(S.getAsmLoc(), diag::err_asm_invalid_type_in_input)
2555           << InputExpr->getType() << InputConstraint;
2556 
2557     // Update largest vector width for any vector types.
2558     if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))
2559       LargestVectorWidth =
2560           std::max((uint64_t)LargestVectorWidth,
2561                    VT->getPrimitiveSizeInBits().getKnownMinSize());
2562 
2563     ArgTypes.push_back(Arg->getType());
2564     ArgElemTypes.push_back(ArgElemType);
2565     Args.push_back(Arg);
2566     Constraints += InputConstraint;
2567   }
2568 
2569   // Append the "input" part of inout constraints.
2570   for (unsigned i = 0, e = InOutArgs.size(); i != e; i++) {
2571     ArgTypes.push_back(InOutArgTypes[i]);
2572     ArgElemTypes.push_back(InOutArgElemTypes[i]);
2573     Args.push_back(InOutArgs[i]);
2574   }
2575   Constraints += InOutConstraints;
2576 
2577   // Labels
2578   SmallVector<llvm::BasicBlock *, 16> Transfer;
2579   llvm::BasicBlock *Fallthrough = nullptr;
2580   bool IsGCCAsmGoto = false;
2581   if (const auto *GS =  dyn_cast<GCCAsmStmt>(&S)) {
2582     IsGCCAsmGoto = GS->isAsmGoto();
2583     if (IsGCCAsmGoto) {
2584       for (const auto *E : GS->labels()) {
2585         JumpDest Dest = getJumpDestForLabel(E->getLabel());
2586         Transfer.push_back(Dest.getBlock());
2587         llvm::BlockAddress *BA =
2588             llvm::BlockAddress::get(CurFn, Dest.getBlock());
2589         Args.push_back(BA);
2590         ArgTypes.push_back(BA->getType());
2591         ArgElemTypes.push_back(nullptr);
2592         if (!Constraints.empty())
2593           Constraints += ',';
2594         Constraints += 'i';
2595       }
2596       Fallthrough = createBasicBlock("asm.fallthrough");
2597     }
2598   }
2599 
2600   bool HasUnwindClobber = false;
2601 
2602   // Clobbers
2603   for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
2604     StringRef Clobber = S.getClobber(i);
2605 
2606     if (Clobber == "memory")
2607       ReadOnly = ReadNone = false;
2608     else if (Clobber == "unwind") {
2609       HasUnwindClobber = true;
2610       continue;
2611     } else if (Clobber != "cc") {
2612       Clobber = getTarget().getNormalizedGCCRegisterName(Clobber);
2613       if (CGM.getCodeGenOpts().StackClashProtector &&
2614           getTarget().isSPRegName(Clobber)) {
2615         CGM.getDiags().Report(S.getAsmLoc(),
2616                               diag::warn_stack_clash_protection_inline_asm);
2617       }
2618     }
2619 
2620     if (isa<MSAsmStmt>(&S)) {
2621       if (Clobber == "eax" || Clobber == "edx") {
2622         if (Constraints.find("=&A") != std::string::npos)
2623           continue;
2624         std::string::size_type position1 =
2625             Constraints.find("={" + Clobber.str() + "}");
2626         if (position1 != std::string::npos) {
2627           Constraints.insert(position1 + 1, "&");
2628           continue;
2629         }
2630         std::string::size_type position2 = Constraints.find("=A");
2631         if (position2 != std::string::npos) {
2632           Constraints.insert(position2 + 1, "&");
2633           continue;
2634         }
2635       }
2636     }
2637     if (!Constraints.empty())
2638       Constraints += ',';
2639 
2640     Constraints += "~{";
2641     Constraints += Clobber;
2642     Constraints += '}';
2643   }
2644 
2645   assert(!(HasUnwindClobber && IsGCCAsmGoto) &&
2646          "unwind clobber can't be used with asm goto");
2647 
2648   // Add machine specific clobbers
2649   std::string MachineClobbers = getTarget().getClobbers();
2650   if (!MachineClobbers.empty()) {
2651     if (!Constraints.empty())
2652       Constraints += ',';
2653     Constraints += MachineClobbers;
2654   }
2655 
2656   llvm::Type *ResultType;
2657   if (ResultRegTypes.empty())
2658     ResultType = VoidTy;
2659   else if (ResultRegTypes.size() == 1)
2660     ResultType = ResultRegTypes[0];
2661   else
2662     ResultType = llvm::StructType::get(getLLVMContext(), ResultRegTypes);
2663 
2664   llvm::FunctionType *FTy =
2665     llvm::FunctionType::get(ResultType, ArgTypes, false);
2666 
2667   bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
2668 
2669   llvm::InlineAsm::AsmDialect GnuAsmDialect =
2670       CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
2671           ? llvm::InlineAsm::AD_ATT
2672           : llvm::InlineAsm::AD_Intel;
2673   llvm::InlineAsm::AsmDialect AsmDialect = isa<MSAsmStmt>(&S) ?
2674     llvm::InlineAsm::AD_Intel : GnuAsmDialect;
2675 
2676   llvm::InlineAsm *IA = llvm::InlineAsm::get(
2677       FTy, AsmString, Constraints, HasSideEffect,
2678       /* IsAlignStack */ false, AsmDialect, HasUnwindClobber);
2679   std::vector<llvm::Value*> RegResults;
2680   if (IsGCCAsmGoto) {
2681     llvm::CallBrInst *Result =
2682         Builder.CreateCallBr(IA, Fallthrough, Transfer, Args);
2683     EmitBlock(Fallthrough);
2684     UpdateAsmCallInst(cast<llvm::CallBase>(*Result), HasSideEffect, false,
2685                       ReadOnly, ReadNone, InNoMergeAttributedStmt, S,
2686                       ResultRegTypes, ArgElemTypes, *this, RegResults);
2687   } else if (HasUnwindClobber) {
2688     llvm::CallBase *Result = EmitCallOrInvoke(IA, Args, "");
2689     UpdateAsmCallInst(*Result, HasSideEffect, true, ReadOnly, ReadNone,
2690                       InNoMergeAttributedStmt, S, ResultRegTypes, ArgElemTypes,
2691                       *this, RegResults);
2692   } else {
2693     llvm::CallInst *Result =
2694         Builder.CreateCall(IA, Args, getBundlesForFunclet(IA));
2695     UpdateAsmCallInst(cast<llvm::CallBase>(*Result), HasSideEffect, false,
2696                       ReadOnly, ReadNone, InNoMergeAttributedStmt, S,
2697                       ResultRegTypes, ArgElemTypes, *this, RegResults);
2698   }
2699 
2700   assert(RegResults.size() == ResultRegTypes.size());
2701   assert(RegResults.size() == ResultTruncRegTypes.size());
2702   assert(RegResults.size() == ResultRegDests.size());
2703   // ResultRegDests can be also populated by addReturnRegisterOutputs() above,
2704   // in which case its size may grow.
2705   assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
2706   for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
2707     llvm::Value *Tmp = RegResults[i];
2708     llvm::Type *TruncTy = ResultTruncRegTypes[i];
2709 
2710     // If the result type of the LLVM IR asm doesn't match the result type of
2711     // the expression, do the conversion.
2712     if (ResultRegTypes[i] != ResultTruncRegTypes[i]) {
2713 
2714       // Truncate the integer result to the right size, note that TruncTy can be
2715       // a pointer.
2716       if (TruncTy->isFloatingPointTy())
2717         Tmp = Builder.CreateFPTrunc(Tmp, TruncTy);
2718       else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
2719         uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(TruncTy);
2720         Tmp = Builder.CreateTrunc(Tmp,
2721                    llvm::IntegerType::get(getLLVMContext(), (unsigned)ResSize));
2722         Tmp = Builder.CreateIntToPtr(Tmp, TruncTy);
2723       } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
2724         uint64_t TmpSize =CGM.getDataLayout().getTypeSizeInBits(Tmp->getType());
2725         Tmp = Builder.CreatePtrToInt(Tmp,
2726                    llvm::IntegerType::get(getLLVMContext(), (unsigned)TmpSize));
2727         Tmp = Builder.CreateTrunc(Tmp, TruncTy);
2728       } else if (TruncTy->isIntegerTy()) {
2729         Tmp = Builder.CreateZExtOrTrunc(Tmp, TruncTy);
2730       } else if (TruncTy->isVectorTy()) {
2731         Tmp = Builder.CreateBitCast(Tmp, TruncTy);
2732       }
2733     }
2734 
2735     LValue Dest = ResultRegDests[i];
2736     // ResultTypeRequiresCast elements correspond to the first
2737     // ResultTypeRequiresCast.size() elements of RegResults.
2738     if ((i < ResultTypeRequiresCast.size()) && ResultTypeRequiresCast[i]) {
2739       unsigned Size = getContext().getTypeSize(ResultRegQualTys[i]);
2740       Address A = Builder.CreateElementBitCast(Dest.getAddress(*this),
2741                                                ResultRegTypes[i]);
2742       if (getTargetHooks().isScalarizableAsmOperand(*this, TruncTy)) {
2743         Builder.CreateStore(Tmp, A);
2744         continue;
2745       }
2746 
2747       QualType Ty = getContext().getIntTypeForBitwidth(Size, /*Signed*/ false);
2748       if (Ty.isNull()) {
2749         const Expr *OutExpr = S.getOutputExpr(i);
2750         CGM.getDiags().Report(OutExpr->getExprLoc(),
2751                               diag::err_store_value_to_reg);
2752         return;
2753       }
2754       Dest = MakeAddrLValue(A, Ty);
2755     }
2756     EmitStoreThroughLValue(RValue::get(Tmp), Dest);
2757   }
2758 }
2759 
2760 LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
2761   const RecordDecl *RD = S.getCapturedRecordDecl();
2762   QualType RecordTy = getContext().getRecordType(RD);
2763 
2764   // Initialize the captured struct.
2765   LValue SlotLV =
2766     MakeAddrLValue(CreateMemTemp(RecordTy, "agg.captured"), RecordTy);
2767 
2768   RecordDecl::field_iterator CurField = RD->field_begin();
2769   for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
2770                                                  E = S.capture_init_end();
2771        I != E; ++I, ++CurField) {
2772     LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField);
2773     if (CurField->hasCapturedVLAType()) {
2774       EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);
2775     } else {
2776       EmitInitializerForField(*CurField, LV, *I);
2777     }
2778   }
2779 
2780   return SlotLV;
2781 }
2782 
2783 /// Generate an outlined function for the body of a CapturedStmt, store any
2784 /// captured variables into the captured struct, and call the outlined function.
2785 llvm::Function *
2786 CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
2787   LValue CapStruct = InitCapturedStruct(S);
2788 
2789   // Emit the CapturedDecl
2790   CodeGenFunction CGF(CGM, true);
2791   CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
2792   llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
2793   delete CGF.CapturedStmtInfo;
2794 
2795   // Emit call to the helper function.
2796   EmitCallOrInvoke(F, CapStruct.getPointer(*this));
2797 
2798   return F;
2799 }
2800 
2801 Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
2802   LValue CapStruct = InitCapturedStruct(S);
2803   return CapStruct.getAddress(*this);
2804 }
2805 
2806 /// Creates the outlined function for a CapturedStmt.
2807 llvm::Function *
2808 CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
2809   assert(CapturedStmtInfo &&
2810     "CapturedStmtInfo should be set when generating the captured function");
2811   const CapturedDecl *CD = S.getCapturedDecl();
2812   const RecordDecl *RD = S.getCapturedRecordDecl();
2813   SourceLocation Loc = S.getBeginLoc();
2814   assert(CD->hasBody() && "missing CapturedDecl body");
2815 
2816   // Build the argument list.
2817   ASTContext &Ctx = CGM.getContext();
2818   FunctionArgList Args;
2819   Args.append(CD->param_begin(), CD->param_end());
2820 
2821   // Create the function declaration.
2822   const CGFunctionInfo &FuncInfo =
2823     CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
2824   llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
2825 
2826   llvm::Function *F =
2827     llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
2828                            CapturedStmtInfo->getHelperName(), &CGM.getModule());
2829   CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
2830   if (CD->isNothrow())
2831     F->addFnAttr(llvm::Attribute::NoUnwind);
2832 
2833   // Generate the function.
2834   StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
2835                 CD->getBody()->getBeginLoc());
2836   // Set the context parameter in CapturedStmtInfo.
2837   Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam());
2838   CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
2839 
2840   // Initialize variable-length arrays.
2841   LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(),
2842                                            Ctx.getTagDeclType(RD));
2843   for (auto *FD : RD->fields()) {
2844     if (FD->hasCapturedVLAType()) {
2845       auto *ExprArg =
2846           EmitLoadOfLValue(EmitLValueForField(Base, FD), S.getBeginLoc())
2847               .getScalarVal();
2848       auto VAT = FD->getCapturedVLAType();
2849       VLASizeMap[VAT->getSizeExpr()] = ExprArg;
2850     }
2851   }
2852 
2853   // If 'this' is captured, load it into CXXThisValue.
2854   if (CapturedStmtInfo->isCXXThisExprCaptured()) {
2855     FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
2856     LValue ThisLValue = EmitLValueForField(Base, FD);
2857     CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal();
2858   }
2859 
2860   PGO.assignRegionCounters(GlobalDecl(CD), F);
2861   CapturedStmtInfo->EmitBody(*this, CD->getBody());
2862   FinishFunction(CD->getBodyRBrace());
2863 
2864   return F;
2865 }
2866