19dfe4e7cSTobias Grosser //===------ PPCGCodeGeneration.cpp - Polly Accelerator Code Generation. ---===//
29dfe4e7cSTobias Grosser //
39dfe4e7cSTobias Grosser //                     The LLVM Compiler Infrastructure
49dfe4e7cSTobias Grosser //
59dfe4e7cSTobias Grosser // This file is distributed under the University of Illinois Open Source
69dfe4e7cSTobias Grosser // License. See LICENSE.TXT for details.
79dfe4e7cSTobias Grosser //
89dfe4e7cSTobias Grosser //===----------------------------------------------------------------------===//
99dfe4e7cSTobias Grosser //
109dfe4e7cSTobias Grosser // Take a scop created by ScopInfo and map it to GPU code using the ppcg
119dfe4e7cSTobias Grosser // GPU mapping strategy.
129dfe4e7cSTobias Grosser //
139dfe4e7cSTobias Grosser //===----------------------------------------------------------------------===//
149dfe4e7cSTobias Grosser 
159dfe4e7cSTobias Grosser #include "polly/CodeGen/IslNodeBuilder.h"
169dfe4e7cSTobias Grosser #include "polly/DependenceInfo.h"
179dfe4e7cSTobias Grosser #include "polly/LinkAllPasses.h"
18f384594dSTobias Grosser #include "polly/Options.h"
199dfe4e7cSTobias Grosser #include "polly/ScopInfo.h"
209dfe4e7cSTobias Grosser #include "llvm/Analysis/AliasAnalysis.h"
219dfe4e7cSTobias Grosser #include "llvm/Analysis/BasicAliasAnalysis.h"
229dfe4e7cSTobias Grosser #include "llvm/Analysis/GlobalsModRef.h"
239dfe4e7cSTobias Grosser #include "llvm/Analysis/PostDominators.h"
249dfe4e7cSTobias Grosser #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
259dfe4e7cSTobias Grosser 
26f384594dSTobias Grosser #include "isl/union_map.h"
27f384594dSTobias Grosser 
28e938517eSTobias Grosser extern "C" {
29a56f8f8eSTobias Grosser #include "ppcg/cuda.h"
30a56f8f8eSTobias Grosser #include "ppcg/gpu.h"
31a56f8f8eSTobias Grosser #include "ppcg/gpu_print.h"
32a56f8f8eSTobias Grosser #include "ppcg/ppcg.h"
33a56f8f8eSTobias Grosser #include "ppcg/schedule.h"
34e938517eSTobias Grosser }
35e938517eSTobias Grosser 
369dfe4e7cSTobias Grosser #include "llvm/Support/Debug.h"
379dfe4e7cSTobias Grosser 
389dfe4e7cSTobias Grosser using namespace polly;
399dfe4e7cSTobias Grosser using namespace llvm;
409dfe4e7cSTobias Grosser 
419dfe4e7cSTobias Grosser #define DEBUG_TYPE "polly-codegen-ppcg"
429dfe4e7cSTobias Grosser 
43f384594dSTobias Grosser static cl::opt<bool> DumpSchedule("polly-acc-dump-schedule",
44f384594dSTobias Grosser                                   cl::desc("Dump the computed GPU Schedule"),
45681bd568STobias Grosser                                   cl::Hidden, cl::init(false), cl::ZeroOrMore,
46f384594dSTobias Grosser                                   cl::cat(PollyCategory));
4769b46751STobias Grosser 
4869b46751STobias Grosser static cl::opt<bool>
4969b46751STobias Grosser     DumpCode("polly-acc-dump-code",
5069b46751STobias Grosser              cl::desc("Dump C code describing the GPU mapping"), cl::Hidden,
5169b46751STobias Grosser              cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
5269b46751STobias Grosser 
5360c60025STobias Grosser /// Create the ast expressions for a ScopStmt.
5460c60025STobias Grosser ///
5560c60025STobias Grosser /// This function is a callback for to generate the ast expressions for each
5660c60025STobias Grosser /// of the scheduled ScopStmts.
5760c60025STobias Grosser static __isl_give isl_id_to_ast_expr *pollyBuildAstExprForStmt(
5860c60025STobias Grosser     void *Stmt, isl_ast_build *Build,
5960c60025STobias Grosser     isl_multi_pw_aff *(*FunctionIndex)(__isl_take isl_multi_pw_aff *MPA,
6060c60025STobias Grosser                                        isl_id *Id, void *User),
6160c60025STobias Grosser     void *UserIndex,
6260c60025STobias Grosser     isl_ast_expr *(*FunctionExpr)(isl_ast_expr *Expr, isl_id *Id, void *User),
6360c60025STobias Grosser     void *User_expr) {
6460c60025STobias Grosser 
6560c60025STobias Grosser   // TODO: Implement the AST expression generation. For now we just return a
6660c60025STobias Grosser   // nullptr to ensure that we do not free uninitialized pointers.
6760c60025STobias Grosser 
6860c60025STobias Grosser   return nullptr;
6960c60025STobias Grosser }
70f384594dSTobias Grosser 
719dfe4e7cSTobias Grosser namespace {
729dfe4e7cSTobias Grosser class PPCGCodeGeneration : public ScopPass {
739dfe4e7cSTobias Grosser public:
749dfe4e7cSTobias Grosser   static char ID;
759dfe4e7cSTobias Grosser 
76e938517eSTobias Grosser   /// The scop that is currently processed.
77e938517eSTobias Grosser   Scop *S;
78e938517eSTobias Grosser 
799dfe4e7cSTobias Grosser   PPCGCodeGeneration() : ScopPass(ID) {}
809dfe4e7cSTobias Grosser 
81e938517eSTobias Grosser   /// Construct compilation options for PPCG.
82e938517eSTobias Grosser   ///
83e938517eSTobias Grosser   /// @returns The compilation options.
84e938517eSTobias Grosser   ppcg_options *createPPCGOptions() {
85e938517eSTobias Grosser     auto DebugOptions =
86e938517eSTobias Grosser         (ppcg_debug_options *)malloc(sizeof(ppcg_debug_options));
87e938517eSTobias Grosser     auto Options = (ppcg_options *)malloc(sizeof(ppcg_options));
88e938517eSTobias Grosser 
89e938517eSTobias Grosser     DebugOptions->dump_schedule_constraints = false;
90e938517eSTobias Grosser     DebugOptions->dump_schedule = false;
91e938517eSTobias Grosser     DebugOptions->dump_final_schedule = false;
92e938517eSTobias Grosser     DebugOptions->dump_sizes = false;
93e938517eSTobias Grosser 
94e938517eSTobias Grosser     Options->debug = DebugOptions;
95e938517eSTobias Grosser 
96e938517eSTobias Grosser     Options->reschedule = true;
97e938517eSTobias Grosser     Options->scale_tile_loops = false;
98e938517eSTobias Grosser     Options->wrap = false;
99e938517eSTobias Grosser 
100e938517eSTobias Grosser     Options->non_negative_parameters = false;
101e938517eSTobias Grosser     Options->ctx = nullptr;
102e938517eSTobias Grosser     Options->sizes = nullptr;
103e938517eSTobias Grosser 
1044eaedde5STobias Grosser     Options->tile_size = 32;
1054eaedde5STobias Grosser 
106e938517eSTobias Grosser     Options->use_private_memory = false;
107e938517eSTobias Grosser     Options->use_shared_memory = false;
108e938517eSTobias Grosser     Options->max_shared_memory = 0;
109e938517eSTobias Grosser 
110e938517eSTobias Grosser     Options->target = PPCG_TARGET_CUDA;
111e938517eSTobias Grosser     Options->openmp = false;
112e938517eSTobias Grosser     Options->linearize_device_arrays = true;
113e938517eSTobias Grosser     Options->live_range_reordering = false;
114e938517eSTobias Grosser 
115e938517eSTobias Grosser     Options->opencl_compiler_options = nullptr;
116e938517eSTobias Grosser     Options->opencl_use_gpu = false;
117e938517eSTobias Grosser     Options->opencl_n_include_file = 0;
118e938517eSTobias Grosser     Options->opencl_include_files = nullptr;
119e938517eSTobias Grosser     Options->opencl_print_kernel_types = false;
120e938517eSTobias Grosser     Options->opencl_embed_kernel_code = false;
121e938517eSTobias Grosser 
122e938517eSTobias Grosser     Options->save_schedule_file = nullptr;
123e938517eSTobias Grosser     Options->load_schedule_file = nullptr;
124e938517eSTobias Grosser 
125e938517eSTobias Grosser     return Options;
126e938517eSTobias Grosser   }
127e938517eSTobias Grosser 
128f384594dSTobias Grosser   /// Get a tagged access relation containing all accesses of type @p AccessTy.
129f384594dSTobias Grosser   ///
130f384594dSTobias Grosser   /// Instead of a normal access of the form:
131f384594dSTobias Grosser   ///
132f384594dSTobias Grosser   ///   Stmt[i,j,k] -> Array[f_0(i,j,k), f_1(i,j,k)]
133f384594dSTobias Grosser   ///
134f384594dSTobias Grosser   /// a tagged access has the form
135f384594dSTobias Grosser   ///
136f384594dSTobias Grosser   ///   [Stmt[i,j,k] -> id[]] -> Array[f_0(i,j,k), f_1(i,j,k)]
137f384594dSTobias Grosser   ///
138f384594dSTobias Grosser   /// where 'id' is an additional space that references the memory access that
139f384594dSTobias Grosser   /// triggered the access.
140f384594dSTobias Grosser   ///
141f384594dSTobias Grosser   /// @param AccessTy The type of the memory accesses to collect.
142f384594dSTobias Grosser   ///
143f384594dSTobias Grosser   /// @return The relation describing all tagged memory accesses.
144f384594dSTobias Grosser   isl_union_map *getTaggedAccesses(enum MemoryAccess::AccessType AccessTy) {
145f384594dSTobias Grosser     isl_union_map *Accesses = isl_union_map_empty(S->getParamSpace());
146f384594dSTobias Grosser 
147f384594dSTobias Grosser     for (auto &Stmt : *S)
148f384594dSTobias Grosser       for (auto &Acc : Stmt)
149f384594dSTobias Grosser         if (Acc->getType() == AccessTy) {
150f384594dSTobias Grosser           isl_map *Relation = Acc->getAccessRelation();
151f384594dSTobias Grosser           Relation = isl_map_intersect_domain(Relation, Stmt.getDomain());
152f384594dSTobias Grosser 
153f384594dSTobias Grosser           isl_space *Space = isl_map_get_space(Relation);
154f384594dSTobias Grosser           Space = isl_space_range(Space);
155f384594dSTobias Grosser           Space = isl_space_from_range(Space);
156f384594dSTobias Grosser           isl_map *Universe = isl_map_universe(Space);
157f384594dSTobias Grosser           Relation = isl_map_domain_product(Relation, Universe);
158f384594dSTobias Grosser           Accesses = isl_union_map_add_map(Accesses, Relation);
159f384594dSTobias Grosser         }
160f384594dSTobias Grosser 
161f384594dSTobias Grosser     return Accesses;
162f384594dSTobias Grosser   }
163f384594dSTobias Grosser 
164f384594dSTobias Grosser   /// Get the set of all read accesses, tagged with the access id.
165f384594dSTobias Grosser   ///
166f384594dSTobias Grosser   /// @see getTaggedAccesses
167f384594dSTobias Grosser   isl_union_map *getTaggedReads() {
168f384594dSTobias Grosser     return getTaggedAccesses(MemoryAccess::READ);
169f384594dSTobias Grosser   }
170f384594dSTobias Grosser 
171f384594dSTobias Grosser   /// Get the set of all may (and must) accesses, tagged with the access id.
172f384594dSTobias Grosser   ///
173f384594dSTobias Grosser   /// @see getTaggedAccesses
174f384594dSTobias Grosser   isl_union_map *getTaggedMayWrites() {
175f384594dSTobias Grosser     return isl_union_map_union(getTaggedAccesses(MemoryAccess::MAY_WRITE),
176f384594dSTobias Grosser                                getTaggedAccesses(MemoryAccess::MUST_WRITE));
177f384594dSTobias Grosser   }
178f384594dSTobias Grosser 
179f384594dSTobias Grosser   /// Get the set of all must accesses, tagged with the access id.
180f384594dSTobias Grosser   ///
181f384594dSTobias Grosser   /// @see getTaggedAccesses
182f384594dSTobias Grosser   isl_union_map *getTaggedMustWrites() {
183f384594dSTobias Grosser     return getTaggedAccesses(MemoryAccess::MUST_WRITE);
184f384594dSTobias Grosser   }
185f384594dSTobias Grosser 
186aef5196fSTobias Grosser   /// Collect parameter and array names as isl_ids.
187aef5196fSTobias Grosser   ///
188aef5196fSTobias Grosser   /// To reason about the different parameters and arrays used, ppcg requires
189aef5196fSTobias Grosser   /// a list of all isl_ids in use. As PPCG traditionally performs
190aef5196fSTobias Grosser   /// source-to-source compilation each of these isl_ids is mapped to the
191aef5196fSTobias Grosser   /// expression that represents it. As we do not have a corresponding
192aef5196fSTobias Grosser   /// expression in Polly, we just map each id to a 'zero' expression to match
193aef5196fSTobias Grosser   /// the data format that ppcg expects.
194aef5196fSTobias Grosser   ///
195aef5196fSTobias Grosser   /// @returns Retun a map from collected ids to 'zero' ast expressions.
196aef5196fSTobias Grosser   __isl_give isl_id_to_ast_expr *getNames() {
197aef5196fSTobias Grosser     auto *Names = isl_id_to_ast_expr_alloc(
198bd81a7eeSTobias Grosser         S->getIslCtx(),
199bd81a7eeSTobias Grosser         S->getNumParams() + std::distance(S->array_begin(), S->array_end()));
200aef5196fSTobias Grosser     auto *Zero = isl_ast_expr_from_val(isl_val_zero(S->getIslCtx()));
201aef5196fSTobias Grosser     auto *Space = S->getParamSpace();
202aef5196fSTobias Grosser 
203aef5196fSTobias Grosser     for (int I = 0, E = S->getNumParams(); I < E; ++I) {
204aef5196fSTobias Grosser       isl_id *Id = isl_space_get_dim_id(Space, isl_dim_param, I);
205aef5196fSTobias Grosser       Names = isl_id_to_ast_expr_set(Names, Id, isl_ast_expr_copy(Zero));
206aef5196fSTobias Grosser     }
207aef5196fSTobias Grosser 
208aef5196fSTobias Grosser     for (auto &Array : S->arrays()) {
209aef5196fSTobias Grosser       auto Id = Array.second->getBasePtrId();
210aef5196fSTobias Grosser       Names = isl_id_to_ast_expr_set(Names, Id, isl_ast_expr_copy(Zero));
211aef5196fSTobias Grosser     }
212aef5196fSTobias Grosser 
213aef5196fSTobias Grosser     isl_space_free(Space);
214aef5196fSTobias Grosser     isl_ast_expr_free(Zero);
215aef5196fSTobias Grosser 
216aef5196fSTobias Grosser     return Names;
217aef5196fSTobias Grosser   }
218aef5196fSTobias Grosser 
219e938517eSTobias Grosser   /// Create a new PPCG scop from the current scop.
220e938517eSTobias Grosser   ///
221f384594dSTobias Grosser   /// The PPCG scop is initialized with data from the current polly::Scop. From
222f384594dSTobias Grosser   /// this initial data, the data-dependences in the PPCG scop are initialized.
223f384594dSTobias Grosser   /// We do not use Polly's dependence analysis for now, to ensure we match
224f384594dSTobias Grosser   /// the PPCG default behaviour more closely.
225e938517eSTobias Grosser   ///
226e938517eSTobias Grosser   /// @returns A new ppcg scop.
227e938517eSTobias Grosser   ppcg_scop *createPPCGScop() {
228e938517eSTobias Grosser     auto PPCGScop = (ppcg_scop *)malloc(sizeof(ppcg_scop));
229e938517eSTobias Grosser 
230e938517eSTobias Grosser     PPCGScop->options = createPPCGOptions();
231e938517eSTobias Grosser 
232e938517eSTobias Grosser     PPCGScop->start = 0;
233e938517eSTobias Grosser     PPCGScop->end = 0;
234e938517eSTobias Grosser 
235f384594dSTobias Grosser     PPCGScop->context = S->getContext();
236f384594dSTobias Grosser     PPCGScop->domain = S->getDomains();
237e938517eSTobias Grosser     PPCGScop->call = nullptr;
238f384594dSTobias Grosser     PPCGScop->tagged_reads = getTaggedReads();
239f384594dSTobias Grosser     PPCGScop->reads = S->getReads();
240e938517eSTobias Grosser     PPCGScop->live_in = nullptr;
241f384594dSTobias Grosser     PPCGScop->tagged_may_writes = getTaggedMayWrites();
242f384594dSTobias Grosser     PPCGScop->may_writes = S->getWrites();
243f384594dSTobias Grosser     PPCGScop->tagged_must_writes = getTaggedMustWrites();
244f384594dSTobias Grosser     PPCGScop->must_writes = S->getMustWrites();
245e938517eSTobias Grosser     PPCGScop->live_out = nullptr;
246f384594dSTobias Grosser     PPCGScop->tagged_must_kills = isl_union_map_empty(S->getParamSpace());
247e938517eSTobias Grosser     PPCGScop->tagger = nullptr;
248e938517eSTobias Grosser 
249e938517eSTobias Grosser     PPCGScop->independence = nullptr;
250e938517eSTobias Grosser     PPCGScop->dep_flow = nullptr;
251e938517eSTobias Grosser     PPCGScop->tagged_dep_flow = nullptr;
252e938517eSTobias Grosser     PPCGScop->dep_false = nullptr;
253e938517eSTobias Grosser     PPCGScop->dep_forced = nullptr;
254e938517eSTobias Grosser     PPCGScop->dep_order = nullptr;
255e938517eSTobias Grosser     PPCGScop->tagged_dep_order = nullptr;
256e938517eSTobias Grosser 
257f384594dSTobias Grosser     PPCGScop->schedule = S->getScheduleTree();
258aef5196fSTobias Grosser     PPCGScop->names = getNames();
259e938517eSTobias Grosser 
260e938517eSTobias Grosser     PPCGScop->pet = nullptr;
261e938517eSTobias Grosser 
262f384594dSTobias Grosser     compute_tagger(PPCGScop);
263f384594dSTobias Grosser     compute_dependences(PPCGScop);
264f384594dSTobias Grosser 
265e938517eSTobias Grosser     return PPCGScop;
266e938517eSTobias Grosser   }
267e938517eSTobias Grosser 
26860f63b49STobias Grosser   /// Collect the array acesses in a statement.
26960f63b49STobias Grosser   ///
27060f63b49STobias Grosser   /// @param Stmt The statement for which to collect the accesses.
27160f63b49STobias Grosser   ///
27260f63b49STobias Grosser   /// @returns A list of array accesses.
27360f63b49STobias Grosser   gpu_stmt_access *getStmtAccesses(ScopStmt &Stmt) {
27460f63b49STobias Grosser     gpu_stmt_access *Accesses = nullptr;
27560f63b49STobias Grosser 
27660f63b49STobias Grosser     for (MemoryAccess *Acc : Stmt) {
27760f63b49STobias Grosser       auto Access = isl_alloc_type(S->getIslCtx(), struct gpu_stmt_access);
27860f63b49STobias Grosser       Access->read = Acc->isRead();
27960f63b49STobias Grosser       Access->write = Acc->isWrite();
28060f63b49STobias Grosser       Access->access = Acc->getAccessRelation();
28160f63b49STobias Grosser       isl_space *Space = isl_map_get_space(Access->access);
28260f63b49STobias Grosser       Space = isl_space_range(Space);
28360f63b49STobias Grosser       Space = isl_space_from_range(Space);
28460f63b49STobias Grosser       isl_map *Universe = isl_map_universe(Space);
28560f63b49STobias Grosser       Access->tagged_access =
28660f63b49STobias Grosser           isl_map_domain_product(Acc->getAccessRelation(), Universe);
28760f63b49STobias Grosser       Access->exact_write = Acc->isWrite();
28860f63b49STobias Grosser       Access->ref_id = Acc->getId();
28960f63b49STobias Grosser       Access->next = Accesses;
29060f63b49STobias Grosser       Accesses = Access;
29160f63b49STobias Grosser     }
29260f63b49STobias Grosser 
29360f63b49STobias Grosser     return Accesses;
29460f63b49STobias Grosser   }
29560f63b49STobias Grosser 
29669b46751STobias Grosser   /// Collect the list of GPU statements.
29769b46751STobias Grosser   ///
29869b46751STobias Grosser   /// Each statement has an id, a pointer to the underlying data structure,
29969b46751STobias Grosser   /// as well as a list with all memory accesses.
30069b46751STobias Grosser   ///
30169b46751STobias Grosser   /// TODO: Initialize the list of memory accesses.
30269b46751STobias Grosser   ///
30369b46751STobias Grosser   /// @returns A linked-list of statements.
30469b46751STobias Grosser   gpu_stmt *getStatements() {
30569b46751STobias Grosser     gpu_stmt *Stmts = isl_calloc_array(S->getIslCtx(), struct gpu_stmt,
30669b46751STobias Grosser                                        std::distance(S->begin(), S->end()));
30769b46751STobias Grosser 
30869b46751STobias Grosser     int i = 0;
30969b46751STobias Grosser     for (auto &Stmt : *S) {
31069b46751STobias Grosser       gpu_stmt *GPUStmt = &Stmts[i];
31169b46751STobias Grosser 
31269b46751STobias Grosser       GPUStmt->id = Stmt.getDomainId();
31369b46751STobias Grosser 
31469b46751STobias Grosser       // We use the pet stmt pointer to keep track of the Polly statements.
31569b46751STobias Grosser       GPUStmt->stmt = (pet_stmt *)&Stmt;
31660f63b49STobias Grosser       GPUStmt->accesses = getStmtAccesses(Stmt);
31769b46751STobias Grosser       i++;
31869b46751STobias Grosser     }
31969b46751STobias Grosser 
32069b46751STobias Grosser     return Stmts;
32169b46751STobias Grosser   }
32269b46751STobias Grosser 
32360f63b49STobias Grosser   /// Derive the extent of an array.
32460f63b49STobias Grosser   ///
32560f63b49STobias Grosser   /// The extent of an array is defined by the set of memory locations for
32660f63b49STobias Grosser   /// which a memory access in the iteration domain exists.
32760f63b49STobias Grosser   ///
32860f63b49STobias Grosser   /// @param Array The array to derive the extent for.
32960f63b49STobias Grosser   ///
33060f63b49STobias Grosser   /// @returns An isl_set describing the extent of the array.
33160f63b49STobias Grosser   __isl_give isl_set *getExtent(ScopArrayInfo *Array) {
33260f63b49STobias Grosser     isl_union_map *Accesses = S->getAccesses();
33360f63b49STobias Grosser     Accesses = isl_union_map_intersect_domain(Accesses, S->getDomains());
33460f63b49STobias Grosser     isl_union_set *AccessUSet = isl_union_map_range(Accesses);
33560f63b49STobias Grosser     isl_set *AccessSet =
33660f63b49STobias Grosser         isl_union_set_extract_set(AccessUSet, Array->getSpace());
33760f63b49STobias Grosser     isl_union_set_free(AccessUSet);
33860f63b49STobias Grosser 
33960f63b49STobias Grosser     return AccessSet;
34060f63b49STobias Grosser   }
34160f63b49STobias Grosser 
34260f63b49STobias Grosser   /// Derive the bounds of an array.
34360f63b49STobias Grosser   ///
34460f63b49STobias Grosser   /// For the first dimension we derive the bound of the array from the extent
34560f63b49STobias Grosser   /// of this dimension. For inner dimensions we obtain their size directly from
34660f63b49STobias Grosser   /// ScopArrayInfo.
34760f63b49STobias Grosser   ///
34860f63b49STobias Grosser   /// @param PPCGArray The array to compute bounds for.
34960f63b49STobias Grosser   /// @param Array The polly array from which to take the information.
35060f63b49STobias Grosser   void setArrayBounds(gpu_array_info &PPCGArray, ScopArrayInfo *Array) {
35160f63b49STobias Grosser     if (PPCGArray.n_index > 0) {
35260f63b49STobias Grosser       isl_set *Dom = isl_set_copy(PPCGArray.extent);
35360f63b49STobias Grosser       Dom = isl_set_project_out(Dom, isl_dim_set, 1, PPCGArray.n_index - 1);
35460f63b49STobias Grosser       isl_pw_aff *Bound = isl_set_dim_max(isl_set_copy(Dom), 0);
35560f63b49STobias Grosser       isl_set_free(Dom);
35660f63b49STobias Grosser       Dom = isl_pw_aff_domain(isl_pw_aff_copy(Bound));
35760f63b49STobias Grosser       isl_local_space *LS = isl_local_space_from_space(isl_set_get_space(Dom));
35860f63b49STobias Grosser       isl_aff *One = isl_aff_zero_on_domain(LS);
35960f63b49STobias Grosser       One = isl_aff_add_constant_si(One, 1);
36060f63b49STobias Grosser       Bound = isl_pw_aff_add(Bound, isl_pw_aff_alloc(Dom, One));
36160f63b49STobias Grosser       Bound = isl_pw_aff_gist(Bound, S->getContext());
36260f63b49STobias Grosser       PPCGArray.bound[0] = Bound;
36360f63b49STobias Grosser     }
36460f63b49STobias Grosser 
36560f63b49STobias Grosser     for (unsigned i = 1; i < PPCGArray.n_index; ++i) {
36660f63b49STobias Grosser       isl_pw_aff *Bound = Array->getDimensionSizePw(i);
36760f63b49STobias Grosser       auto LS = isl_pw_aff_get_domain_space(Bound);
36860f63b49STobias Grosser       auto Aff = isl_multi_aff_zero(LS);
36960f63b49STobias Grosser       Bound = isl_pw_aff_pullback_multi_aff(Bound, Aff);
37060f63b49STobias Grosser       PPCGArray.bound[i] = Bound;
37160f63b49STobias Grosser     }
37260f63b49STobias Grosser   }
37360f63b49STobias Grosser 
37460f63b49STobias Grosser   /// Create the arrays for @p PPCGProg.
37560f63b49STobias Grosser   ///
37660f63b49STobias Grosser   /// @param PPCGProg The program to compute the arrays for.
37760f63b49STobias Grosser   void createArrays(gpu_prog *PPCGProg) {
37860f63b49STobias Grosser     int i = 0;
37960f63b49STobias Grosser     for (auto &Element : S->arrays()) {
38060f63b49STobias Grosser       ScopArrayInfo *Array = Element.second.get();
38160f63b49STobias Grosser 
38260f63b49STobias Grosser       std::string TypeName;
38360f63b49STobias Grosser       raw_string_ostream OS(TypeName);
38460f63b49STobias Grosser 
38560f63b49STobias Grosser       OS << *Array->getElementType();
38660f63b49STobias Grosser       TypeName = OS.str();
38760f63b49STobias Grosser 
38860f63b49STobias Grosser       gpu_array_info &PPCGArray = PPCGProg->array[i];
38960f63b49STobias Grosser 
39060f63b49STobias Grosser       PPCGArray.space = Array->getSpace();
39160f63b49STobias Grosser       PPCGArray.type = strdup(TypeName.c_str());
39260f63b49STobias Grosser       PPCGArray.size = Array->getElementType()->getPrimitiveSizeInBits() / 8;
39360f63b49STobias Grosser       PPCGArray.name = strdup(Array->getName().c_str());
39460f63b49STobias Grosser       PPCGArray.extent = nullptr;
39560f63b49STobias Grosser       PPCGArray.n_index = Array->getNumberOfDimensions();
39660f63b49STobias Grosser       PPCGArray.bound =
39760f63b49STobias Grosser           isl_alloc_array(S->getIslCtx(), isl_pw_aff *, PPCGArray.n_index);
39860f63b49STobias Grosser       PPCGArray.extent = getExtent(Array);
39960f63b49STobias Grosser       PPCGArray.n_ref = 0;
40060f63b49STobias Grosser       PPCGArray.refs = nullptr;
40160f63b49STobias Grosser       PPCGArray.accessed = true;
40260f63b49STobias Grosser       PPCGArray.read_only_scalar = false;
40360f63b49STobias Grosser       PPCGArray.has_compound_element = false;
40460f63b49STobias Grosser       PPCGArray.local = false;
40560f63b49STobias Grosser       PPCGArray.declare_local = false;
40660f63b49STobias Grosser       PPCGArray.global = false;
40760f63b49STobias Grosser       PPCGArray.linearize = false;
40860f63b49STobias Grosser       PPCGArray.dep_order = nullptr;
40960f63b49STobias Grosser 
41060f63b49STobias Grosser       setArrayBounds(PPCGArray, Array);
411*2d010dafSTobias Grosser       i++;
41260f63b49STobias Grosser     }
41360f63b49STobias Grosser   }
41460f63b49STobias Grosser 
41560f63b49STobias Grosser   /// Create an identity map between the arrays in the scop.
41660f63b49STobias Grosser   ///
41760f63b49STobias Grosser   /// @returns An identity map between the arrays in the scop.
41860f63b49STobias Grosser   isl_union_map *getArrayIdentity() {
41960f63b49STobias Grosser     isl_union_map *Maps = isl_union_map_empty(S->getParamSpace());
42060f63b49STobias Grosser 
42160f63b49STobias Grosser     for (auto &Item : S->arrays()) {
42260f63b49STobias Grosser       ScopArrayInfo *Array = Item.second.get();
42360f63b49STobias Grosser       isl_space *Space = Array->getSpace();
42460f63b49STobias Grosser       Space = isl_space_map_from_set(Space);
42560f63b49STobias Grosser       isl_map *Identity = isl_map_identity(Space);
42660f63b49STobias Grosser       Maps = isl_union_map_add_map(Maps, Identity);
42760f63b49STobias Grosser     }
42860f63b49STobias Grosser 
42960f63b49STobias Grosser     return Maps;
43060f63b49STobias Grosser   }
43160f63b49STobias Grosser 
432e938517eSTobias Grosser   /// Create a default-initialized PPCG GPU program.
433e938517eSTobias Grosser   ///
434e938517eSTobias Grosser   /// @returns A new gpu grogram description.
435e938517eSTobias Grosser   gpu_prog *createPPCGProg(ppcg_scop *PPCGScop) {
436e938517eSTobias Grosser 
437e938517eSTobias Grosser     if (!PPCGScop)
438e938517eSTobias Grosser       return nullptr;
439e938517eSTobias Grosser 
440e938517eSTobias Grosser     auto PPCGProg = isl_calloc_type(S->getIslCtx(), struct gpu_prog);
441e938517eSTobias Grosser 
442e938517eSTobias Grosser     PPCGProg->ctx = S->getIslCtx();
443e938517eSTobias Grosser     PPCGProg->scop = PPCGScop;
444aef5196fSTobias Grosser     PPCGProg->context = isl_set_copy(PPCGScop->context);
44560f63b49STobias Grosser     PPCGProg->read = isl_union_map_copy(PPCGScop->reads);
44660f63b49STobias Grosser     PPCGProg->may_write = isl_union_map_copy(PPCGScop->may_writes);
44760f63b49STobias Grosser     PPCGProg->must_write = isl_union_map_copy(PPCGScop->must_writes);
44860f63b49STobias Grosser     PPCGProg->tagged_must_kill =
44960f63b49STobias Grosser         isl_union_map_copy(PPCGScop->tagged_must_kills);
45060f63b49STobias Grosser     PPCGProg->to_inner = getArrayIdentity();
45160f63b49STobias Grosser     PPCGProg->to_outer = getArrayIdentity();
45260f63b49STobias Grosser     PPCGProg->may_persist = compute_may_persist(PPCGProg);
453e938517eSTobias Grosser     PPCGProg->any_to_outer = nullptr;
454e938517eSTobias Grosser     PPCGProg->array_order = nullptr;
45569b46751STobias Grosser     PPCGProg->n_stmts = std::distance(S->begin(), S->end());
45669b46751STobias Grosser     PPCGProg->stmts = getStatements();
45760f63b49STobias Grosser     PPCGProg->n_array = std::distance(S->array_begin(), S->array_end());
45860f63b49STobias Grosser     PPCGProg->array = isl_calloc_array(S->getIslCtx(), struct gpu_array_info,
45960f63b49STobias Grosser                                        PPCGProg->n_array);
46060f63b49STobias Grosser 
46160f63b49STobias Grosser     createArrays(PPCGProg);
462e938517eSTobias Grosser 
463e938517eSTobias Grosser     return PPCGProg;
464e938517eSTobias Grosser   }
465e938517eSTobias Grosser 
46669b46751STobias Grosser   struct PrintGPUUserData {
46769b46751STobias Grosser     struct cuda_info *CudaInfo;
46869b46751STobias Grosser     struct gpu_prog *PPCGProg;
46969b46751STobias Grosser     std::vector<ppcg_kernel *> Kernels;
47069b46751STobias Grosser   };
47169b46751STobias Grosser 
47269b46751STobias Grosser   /// Print a user statement node in the host code.
47369b46751STobias Grosser   ///
47469b46751STobias Grosser   /// We use ppcg's printing facilities to print the actual statement and
47569b46751STobias Grosser   /// additionally build up a list of all kernels that are encountered in the
47669b46751STobias Grosser   /// host ast.
47769b46751STobias Grosser   ///
47869b46751STobias Grosser   /// @param P The printer to print to
47969b46751STobias Grosser   /// @param Options The printing options to use
48069b46751STobias Grosser   /// @param Node The node to print
48169b46751STobias Grosser   /// @param User A user pointer to carry additional data. This pointer is
48269b46751STobias Grosser   ///             expected to be of type PrintGPUUserData.
48369b46751STobias Grosser   ///
48469b46751STobias Grosser   /// @returns A printer to which the output has been printed.
48569b46751STobias Grosser   static __isl_give isl_printer *
48669b46751STobias Grosser   printHostUser(__isl_take isl_printer *P,
48769b46751STobias Grosser                 __isl_take isl_ast_print_options *Options,
48869b46751STobias Grosser                 __isl_take isl_ast_node *Node, void *User) {
48969b46751STobias Grosser     auto Data = (struct PrintGPUUserData *)User;
49069b46751STobias Grosser     auto Id = isl_ast_node_get_annotation(Node);
49169b46751STobias Grosser 
49269b46751STobias Grosser     if (Id) {
49369b46751STobias Grosser       auto Kernel = (struct ppcg_kernel *)isl_id_get_user(Id);
49469b46751STobias Grosser       isl_id_free(Id);
49569b46751STobias Grosser       Data->Kernels.push_back(Kernel);
49669b46751STobias Grosser     }
49769b46751STobias Grosser 
49869b46751STobias Grosser     return print_host_user(P, Options, Node, User);
49969b46751STobias Grosser   }
50069b46751STobias Grosser 
50169b46751STobias Grosser   /// Print C code corresponding to the control flow in @p Kernel.
50269b46751STobias Grosser   ///
50369b46751STobias Grosser   /// @param Kernel The kernel to print
50469b46751STobias Grosser   void printKernel(ppcg_kernel *Kernel) {
50569b46751STobias Grosser     auto *P = isl_printer_to_str(S->getIslCtx());
50669b46751STobias Grosser     P = isl_printer_set_output_format(P, ISL_FORMAT_C);
50769b46751STobias Grosser     auto *Options = isl_ast_print_options_alloc(S->getIslCtx());
50869b46751STobias Grosser     P = isl_ast_node_print(Kernel->tree, P, Options);
50969b46751STobias Grosser     char *String = isl_printer_get_str(P);
51069b46751STobias Grosser     printf("%s\n", String);
51169b46751STobias Grosser     free(String);
51269b46751STobias Grosser     isl_printer_free(P);
51369b46751STobias Grosser   }
51469b46751STobias Grosser 
51569b46751STobias Grosser   /// Print C code corresponding to the GPU code described by @p Tree.
51669b46751STobias Grosser   ///
51769b46751STobias Grosser   /// @param Tree An AST describing GPU code
51869b46751STobias Grosser   /// @param PPCGProg The PPCG program from which @Tree has been constructed.
51969b46751STobias Grosser   void printGPUTree(isl_ast_node *Tree, gpu_prog *PPCGProg) {
52069b46751STobias Grosser     auto *P = isl_printer_to_str(S->getIslCtx());
52169b46751STobias Grosser     P = isl_printer_set_output_format(P, ISL_FORMAT_C);
52269b46751STobias Grosser 
52369b46751STobias Grosser     PrintGPUUserData Data;
52469b46751STobias Grosser     Data.PPCGProg = PPCGProg;
52569b46751STobias Grosser 
52669b46751STobias Grosser     auto *Options = isl_ast_print_options_alloc(S->getIslCtx());
52769b46751STobias Grosser     Options =
52869b46751STobias Grosser         isl_ast_print_options_set_print_user(Options, printHostUser, &Data);
52969b46751STobias Grosser     P = isl_ast_node_print(Tree, P, Options);
53069b46751STobias Grosser     char *String = isl_printer_get_str(P);
53169b46751STobias Grosser     printf("# host\n");
53269b46751STobias Grosser     printf("%s\n", String);
53369b46751STobias Grosser     free(String);
53469b46751STobias Grosser     isl_printer_free(P);
53569b46751STobias Grosser 
53669b46751STobias Grosser     for (auto Kernel : Data.Kernels) {
53769b46751STobias Grosser       printf("# kernel%d\n", Kernel->id);
53869b46751STobias Grosser       printKernel(Kernel);
53969b46751STobias Grosser     }
54069b46751STobias Grosser   }
54169b46751STobias Grosser 
542f384594dSTobias Grosser   // Generate a GPU program using PPCG.
543f384594dSTobias Grosser   //
544f384594dSTobias Grosser   // GPU mapping consists of multiple steps:
545f384594dSTobias Grosser   //
546f384594dSTobias Grosser   //  1) Compute new schedule for the program.
547f384594dSTobias Grosser   //  2) Map schedule to GPU (TODO)
548f384594dSTobias Grosser   //  3) Generate code for new schedule (TODO)
549f384594dSTobias Grosser   //
550f384594dSTobias Grosser   // We do not use here the Polly ScheduleOptimizer, as the schedule optimizer
551f384594dSTobias Grosser   // is mostly CPU specific. Instead, we use PPCG's GPU code generation
552f384594dSTobias Grosser   // strategy directly from this pass.
553f384594dSTobias Grosser   gpu_gen *generateGPU(ppcg_scop *PPCGScop, gpu_prog *PPCGProg) {
554f384594dSTobias Grosser 
555f384594dSTobias Grosser     auto PPCGGen = isl_calloc_type(S->getIslCtx(), struct gpu_gen);
556f384594dSTobias Grosser 
557f384594dSTobias Grosser     PPCGGen->ctx = S->getIslCtx();
558f384594dSTobias Grosser     PPCGGen->options = PPCGScop->options;
559f384594dSTobias Grosser     PPCGGen->print = nullptr;
560f384594dSTobias Grosser     PPCGGen->print_user = nullptr;
56160c60025STobias Grosser     PPCGGen->build_ast_expr = &pollyBuildAstExprForStmt;
562f384594dSTobias Grosser     PPCGGen->prog = PPCGProg;
563f384594dSTobias Grosser     PPCGGen->tree = nullptr;
564f384594dSTobias Grosser     PPCGGen->types.n = 0;
565f384594dSTobias Grosser     PPCGGen->types.name = nullptr;
566f384594dSTobias Grosser     PPCGGen->sizes = nullptr;
567f384594dSTobias Grosser     PPCGGen->used_sizes = nullptr;
568f384594dSTobias Grosser     PPCGGen->kernel_id = 0;
569f384594dSTobias Grosser 
570f384594dSTobias Grosser     // Set scheduling strategy to same strategy PPCG is using.
571f384594dSTobias Grosser     isl_options_set_schedule_outer_coincidence(PPCGGen->ctx, true);
572f384594dSTobias Grosser     isl_options_set_schedule_maximize_band_depth(PPCGGen->ctx, true);
573f384594dSTobias Grosser 
574f384594dSTobias Grosser     isl_schedule *Schedule = get_schedule(PPCGGen);
575f384594dSTobias Grosser 
576aef5196fSTobias Grosser     int has_permutable = has_any_permutable_node(Schedule);
577aef5196fSTobias Grosser 
57869b46751STobias Grosser     if (!has_permutable || has_permutable < 0) {
579aef5196fSTobias Grosser       Schedule = isl_schedule_free(Schedule);
58069b46751STobias Grosser     } else {
581aef5196fSTobias Grosser       Schedule = map_to_device(PPCGGen, Schedule);
58269b46751STobias Grosser       PPCGGen->tree = generate_code(PPCGGen, isl_schedule_copy(Schedule));
58369b46751STobias Grosser     }
584aef5196fSTobias Grosser 
585f384594dSTobias Grosser     if (DumpSchedule) {
586f384594dSTobias Grosser       isl_printer *P = isl_printer_to_str(S->getIslCtx());
587f384594dSTobias Grosser       P = isl_printer_set_yaml_style(P, ISL_YAML_STYLE_BLOCK);
588f384594dSTobias Grosser       P = isl_printer_print_str(P, "Schedule\n");
589f384594dSTobias Grosser       P = isl_printer_print_str(P, "========\n");
590f384594dSTobias Grosser       if (Schedule)
591f384594dSTobias Grosser         P = isl_printer_print_schedule(P, Schedule);
592f384594dSTobias Grosser       else
593f384594dSTobias Grosser         P = isl_printer_print_str(P, "No schedule found\n");
594f384594dSTobias Grosser 
595f384594dSTobias Grosser       printf("%s\n", isl_printer_get_str(P));
596f384594dSTobias Grosser       isl_printer_free(P);
597f384594dSTobias Grosser     }
598f384594dSTobias Grosser 
59969b46751STobias Grosser     if (DumpCode) {
60069b46751STobias Grosser       printf("Code\n");
60169b46751STobias Grosser       printf("====\n");
60269b46751STobias Grosser       if (PPCGGen->tree)
60369b46751STobias Grosser         printGPUTree(PPCGGen->tree, PPCGProg);
60469b46751STobias Grosser       else
60569b46751STobias Grosser         printf("No code generated\n");
60669b46751STobias Grosser     }
60769b46751STobias Grosser 
608f384594dSTobias Grosser     isl_schedule_free(Schedule);
609f384594dSTobias Grosser 
610f384594dSTobias Grosser     return PPCGGen;
611f384594dSTobias Grosser   }
612f384594dSTobias Grosser 
613f384594dSTobias Grosser   /// Free gpu_gen structure.
614f384594dSTobias Grosser   ///
615f384594dSTobias Grosser   /// @param PPCGGen The ppcg_gen object to free.
616f384594dSTobias Grosser   void freePPCGGen(gpu_gen *PPCGGen) {
617f384594dSTobias Grosser     isl_ast_node_free(PPCGGen->tree);
618f384594dSTobias Grosser     isl_union_map_free(PPCGGen->sizes);
619f384594dSTobias Grosser     isl_union_map_free(PPCGGen->used_sizes);
620f384594dSTobias Grosser     free(PPCGGen);
621f384594dSTobias Grosser   }
622f384594dSTobias Grosser 
623b307ed4dSTobias Grosser   /// Free the options in the ppcg scop structure.
624b307ed4dSTobias Grosser   ///
625b307ed4dSTobias Grosser   /// ppcg is not freeing these options for us. To avoid leaks we do this
626b307ed4dSTobias Grosser   /// ourselves.
627b307ed4dSTobias Grosser   ///
628b307ed4dSTobias Grosser   /// @param PPCGScop The scop referencing the options to free.
629b307ed4dSTobias Grosser   void freeOptions(ppcg_scop *PPCGScop) {
630b307ed4dSTobias Grosser     free(PPCGScop->options->debug);
631b307ed4dSTobias Grosser     PPCGScop->options->debug = nullptr;
632b307ed4dSTobias Grosser     free(PPCGScop->options);
633b307ed4dSTobias Grosser     PPCGScop->options = nullptr;
634b307ed4dSTobias Grosser   }
635b307ed4dSTobias Grosser 
636e938517eSTobias Grosser   bool runOnScop(Scop &CurrentScop) override {
637e938517eSTobias Grosser     S = &CurrentScop;
638e938517eSTobias Grosser 
639e938517eSTobias Grosser     auto PPCGScop = createPPCGScop();
640e938517eSTobias Grosser     auto PPCGProg = createPPCGProg(PPCGScop);
641f384594dSTobias Grosser     auto PPCGGen = generateGPU(PPCGScop, PPCGProg);
642b307ed4dSTobias Grosser     freeOptions(PPCGScop);
643f384594dSTobias Grosser     freePPCGGen(PPCGGen);
644e938517eSTobias Grosser     gpu_prog_free(PPCGProg);
645e938517eSTobias Grosser     ppcg_scop_free(PPCGScop);
646e938517eSTobias Grosser 
647e938517eSTobias Grosser     return true;
648e938517eSTobias Grosser   }
6499dfe4e7cSTobias Grosser 
6509dfe4e7cSTobias Grosser   void printScop(raw_ostream &, Scop &) const override {}
6519dfe4e7cSTobias Grosser 
6529dfe4e7cSTobias Grosser   void getAnalysisUsage(AnalysisUsage &AU) const override {
6539dfe4e7cSTobias Grosser     AU.addRequired<DominatorTreeWrapperPass>();
6549dfe4e7cSTobias Grosser     AU.addRequired<RegionInfoPass>();
6559dfe4e7cSTobias Grosser     AU.addRequired<ScalarEvolutionWrapperPass>();
6569dfe4e7cSTobias Grosser     AU.addRequired<ScopDetection>();
6579dfe4e7cSTobias Grosser     AU.addRequired<ScopInfoRegionPass>();
6589dfe4e7cSTobias Grosser     AU.addRequired<LoopInfoWrapperPass>();
6599dfe4e7cSTobias Grosser 
6609dfe4e7cSTobias Grosser     AU.addPreserved<AAResultsWrapperPass>();
6619dfe4e7cSTobias Grosser     AU.addPreserved<BasicAAWrapperPass>();
6629dfe4e7cSTobias Grosser     AU.addPreserved<LoopInfoWrapperPass>();
6639dfe4e7cSTobias Grosser     AU.addPreserved<DominatorTreeWrapperPass>();
6649dfe4e7cSTobias Grosser     AU.addPreserved<GlobalsAAWrapperPass>();
6659dfe4e7cSTobias Grosser     AU.addPreserved<PostDominatorTreeWrapperPass>();
6669dfe4e7cSTobias Grosser     AU.addPreserved<ScopDetection>();
6679dfe4e7cSTobias Grosser     AU.addPreserved<ScalarEvolutionWrapperPass>();
6689dfe4e7cSTobias Grosser     AU.addPreserved<SCEVAAWrapperPass>();
6699dfe4e7cSTobias Grosser 
6709dfe4e7cSTobias Grosser     // FIXME: We do not yet add regions for the newly generated code to the
6719dfe4e7cSTobias Grosser     //        region tree.
6729dfe4e7cSTobias Grosser     AU.addPreserved<RegionInfoPass>();
6739dfe4e7cSTobias Grosser     AU.addPreserved<ScopInfoRegionPass>();
6749dfe4e7cSTobias Grosser   }
6759dfe4e7cSTobias Grosser };
6769dfe4e7cSTobias Grosser }
6779dfe4e7cSTobias Grosser 
6789dfe4e7cSTobias Grosser char PPCGCodeGeneration::ID = 1;
6799dfe4e7cSTobias Grosser 
6809dfe4e7cSTobias Grosser Pass *polly::createPPCGCodeGenerationPass() { return new PPCGCodeGeneration(); }
6819dfe4e7cSTobias Grosser 
6829dfe4e7cSTobias Grosser INITIALIZE_PASS_BEGIN(PPCGCodeGeneration, "polly-codegen-ppcg",
6839dfe4e7cSTobias Grosser                       "Polly - Apply PPCG translation to SCOP", false, false)
6849dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(DependenceInfo);
6859dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass);
6869dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass);
6879dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(RegionInfoPass);
6889dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass);
6899dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(ScopDetection);
6909dfe4e7cSTobias Grosser INITIALIZE_PASS_END(PPCGCodeGeneration, "polly-codegen-ppcg",
6919dfe4e7cSTobias Grosser                     "Polly - Apply PPCG translation to SCOP", false, false)
692