19dfe4e7cSTobias Grosser //===------ PPCGCodeGeneration.cpp - Polly Accelerator Code Generation. ---===// 29dfe4e7cSTobias Grosser // 39dfe4e7cSTobias Grosser // The LLVM Compiler Infrastructure 49dfe4e7cSTobias Grosser // 59dfe4e7cSTobias Grosser // This file is distributed under the University of Illinois Open Source 69dfe4e7cSTobias Grosser // License. See LICENSE.TXT for details. 79dfe4e7cSTobias Grosser // 89dfe4e7cSTobias Grosser //===----------------------------------------------------------------------===// 99dfe4e7cSTobias Grosser // 109dfe4e7cSTobias Grosser // Take a scop created by ScopInfo and map it to GPU code using the ppcg 119dfe4e7cSTobias Grosser // GPU mapping strategy. 129dfe4e7cSTobias Grosser // 139dfe4e7cSTobias Grosser //===----------------------------------------------------------------------===// 149dfe4e7cSTobias Grosser 159dfe4e7cSTobias Grosser #include "polly/CodeGen/IslNodeBuilder.h" 169dfe4e7cSTobias Grosser #include "polly/DependenceInfo.h" 179dfe4e7cSTobias Grosser #include "polly/LinkAllPasses.h" 18f384594dSTobias Grosser #include "polly/Options.h" 199dfe4e7cSTobias Grosser #include "polly/ScopInfo.h" 209dfe4e7cSTobias Grosser #include "llvm/Analysis/AliasAnalysis.h" 219dfe4e7cSTobias Grosser #include "llvm/Analysis/BasicAliasAnalysis.h" 229dfe4e7cSTobias Grosser #include "llvm/Analysis/GlobalsModRef.h" 239dfe4e7cSTobias Grosser #include "llvm/Analysis/PostDominators.h" 249dfe4e7cSTobias Grosser #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" 259dfe4e7cSTobias Grosser 26f384594dSTobias Grosser #include "isl/union_map.h" 27f384594dSTobias Grosser 28e938517eSTobias Grosser extern "C" { 29a56f8f8eSTobias Grosser #include "ppcg/cuda.h" 30a56f8f8eSTobias Grosser #include "ppcg/gpu.h" 31a56f8f8eSTobias Grosser #include "ppcg/gpu_print.h" 32a56f8f8eSTobias Grosser #include "ppcg/ppcg.h" 33a56f8f8eSTobias Grosser #include "ppcg/schedule.h" 34e938517eSTobias Grosser } 35e938517eSTobias Grosser 369dfe4e7cSTobias Grosser #include "llvm/Support/Debug.h" 379dfe4e7cSTobias Grosser 389dfe4e7cSTobias Grosser using namespace polly; 399dfe4e7cSTobias Grosser using namespace llvm; 409dfe4e7cSTobias Grosser 419dfe4e7cSTobias Grosser #define DEBUG_TYPE "polly-codegen-ppcg" 429dfe4e7cSTobias Grosser 43f384594dSTobias Grosser static cl::opt<bool> DumpSchedule("polly-acc-dump-schedule", 44f384594dSTobias Grosser cl::desc("Dump the computed GPU Schedule"), 45681bd568STobias Grosser cl::Hidden, cl::init(false), cl::ZeroOrMore, 46f384594dSTobias Grosser cl::cat(PollyCategory)); 4769b46751STobias Grosser 4869b46751STobias Grosser static cl::opt<bool> 4969b46751STobias Grosser DumpCode("polly-acc-dump-code", 5069b46751STobias Grosser cl::desc("Dump C code describing the GPU mapping"), cl::Hidden, 5169b46751STobias Grosser cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory)); 5269b46751STobias Grosser 5360c60025STobias Grosser /// Create the ast expressions for a ScopStmt. 5460c60025STobias Grosser /// 5560c60025STobias Grosser /// This function is a callback for to generate the ast expressions for each 5660c60025STobias Grosser /// of the scheduled ScopStmts. 5760c60025STobias Grosser static __isl_give isl_id_to_ast_expr *pollyBuildAstExprForStmt( 5860c60025STobias Grosser void *Stmt, isl_ast_build *Build, 5960c60025STobias Grosser isl_multi_pw_aff *(*FunctionIndex)(__isl_take isl_multi_pw_aff *MPA, 6060c60025STobias Grosser isl_id *Id, void *User), 6160c60025STobias Grosser void *UserIndex, 6260c60025STobias Grosser isl_ast_expr *(*FunctionExpr)(isl_ast_expr *Expr, isl_id *Id, void *User), 6360c60025STobias Grosser void *User_expr) { 6460c60025STobias Grosser 6560c60025STobias Grosser // TODO: Implement the AST expression generation. For now we just return a 6660c60025STobias Grosser // nullptr to ensure that we do not free uninitialized pointers. 6760c60025STobias Grosser 6860c60025STobias Grosser return nullptr; 6960c60025STobias Grosser } 70f384594dSTobias Grosser 719dfe4e7cSTobias Grosser namespace { 729dfe4e7cSTobias Grosser class PPCGCodeGeneration : public ScopPass { 739dfe4e7cSTobias Grosser public: 749dfe4e7cSTobias Grosser static char ID; 759dfe4e7cSTobias Grosser 76e938517eSTobias Grosser /// The scop that is currently processed. 77e938517eSTobias Grosser Scop *S; 78e938517eSTobias Grosser 799dfe4e7cSTobias Grosser PPCGCodeGeneration() : ScopPass(ID) {} 809dfe4e7cSTobias Grosser 81e938517eSTobias Grosser /// Construct compilation options for PPCG. 82e938517eSTobias Grosser /// 83e938517eSTobias Grosser /// @returns The compilation options. 84e938517eSTobias Grosser ppcg_options *createPPCGOptions() { 85e938517eSTobias Grosser auto DebugOptions = 86e938517eSTobias Grosser (ppcg_debug_options *)malloc(sizeof(ppcg_debug_options)); 87e938517eSTobias Grosser auto Options = (ppcg_options *)malloc(sizeof(ppcg_options)); 88e938517eSTobias Grosser 89e938517eSTobias Grosser DebugOptions->dump_schedule_constraints = false; 90e938517eSTobias Grosser DebugOptions->dump_schedule = false; 91e938517eSTobias Grosser DebugOptions->dump_final_schedule = false; 92e938517eSTobias Grosser DebugOptions->dump_sizes = false; 93e938517eSTobias Grosser 94e938517eSTobias Grosser Options->debug = DebugOptions; 95e938517eSTobias Grosser 96e938517eSTobias Grosser Options->reschedule = true; 97e938517eSTobias Grosser Options->scale_tile_loops = false; 98e938517eSTobias Grosser Options->wrap = false; 99e938517eSTobias Grosser 100e938517eSTobias Grosser Options->non_negative_parameters = false; 101e938517eSTobias Grosser Options->ctx = nullptr; 102e938517eSTobias Grosser Options->sizes = nullptr; 103e938517eSTobias Grosser 1044eaedde5STobias Grosser Options->tile_size = 32; 1054eaedde5STobias Grosser 106e938517eSTobias Grosser Options->use_private_memory = false; 107e938517eSTobias Grosser Options->use_shared_memory = false; 108e938517eSTobias Grosser Options->max_shared_memory = 0; 109e938517eSTobias Grosser 110e938517eSTobias Grosser Options->target = PPCG_TARGET_CUDA; 111e938517eSTobias Grosser Options->openmp = false; 112e938517eSTobias Grosser Options->linearize_device_arrays = true; 113e938517eSTobias Grosser Options->live_range_reordering = false; 114e938517eSTobias Grosser 115e938517eSTobias Grosser Options->opencl_compiler_options = nullptr; 116e938517eSTobias Grosser Options->opencl_use_gpu = false; 117e938517eSTobias Grosser Options->opencl_n_include_file = 0; 118e938517eSTobias Grosser Options->opencl_include_files = nullptr; 119e938517eSTobias Grosser Options->opencl_print_kernel_types = false; 120e938517eSTobias Grosser Options->opencl_embed_kernel_code = false; 121e938517eSTobias Grosser 122e938517eSTobias Grosser Options->save_schedule_file = nullptr; 123e938517eSTobias Grosser Options->load_schedule_file = nullptr; 124e938517eSTobias Grosser 125e938517eSTobias Grosser return Options; 126e938517eSTobias Grosser } 127e938517eSTobias Grosser 128f384594dSTobias Grosser /// Get a tagged access relation containing all accesses of type @p AccessTy. 129f384594dSTobias Grosser /// 130f384594dSTobias Grosser /// Instead of a normal access of the form: 131f384594dSTobias Grosser /// 132f384594dSTobias Grosser /// Stmt[i,j,k] -> Array[f_0(i,j,k), f_1(i,j,k)] 133f384594dSTobias Grosser /// 134f384594dSTobias Grosser /// a tagged access has the form 135f384594dSTobias Grosser /// 136f384594dSTobias Grosser /// [Stmt[i,j,k] -> id[]] -> Array[f_0(i,j,k), f_1(i,j,k)] 137f384594dSTobias Grosser /// 138f384594dSTobias Grosser /// where 'id' is an additional space that references the memory access that 139f384594dSTobias Grosser /// triggered the access. 140f384594dSTobias Grosser /// 141f384594dSTobias Grosser /// @param AccessTy The type of the memory accesses to collect. 142f384594dSTobias Grosser /// 143f384594dSTobias Grosser /// @return The relation describing all tagged memory accesses. 144f384594dSTobias Grosser isl_union_map *getTaggedAccesses(enum MemoryAccess::AccessType AccessTy) { 145f384594dSTobias Grosser isl_union_map *Accesses = isl_union_map_empty(S->getParamSpace()); 146f384594dSTobias Grosser 147f384594dSTobias Grosser for (auto &Stmt : *S) 148f384594dSTobias Grosser for (auto &Acc : Stmt) 149f384594dSTobias Grosser if (Acc->getType() == AccessTy) { 150f384594dSTobias Grosser isl_map *Relation = Acc->getAccessRelation(); 151f384594dSTobias Grosser Relation = isl_map_intersect_domain(Relation, Stmt.getDomain()); 152f384594dSTobias Grosser 153f384594dSTobias Grosser isl_space *Space = isl_map_get_space(Relation); 154f384594dSTobias Grosser Space = isl_space_range(Space); 155f384594dSTobias Grosser Space = isl_space_from_range(Space); 156f384594dSTobias Grosser isl_map *Universe = isl_map_universe(Space); 157f384594dSTobias Grosser Relation = isl_map_domain_product(Relation, Universe); 158f384594dSTobias Grosser Accesses = isl_union_map_add_map(Accesses, Relation); 159f384594dSTobias Grosser } 160f384594dSTobias Grosser 161f384594dSTobias Grosser return Accesses; 162f384594dSTobias Grosser } 163f384594dSTobias Grosser 164f384594dSTobias Grosser /// Get the set of all read accesses, tagged with the access id. 165f384594dSTobias Grosser /// 166f384594dSTobias Grosser /// @see getTaggedAccesses 167f384594dSTobias Grosser isl_union_map *getTaggedReads() { 168f384594dSTobias Grosser return getTaggedAccesses(MemoryAccess::READ); 169f384594dSTobias Grosser } 170f384594dSTobias Grosser 171f384594dSTobias Grosser /// Get the set of all may (and must) accesses, tagged with the access id. 172f384594dSTobias Grosser /// 173f384594dSTobias Grosser /// @see getTaggedAccesses 174f384594dSTobias Grosser isl_union_map *getTaggedMayWrites() { 175f384594dSTobias Grosser return isl_union_map_union(getTaggedAccesses(MemoryAccess::MAY_WRITE), 176f384594dSTobias Grosser getTaggedAccesses(MemoryAccess::MUST_WRITE)); 177f384594dSTobias Grosser } 178f384594dSTobias Grosser 179f384594dSTobias Grosser /// Get the set of all must accesses, tagged with the access id. 180f384594dSTobias Grosser /// 181f384594dSTobias Grosser /// @see getTaggedAccesses 182f384594dSTobias Grosser isl_union_map *getTaggedMustWrites() { 183f384594dSTobias Grosser return getTaggedAccesses(MemoryAccess::MUST_WRITE); 184f384594dSTobias Grosser } 185f384594dSTobias Grosser 186aef5196fSTobias Grosser /// Collect parameter and array names as isl_ids. 187aef5196fSTobias Grosser /// 188aef5196fSTobias Grosser /// To reason about the different parameters and arrays used, ppcg requires 189aef5196fSTobias Grosser /// a list of all isl_ids in use. As PPCG traditionally performs 190aef5196fSTobias Grosser /// source-to-source compilation each of these isl_ids is mapped to the 191aef5196fSTobias Grosser /// expression that represents it. As we do not have a corresponding 192aef5196fSTobias Grosser /// expression in Polly, we just map each id to a 'zero' expression to match 193aef5196fSTobias Grosser /// the data format that ppcg expects. 194aef5196fSTobias Grosser /// 195aef5196fSTobias Grosser /// @returns Retun a map from collected ids to 'zero' ast expressions. 196aef5196fSTobias Grosser __isl_give isl_id_to_ast_expr *getNames() { 197aef5196fSTobias Grosser auto *Names = isl_id_to_ast_expr_alloc( 198bd81a7eeSTobias Grosser S->getIslCtx(), 199bd81a7eeSTobias Grosser S->getNumParams() + std::distance(S->array_begin(), S->array_end())); 200aef5196fSTobias Grosser auto *Zero = isl_ast_expr_from_val(isl_val_zero(S->getIslCtx())); 201aef5196fSTobias Grosser auto *Space = S->getParamSpace(); 202aef5196fSTobias Grosser 203aef5196fSTobias Grosser for (int I = 0, E = S->getNumParams(); I < E; ++I) { 204aef5196fSTobias Grosser isl_id *Id = isl_space_get_dim_id(Space, isl_dim_param, I); 205aef5196fSTobias Grosser Names = isl_id_to_ast_expr_set(Names, Id, isl_ast_expr_copy(Zero)); 206aef5196fSTobias Grosser } 207aef5196fSTobias Grosser 208aef5196fSTobias Grosser for (auto &Array : S->arrays()) { 209aef5196fSTobias Grosser auto Id = Array.second->getBasePtrId(); 210aef5196fSTobias Grosser Names = isl_id_to_ast_expr_set(Names, Id, isl_ast_expr_copy(Zero)); 211aef5196fSTobias Grosser } 212aef5196fSTobias Grosser 213aef5196fSTobias Grosser isl_space_free(Space); 214aef5196fSTobias Grosser isl_ast_expr_free(Zero); 215aef5196fSTobias Grosser 216aef5196fSTobias Grosser return Names; 217aef5196fSTobias Grosser } 218aef5196fSTobias Grosser 219e938517eSTobias Grosser /// Create a new PPCG scop from the current scop. 220e938517eSTobias Grosser /// 221f384594dSTobias Grosser /// The PPCG scop is initialized with data from the current polly::Scop. From 222f384594dSTobias Grosser /// this initial data, the data-dependences in the PPCG scop are initialized. 223f384594dSTobias Grosser /// We do not use Polly's dependence analysis for now, to ensure we match 224f384594dSTobias Grosser /// the PPCG default behaviour more closely. 225e938517eSTobias Grosser /// 226e938517eSTobias Grosser /// @returns A new ppcg scop. 227e938517eSTobias Grosser ppcg_scop *createPPCGScop() { 228e938517eSTobias Grosser auto PPCGScop = (ppcg_scop *)malloc(sizeof(ppcg_scop)); 229e938517eSTobias Grosser 230e938517eSTobias Grosser PPCGScop->options = createPPCGOptions(); 231e938517eSTobias Grosser 232e938517eSTobias Grosser PPCGScop->start = 0; 233e938517eSTobias Grosser PPCGScop->end = 0; 234e938517eSTobias Grosser 235f384594dSTobias Grosser PPCGScop->context = S->getContext(); 236f384594dSTobias Grosser PPCGScop->domain = S->getDomains(); 237e938517eSTobias Grosser PPCGScop->call = nullptr; 238f384594dSTobias Grosser PPCGScop->tagged_reads = getTaggedReads(); 239f384594dSTobias Grosser PPCGScop->reads = S->getReads(); 240e938517eSTobias Grosser PPCGScop->live_in = nullptr; 241f384594dSTobias Grosser PPCGScop->tagged_may_writes = getTaggedMayWrites(); 242f384594dSTobias Grosser PPCGScop->may_writes = S->getWrites(); 243f384594dSTobias Grosser PPCGScop->tagged_must_writes = getTaggedMustWrites(); 244f384594dSTobias Grosser PPCGScop->must_writes = S->getMustWrites(); 245e938517eSTobias Grosser PPCGScop->live_out = nullptr; 246f384594dSTobias Grosser PPCGScop->tagged_must_kills = isl_union_map_empty(S->getParamSpace()); 247e938517eSTobias Grosser PPCGScop->tagger = nullptr; 248e938517eSTobias Grosser 249e938517eSTobias Grosser PPCGScop->independence = nullptr; 250e938517eSTobias Grosser PPCGScop->dep_flow = nullptr; 251e938517eSTobias Grosser PPCGScop->tagged_dep_flow = nullptr; 252e938517eSTobias Grosser PPCGScop->dep_false = nullptr; 253e938517eSTobias Grosser PPCGScop->dep_forced = nullptr; 254e938517eSTobias Grosser PPCGScop->dep_order = nullptr; 255e938517eSTobias Grosser PPCGScop->tagged_dep_order = nullptr; 256e938517eSTobias Grosser 257f384594dSTobias Grosser PPCGScop->schedule = S->getScheduleTree(); 258aef5196fSTobias Grosser PPCGScop->names = getNames(); 259e938517eSTobias Grosser 260e938517eSTobias Grosser PPCGScop->pet = nullptr; 261e938517eSTobias Grosser 262f384594dSTobias Grosser compute_tagger(PPCGScop); 263f384594dSTobias Grosser compute_dependences(PPCGScop); 264f384594dSTobias Grosser 265e938517eSTobias Grosser return PPCGScop; 266e938517eSTobias Grosser } 267e938517eSTobias Grosser 26860f63b49STobias Grosser /// Collect the array acesses in a statement. 26960f63b49STobias Grosser /// 27060f63b49STobias Grosser /// @param Stmt The statement for which to collect the accesses. 27160f63b49STobias Grosser /// 27260f63b49STobias Grosser /// @returns A list of array accesses. 27360f63b49STobias Grosser gpu_stmt_access *getStmtAccesses(ScopStmt &Stmt) { 27460f63b49STobias Grosser gpu_stmt_access *Accesses = nullptr; 27560f63b49STobias Grosser 27660f63b49STobias Grosser for (MemoryAccess *Acc : Stmt) { 27760f63b49STobias Grosser auto Access = isl_alloc_type(S->getIslCtx(), struct gpu_stmt_access); 27860f63b49STobias Grosser Access->read = Acc->isRead(); 27960f63b49STobias Grosser Access->write = Acc->isWrite(); 28060f63b49STobias Grosser Access->access = Acc->getAccessRelation(); 28160f63b49STobias Grosser isl_space *Space = isl_map_get_space(Access->access); 28260f63b49STobias Grosser Space = isl_space_range(Space); 28360f63b49STobias Grosser Space = isl_space_from_range(Space); 28460f63b49STobias Grosser isl_map *Universe = isl_map_universe(Space); 28560f63b49STobias Grosser Access->tagged_access = 28660f63b49STobias Grosser isl_map_domain_product(Acc->getAccessRelation(), Universe); 28760f63b49STobias Grosser Access->exact_write = Acc->isWrite(); 28860f63b49STobias Grosser Access->ref_id = Acc->getId(); 28960f63b49STobias Grosser Access->next = Accesses; 29060f63b49STobias Grosser Accesses = Access; 29160f63b49STobias Grosser } 29260f63b49STobias Grosser 29360f63b49STobias Grosser return Accesses; 29460f63b49STobias Grosser } 29560f63b49STobias Grosser 29669b46751STobias Grosser /// Collect the list of GPU statements. 29769b46751STobias Grosser /// 29869b46751STobias Grosser /// Each statement has an id, a pointer to the underlying data structure, 29969b46751STobias Grosser /// as well as a list with all memory accesses. 30069b46751STobias Grosser /// 30169b46751STobias Grosser /// TODO: Initialize the list of memory accesses. 30269b46751STobias Grosser /// 30369b46751STobias Grosser /// @returns A linked-list of statements. 30469b46751STobias Grosser gpu_stmt *getStatements() { 30569b46751STobias Grosser gpu_stmt *Stmts = isl_calloc_array(S->getIslCtx(), struct gpu_stmt, 30669b46751STobias Grosser std::distance(S->begin(), S->end())); 30769b46751STobias Grosser 30869b46751STobias Grosser int i = 0; 30969b46751STobias Grosser for (auto &Stmt : *S) { 31069b46751STobias Grosser gpu_stmt *GPUStmt = &Stmts[i]; 31169b46751STobias Grosser 31269b46751STobias Grosser GPUStmt->id = Stmt.getDomainId(); 31369b46751STobias Grosser 31469b46751STobias Grosser // We use the pet stmt pointer to keep track of the Polly statements. 31569b46751STobias Grosser GPUStmt->stmt = (pet_stmt *)&Stmt; 31660f63b49STobias Grosser GPUStmt->accesses = getStmtAccesses(Stmt); 31769b46751STobias Grosser i++; 31869b46751STobias Grosser } 31969b46751STobias Grosser 32069b46751STobias Grosser return Stmts; 32169b46751STobias Grosser } 32269b46751STobias Grosser 32360f63b49STobias Grosser /// Derive the extent of an array. 32460f63b49STobias Grosser /// 32560f63b49STobias Grosser /// The extent of an array is defined by the set of memory locations for 32660f63b49STobias Grosser /// which a memory access in the iteration domain exists. 32760f63b49STobias Grosser /// 32860f63b49STobias Grosser /// @param Array The array to derive the extent for. 32960f63b49STobias Grosser /// 33060f63b49STobias Grosser /// @returns An isl_set describing the extent of the array. 33160f63b49STobias Grosser __isl_give isl_set *getExtent(ScopArrayInfo *Array) { 33260f63b49STobias Grosser isl_union_map *Accesses = S->getAccesses(); 33360f63b49STobias Grosser Accesses = isl_union_map_intersect_domain(Accesses, S->getDomains()); 33460f63b49STobias Grosser isl_union_set *AccessUSet = isl_union_map_range(Accesses); 33560f63b49STobias Grosser isl_set *AccessSet = 33660f63b49STobias Grosser isl_union_set_extract_set(AccessUSet, Array->getSpace()); 33760f63b49STobias Grosser isl_union_set_free(AccessUSet); 33860f63b49STobias Grosser 33960f63b49STobias Grosser return AccessSet; 34060f63b49STobias Grosser } 34160f63b49STobias Grosser 34260f63b49STobias Grosser /// Derive the bounds of an array. 34360f63b49STobias Grosser /// 34460f63b49STobias Grosser /// For the first dimension we derive the bound of the array from the extent 34560f63b49STobias Grosser /// of this dimension. For inner dimensions we obtain their size directly from 34660f63b49STobias Grosser /// ScopArrayInfo. 34760f63b49STobias Grosser /// 34860f63b49STobias Grosser /// @param PPCGArray The array to compute bounds for. 34960f63b49STobias Grosser /// @param Array The polly array from which to take the information. 35060f63b49STobias Grosser void setArrayBounds(gpu_array_info &PPCGArray, ScopArrayInfo *Array) { 35160f63b49STobias Grosser if (PPCGArray.n_index > 0) { 35260f63b49STobias Grosser isl_set *Dom = isl_set_copy(PPCGArray.extent); 35360f63b49STobias Grosser Dom = isl_set_project_out(Dom, isl_dim_set, 1, PPCGArray.n_index - 1); 35460f63b49STobias Grosser isl_pw_aff *Bound = isl_set_dim_max(isl_set_copy(Dom), 0); 35560f63b49STobias Grosser isl_set_free(Dom); 35660f63b49STobias Grosser Dom = isl_pw_aff_domain(isl_pw_aff_copy(Bound)); 35760f63b49STobias Grosser isl_local_space *LS = isl_local_space_from_space(isl_set_get_space(Dom)); 35860f63b49STobias Grosser isl_aff *One = isl_aff_zero_on_domain(LS); 35960f63b49STobias Grosser One = isl_aff_add_constant_si(One, 1); 36060f63b49STobias Grosser Bound = isl_pw_aff_add(Bound, isl_pw_aff_alloc(Dom, One)); 36160f63b49STobias Grosser Bound = isl_pw_aff_gist(Bound, S->getContext()); 36260f63b49STobias Grosser PPCGArray.bound[0] = Bound; 36360f63b49STobias Grosser } 36460f63b49STobias Grosser 36560f63b49STobias Grosser for (unsigned i = 1; i < PPCGArray.n_index; ++i) { 36660f63b49STobias Grosser isl_pw_aff *Bound = Array->getDimensionSizePw(i); 36760f63b49STobias Grosser auto LS = isl_pw_aff_get_domain_space(Bound); 36860f63b49STobias Grosser auto Aff = isl_multi_aff_zero(LS); 36960f63b49STobias Grosser Bound = isl_pw_aff_pullback_multi_aff(Bound, Aff); 37060f63b49STobias Grosser PPCGArray.bound[i] = Bound; 37160f63b49STobias Grosser } 37260f63b49STobias Grosser } 37360f63b49STobias Grosser 37460f63b49STobias Grosser /// Create the arrays for @p PPCGProg. 37560f63b49STobias Grosser /// 37660f63b49STobias Grosser /// @param PPCGProg The program to compute the arrays for. 37760f63b49STobias Grosser void createArrays(gpu_prog *PPCGProg) { 37860f63b49STobias Grosser int i = 0; 37960f63b49STobias Grosser for (auto &Element : S->arrays()) { 38060f63b49STobias Grosser ScopArrayInfo *Array = Element.second.get(); 38160f63b49STobias Grosser 38260f63b49STobias Grosser std::string TypeName; 38360f63b49STobias Grosser raw_string_ostream OS(TypeName); 38460f63b49STobias Grosser 38560f63b49STobias Grosser OS << *Array->getElementType(); 38660f63b49STobias Grosser TypeName = OS.str(); 38760f63b49STobias Grosser 38860f63b49STobias Grosser gpu_array_info &PPCGArray = PPCGProg->array[i]; 38960f63b49STobias Grosser 39060f63b49STobias Grosser PPCGArray.space = Array->getSpace(); 39160f63b49STobias Grosser PPCGArray.type = strdup(TypeName.c_str()); 39260f63b49STobias Grosser PPCGArray.size = Array->getElementType()->getPrimitiveSizeInBits() / 8; 39360f63b49STobias Grosser PPCGArray.name = strdup(Array->getName().c_str()); 39460f63b49STobias Grosser PPCGArray.extent = nullptr; 39560f63b49STobias Grosser PPCGArray.n_index = Array->getNumberOfDimensions(); 39660f63b49STobias Grosser PPCGArray.bound = 39760f63b49STobias Grosser isl_alloc_array(S->getIslCtx(), isl_pw_aff *, PPCGArray.n_index); 39860f63b49STobias Grosser PPCGArray.extent = getExtent(Array); 39960f63b49STobias Grosser PPCGArray.n_ref = 0; 40060f63b49STobias Grosser PPCGArray.refs = nullptr; 40160f63b49STobias Grosser PPCGArray.accessed = true; 40260f63b49STobias Grosser PPCGArray.read_only_scalar = false; 40360f63b49STobias Grosser PPCGArray.has_compound_element = false; 40460f63b49STobias Grosser PPCGArray.local = false; 40560f63b49STobias Grosser PPCGArray.declare_local = false; 40660f63b49STobias Grosser PPCGArray.global = false; 40760f63b49STobias Grosser PPCGArray.linearize = false; 40860f63b49STobias Grosser PPCGArray.dep_order = nullptr; 40960f63b49STobias Grosser 41060f63b49STobias Grosser setArrayBounds(PPCGArray, Array); 411*2d010dafSTobias Grosser i++; 41260f63b49STobias Grosser } 41360f63b49STobias Grosser } 41460f63b49STobias Grosser 41560f63b49STobias Grosser /// Create an identity map between the arrays in the scop. 41660f63b49STobias Grosser /// 41760f63b49STobias Grosser /// @returns An identity map between the arrays in the scop. 41860f63b49STobias Grosser isl_union_map *getArrayIdentity() { 41960f63b49STobias Grosser isl_union_map *Maps = isl_union_map_empty(S->getParamSpace()); 42060f63b49STobias Grosser 42160f63b49STobias Grosser for (auto &Item : S->arrays()) { 42260f63b49STobias Grosser ScopArrayInfo *Array = Item.second.get(); 42360f63b49STobias Grosser isl_space *Space = Array->getSpace(); 42460f63b49STobias Grosser Space = isl_space_map_from_set(Space); 42560f63b49STobias Grosser isl_map *Identity = isl_map_identity(Space); 42660f63b49STobias Grosser Maps = isl_union_map_add_map(Maps, Identity); 42760f63b49STobias Grosser } 42860f63b49STobias Grosser 42960f63b49STobias Grosser return Maps; 43060f63b49STobias Grosser } 43160f63b49STobias Grosser 432e938517eSTobias Grosser /// Create a default-initialized PPCG GPU program. 433e938517eSTobias Grosser /// 434e938517eSTobias Grosser /// @returns A new gpu grogram description. 435e938517eSTobias Grosser gpu_prog *createPPCGProg(ppcg_scop *PPCGScop) { 436e938517eSTobias Grosser 437e938517eSTobias Grosser if (!PPCGScop) 438e938517eSTobias Grosser return nullptr; 439e938517eSTobias Grosser 440e938517eSTobias Grosser auto PPCGProg = isl_calloc_type(S->getIslCtx(), struct gpu_prog); 441e938517eSTobias Grosser 442e938517eSTobias Grosser PPCGProg->ctx = S->getIslCtx(); 443e938517eSTobias Grosser PPCGProg->scop = PPCGScop; 444aef5196fSTobias Grosser PPCGProg->context = isl_set_copy(PPCGScop->context); 44560f63b49STobias Grosser PPCGProg->read = isl_union_map_copy(PPCGScop->reads); 44660f63b49STobias Grosser PPCGProg->may_write = isl_union_map_copy(PPCGScop->may_writes); 44760f63b49STobias Grosser PPCGProg->must_write = isl_union_map_copy(PPCGScop->must_writes); 44860f63b49STobias Grosser PPCGProg->tagged_must_kill = 44960f63b49STobias Grosser isl_union_map_copy(PPCGScop->tagged_must_kills); 45060f63b49STobias Grosser PPCGProg->to_inner = getArrayIdentity(); 45160f63b49STobias Grosser PPCGProg->to_outer = getArrayIdentity(); 45260f63b49STobias Grosser PPCGProg->may_persist = compute_may_persist(PPCGProg); 453e938517eSTobias Grosser PPCGProg->any_to_outer = nullptr; 454e938517eSTobias Grosser PPCGProg->array_order = nullptr; 45569b46751STobias Grosser PPCGProg->n_stmts = std::distance(S->begin(), S->end()); 45669b46751STobias Grosser PPCGProg->stmts = getStatements(); 45760f63b49STobias Grosser PPCGProg->n_array = std::distance(S->array_begin(), S->array_end()); 45860f63b49STobias Grosser PPCGProg->array = isl_calloc_array(S->getIslCtx(), struct gpu_array_info, 45960f63b49STobias Grosser PPCGProg->n_array); 46060f63b49STobias Grosser 46160f63b49STobias Grosser createArrays(PPCGProg); 462e938517eSTobias Grosser 463e938517eSTobias Grosser return PPCGProg; 464e938517eSTobias Grosser } 465e938517eSTobias Grosser 46669b46751STobias Grosser struct PrintGPUUserData { 46769b46751STobias Grosser struct cuda_info *CudaInfo; 46869b46751STobias Grosser struct gpu_prog *PPCGProg; 46969b46751STobias Grosser std::vector<ppcg_kernel *> Kernels; 47069b46751STobias Grosser }; 47169b46751STobias Grosser 47269b46751STobias Grosser /// Print a user statement node in the host code. 47369b46751STobias Grosser /// 47469b46751STobias Grosser /// We use ppcg's printing facilities to print the actual statement and 47569b46751STobias Grosser /// additionally build up a list of all kernels that are encountered in the 47669b46751STobias Grosser /// host ast. 47769b46751STobias Grosser /// 47869b46751STobias Grosser /// @param P The printer to print to 47969b46751STobias Grosser /// @param Options The printing options to use 48069b46751STobias Grosser /// @param Node The node to print 48169b46751STobias Grosser /// @param User A user pointer to carry additional data. This pointer is 48269b46751STobias Grosser /// expected to be of type PrintGPUUserData. 48369b46751STobias Grosser /// 48469b46751STobias Grosser /// @returns A printer to which the output has been printed. 48569b46751STobias Grosser static __isl_give isl_printer * 48669b46751STobias Grosser printHostUser(__isl_take isl_printer *P, 48769b46751STobias Grosser __isl_take isl_ast_print_options *Options, 48869b46751STobias Grosser __isl_take isl_ast_node *Node, void *User) { 48969b46751STobias Grosser auto Data = (struct PrintGPUUserData *)User; 49069b46751STobias Grosser auto Id = isl_ast_node_get_annotation(Node); 49169b46751STobias Grosser 49269b46751STobias Grosser if (Id) { 49369b46751STobias Grosser auto Kernel = (struct ppcg_kernel *)isl_id_get_user(Id); 49469b46751STobias Grosser isl_id_free(Id); 49569b46751STobias Grosser Data->Kernels.push_back(Kernel); 49669b46751STobias Grosser } 49769b46751STobias Grosser 49869b46751STobias Grosser return print_host_user(P, Options, Node, User); 49969b46751STobias Grosser } 50069b46751STobias Grosser 50169b46751STobias Grosser /// Print C code corresponding to the control flow in @p Kernel. 50269b46751STobias Grosser /// 50369b46751STobias Grosser /// @param Kernel The kernel to print 50469b46751STobias Grosser void printKernel(ppcg_kernel *Kernel) { 50569b46751STobias Grosser auto *P = isl_printer_to_str(S->getIslCtx()); 50669b46751STobias Grosser P = isl_printer_set_output_format(P, ISL_FORMAT_C); 50769b46751STobias Grosser auto *Options = isl_ast_print_options_alloc(S->getIslCtx()); 50869b46751STobias Grosser P = isl_ast_node_print(Kernel->tree, P, Options); 50969b46751STobias Grosser char *String = isl_printer_get_str(P); 51069b46751STobias Grosser printf("%s\n", String); 51169b46751STobias Grosser free(String); 51269b46751STobias Grosser isl_printer_free(P); 51369b46751STobias Grosser } 51469b46751STobias Grosser 51569b46751STobias Grosser /// Print C code corresponding to the GPU code described by @p Tree. 51669b46751STobias Grosser /// 51769b46751STobias Grosser /// @param Tree An AST describing GPU code 51869b46751STobias Grosser /// @param PPCGProg The PPCG program from which @Tree has been constructed. 51969b46751STobias Grosser void printGPUTree(isl_ast_node *Tree, gpu_prog *PPCGProg) { 52069b46751STobias Grosser auto *P = isl_printer_to_str(S->getIslCtx()); 52169b46751STobias Grosser P = isl_printer_set_output_format(P, ISL_FORMAT_C); 52269b46751STobias Grosser 52369b46751STobias Grosser PrintGPUUserData Data; 52469b46751STobias Grosser Data.PPCGProg = PPCGProg; 52569b46751STobias Grosser 52669b46751STobias Grosser auto *Options = isl_ast_print_options_alloc(S->getIslCtx()); 52769b46751STobias Grosser Options = 52869b46751STobias Grosser isl_ast_print_options_set_print_user(Options, printHostUser, &Data); 52969b46751STobias Grosser P = isl_ast_node_print(Tree, P, Options); 53069b46751STobias Grosser char *String = isl_printer_get_str(P); 53169b46751STobias Grosser printf("# host\n"); 53269b46751STobias Grosser printf("%s\n", String); 53369b46751STobias Grosser free(String); 53469b46751STobias Grosser isl_printer_free(P); 53569b46751STobias Grosser 53669b46751STobias Grosser for (auto Kernel : Data.Kernels) { 53769b46751STobias Grosser printf("# kernel%d\n", Kernel->id); 53869b46751STobias Grosser printKernel(Kernel); 53969b46751STobias Grosser } 54069b46751STobias Grosser } 54169b46751STobias Grosser 542f384594dSTobias Grosser // Generate a GPU program using PPCG. 543f384594dSTobias Grosser // 544f384594dSTobias Grosser // GPU mapping consists of multiple steps: 545f384594dSTobias Grosser // 546f384594dSTobias Grosser // 1) Compute new schedule for the program. 547f384594dSTobias Grosser // 2) Map schedule to GPU (TODO) 548f384594dSTobias Grosser // 3) Generate code for new schedule (TODO) 549f384594dSTobias Grosser // 550f384594dSTobias Grosser // We do not use here the Polly ScheduleOptimizer, as the schedule optimizer 551f384594dSTobias Grosser // is mostly CPU specific. Instead, we use PPCG's GPU code generation 552f384594dSTobias Grosser // strategy directly from this pass. 553f384594dSTobias Grosser gpu_gen *generateGPU(ppcg_scop *PPCGScop, gpu_prog *PPCGProg) { 554f384594dSTobias Grosser 555f384594dSTobias Grosser auto PPCGGen = isl_calloc_type(S->getIslCtx(), struct gpu_gen); 556f384594dSTobias Grosser 557f384594dSTobias Grosser PPCGGen->ctx = S->getIslCtx(); 558f384594dSTobias Grosser PPCGGen->options = PPCGScop->options; 559f384594dSTobias Grosser PPCGGen->print = nullptr; 560f384594dSTobias Grosser PPCGGen->print_user = nullptr; 56160c60025STobias Grosser PPCGGen->build_ast_expr = &pollyBuildAstExprForStmt; 562f384594dSTobias Grosser PPCGGen->prog = PPCGProg; 563f384594dSTobias Grosser PPCGGen->tree = nullptr; 564f384594dSTobias Grosser PPCGGen->types.n = 0; 565f384594dSTobias Grosser PPCGGen->types.name = nullptr; 566f384594dSTobias Grosser PPCGGen->sizes = nullptr; 567f384594dSTobias Grosser PPCGGen->used_sizes = nullptr; 568f384594dSTobias Grosser PPCGGen->kernel_id = 0; 569f384594dSTobias Grosser 570f384594dSTobias Grosser // Set scheduling strategy to same strategy PPCG is using. 571f384594dSTobias Grosser isl_options_set_schedule_outer_coincidence(PPCGGen->ctx, true); 572f384594dSTobias Grosser isl_options_set_schedule_maximize_band_depth(PPCGGen->ctx, true); 573f384594dSTobias Grosser 574f384594dSTobias Grosser isl_schedule *Schedule = get_schedule(PPCGGen); 575f384594dSTobias Grosser 576aef5196fSTobias Grosser int has_permutable = has_any_permutable_node(Schedule); 577aef5196fSTobias Grosser 57869b46751STobias Grosser if (!has_permutable || has_permutable < 0) { 579aef5196fSTobias Grosser Schedule = isl_schedule_free(Schedule); 58069b46751STobias Grosser } else { 581aef5196fSTobias Grosser Schedule = map_to_device(PPCGGen, Schedule); 58269b46751STobias Grosser PPCGGen->tree = generate_code(PPCGGen, isl_schedule_copy(Schedule)); 58369b46751STobias Grosser } 584aef5196fSTobias Grosser 585f384594dSTobias Grosser if (DumpSchedule) { 586f384594dSTobias Grosser isl_printer *P = isl_printer_to_str(S->getIslCtx()); 587f384594dSTobias Grosser P = isl_printer_set_yaml_style(P, ISL_YAML_STYLE_BLOCK); 588f384594dSTobias Grosser P = isl_printer_print_str(P, "Schedule\n"); 589f384594dSTobias Grosser P = isl_printer_print_str(P, "========\n"); 590f384594dSTobias Grosser if (Schedule) 591f384594dSTobias Grosser P = isl_printer_print_schedule(P, Schedule); 592f384594dSTobias Grosser else 593f384594dSTobias Grosser P = isl_printer_print_str(P, "No schedule found\n"); 594f384594dSTobias Grosser 595f384594dSTobias Grosser printf("%s\n", isl_printer_get_str(P)); 596f384594dSTobias Grosser isl_printer_free(P); 597f384594dSTobias Grosser } 598f384594dSTobias Grosser 59969b46751STobias Grosser if (DumpCode) { 60069b46751STobias Grosser printf("Code\n"); 60169b46751STobias Grosser printf("====\n"); 60269b46751STobias Grosser if (PPCGGen->tree) 60369b46751STobias Grosser printGPUTree(PPCGGen->tree, PPCGProg); 60469b46751STobias Grosser else 60569b46751STobias Grosser printf("No code generated\n"); 60669b46751STobias Grosser } 60769b46751STobias Grosser 608f384594dSTobias Grosser isl_schedule_free(Schedule); 609f384594dSTobias Grosser 610f384594dSTobias Grosser return PPCGGen; 611f384594dSTobias Grosser } 612f384594dSTobias Grosser 613f384594dSTobias Grosser /// Free gpu_gen structure. 614f384594dSTobias Grosser /// 615f384594dSTobias Grosser /// @param PPCGGen The ppcg_gen object to free. 616f384594dSTobias Grosser void freePPCGGen(gpu_gen *PPCGGen) { 617f384594dSTobias Grosser isl_ast_node_free(PPCGGen->tree); 618f384594dSTobias Grosser isl_union_map_free(PPCGGen->sizes); 619f384594dSTobias Grosser isl_union_map_free(PPCGGen->used_sizes); 620f384594dSTobias Grosser free(PPCGGen); 621f384594dSTobias Grosser } 622f384594dSTobias Grosser 623b307ed4dSTobias Grosser /// Free the options in the ppcg scop structure. 624b307ed4dSTobias Grosser /// 625b307ed4dSTobias Grosser /// ppcg is not freeing these options for us. To avoid leaks we do this 626b307ed4dSTobias Grosser /// ourselves. 627b307ed4dSTobias Grosser /// 628b307ed4dSTobias Grosser /// @param PPCGScop The scop referencing the options to free. 629b307ed4dSTobias Grosser void freeOptions(ppcg_scop *PPCGScop) { 630b307ed4dSTobias Grosser free(PPCGScop->options->debug); 631b307ed4dSTobias Grosser PPCGScop->options->debug = nullptr; 632b307ed4dSTobias Grosser free(PPCGScop->options); 633b307ed4dSTobias Grosser PPCGScop->options = nullptr; 634b307ed4dSTobias Grosser } 635b307ed4dSTobias Grosser 636e938517eSTobias Grosser bool runOnScop(Scop &CurrentScop) override { 637e938517eSTobias Grosser S = &CurrentScop; 638e938517eSTobias Grosser 639e938517eSTobias Grosser auto PPCGScop = createPPCGScop(); 640e938517eSTobias Grosser auto PPCGProg = createPPCGProg(PPCGScop); 641f384594dSTobias Grosser auto PPCGGen = generateGPU(PPCGScop, PPCGProg); 642b307ed4dSTobias Grosser freeOptions(PPCGScop); 643f384594dSTobias Grosser freePPCGGen(PPCGGen); 644e938517eSTobias Grosser gpu_prog_free(PPCGProg); 645e938517eSTobias Grosser ppcg_scop_free(PPCGScop); 646e938517eSTobias Grosser 647e938517eSTobias Grosser return true; 648e938517eSTobias Grosser } 6499dfe4e7cSTobias Grosser 6509dfe4e7cSTobias Grosser void printScop(raw_ostream &, Scop &) const override {} 6519dfe4e7cSTobias Grosser 6529dfe4e7cSTobias Grosser void getAnalysisUsage(AnalysisUsage &AU) const override { 6539dfe4e7cSTobias Grosser AU.addRequired<DominatorTreeWrapperPass>(); 6549dfe4e7cSTobias Grosser AU.addRequired<RegionInfoPass>(); 6559dfe4e7cSTobias Grosser AU.addRequired<ScalarEvolutionWrapperPass>(); 6569dfe4e7cSTobias Grosser AU.addRequired<ScopDetection>(); 6579dfe4e7cSTobias Grosser AU.addRequired<ScopInfoRegionPass>(); 6589dfe4e7cSTobias Grosser AU.addRequired<LoopInfoWrapperPass>(); 6599dfe4e7cSTobias Grosser 6609dfe4e7cSTobias Grosser AU.addPreserved<AAResultsWrapperPass>(); 6619dfe4e7cSTobias Grosser AU.addPreserved<BasicAAWrapperPass>(); 6629dfe4e7cSTobias Grosser AU.addPreserved<LoopInfoWrapperPass>(); 6639dfe4e7cSTobias Grosser AU.addPreserved<DominatorTreeWrapperPass>(); 6649dfe4e7cSTobias Grosser AU.addPreserved<GlobalsAAWrapperPass>(); 6659dfe4e7cSTobias Grosser AU.addPreserved<PostDominatorTreeWrapperPass>(); 6669dfe4e7cSTobias Grosser AU.addPreserved<ScopDetection>(); 6679dfe4e7cSTobias Grosser AU.addPreserved<ScalarEvolutionWrapperPass>(); 6689dfe4e7cSTobias Grosser AU.addPreserved<SCEVAAWrapperPass>(); 6699dfe4e7cSTobias Grosser 6709dfe4e7cSTobias Grosser // FIXME: We do not yet add regions for the newly generated code to the 6719dfe4e7cSTobias Grosser // region tree. 6729dfe4e7cSTobias Grosser AU.addPreserved<RegionInfoPass>(); 6739dfe4e7cSTobias Grosser AU.addPreserved<ScopInfoRegionPass>(); 6749dfe4e7cSTobias Grosser } 6759dfe4e7cSTobias Grosser }; 6769dfe4e7cSTobias Grosser } 6779dfe4e7cSTobias Grosser 6789dfe4e7cSTobias Grosser char PPCGCodeGeneration::ID = 1; 6799dfe4e7cSTobias Grosser 6809dfe4e7cSTobias Grosser Pass *polly::createPPCGCodeGenerationPass() { return new PPCGCodeGeneration(); } 6819dfe4e7cSTobias Grosser 6829dfe4e7cSTobias Grosser INITIALIZE_PASS_BEGIN(PPCGCodeGeneration, "polly-codegen-ppcg", 6839dfe4e7cSTobias Grosser "Polly - Apply PPCG translation to SCOP", false, false) 6849dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(DependenceInfo); 6859dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass); 6869dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass); 6879dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(RegionInfoPass); 6889dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass); 6899dfe4e7cSTobias Grosser INITIALIZE_PASS_DEPENDENCY(ScopDetection); 6909dfe4e7cSTobias Grosser INITIALIZE_PASS_END(PPCGCodeGeneration, "polly-codegen-ppcg", 6919dfe4e7cSTobias Grosser "Polly - Apply PPCG translation to SCOP", false, false) 692