Lines Matching refs:kernel
615 static void read_block_sizes(struct ppcg_kernel *kernel, in read_block_sizes() argument
620 if (kernel->n_block > 3) in read_block_sizes()
621 kernel->n_block = 3; in read_block_sizes()
622 switch (kernel->n_block) { in read_block_sizes()
624 kernel->block_dim[0] = 512; in read_block_sizes()
627 kernel->block_dim[0] = 32; in read_block_sizes()
628 kernel->block_dim[1] = 16; in read_block_sizes()
631 kernel->block_dim[0] = 32; in read_block_sizes()
632 kernel->block_dim[1] = 4; in read_block_sizes()
633 kernel->block_dim[2] = 4; in read_block_sizes()
637 size = extract_sizes(sizes, "block", kernel->id); in read_block_sizes()
638 read_sizes_from_set(size, kernel->block_dim, &kernel->n_block); in read_block_sizes()
644 static void read_grid_sizes(struct ppcg_kernel *kernel, in read_grid_sizes() argument
649 if (kernel->n_grid > 2) in read_grid_sizes()
650 kernel->n_grid = 2; in read_grid_sizes()
651 switch (kernel->n_grid) { in read_grid_sizes()
653 kernel->grid_dim[0] = 32768; in read_grid_sizes()
656 kernel->grid_dim[0] = 256; in read_grid_sizes()
657 kernel->grid_dim[1] = 256; in read_grid_sizes()
661 size = extract_sizes(sizes, "grid", kernel->id); in read_grid_sizes()
662 read_sizes_from_set(size, kernel->grid_dim, &kernel->n_grid); in read_grid_sizes()
670 static void read_grid_and_block_sizes(struct ppcg_kernel *kernel, in read_grid_and_block_sizes() argument
673 read_block_sizes(kernel, gen->sizes); in read_grid_and_block_sizes()
674 read_grid_sizes(kernel, gen->sizes); in read_grid_and_block_sizes()
675 set_used_sizes(gen, "block", kernel->id, in read_grid_and_block_sizes()
676 kernel->block_dim, kernel->n_block); in read_grid_and_block_sizes()
677 set_used_sizes(gen, "grid", kernel->id, in read_grid_and_block_sizes()
678 kernel->grid_dim, kernel->n_grid); in read_grid_and_block_sizes()
906 struct ppcg_kernel *kernel, __isl_take isl_pw_multi_aff *iterator_map) in compute_sched_to_copy() argument
915 kernel->copy_schedule_dim); in compute_sched_to_copy()
917 upma = isl_union_pw_multi_aff_copy(kernel->copy_schedule); in compute_sched_to_copy()
937 static void check_shared_memory_bound(struct ppcg_kernel *kernel) in check_shared_memory_bound() argument
942 if (kernel->options->max_shared_memory < 0) in check_shared_memory_bound()
945 left = isl_val_int_from_si(kernel->ctx, in check_shared_memory_bound()
946 kernel->options->max_shared_memory); in check_shared_memory_bound()
948 for (i = 0; i < kernel->n_array; ++i) { in check_shared_memory_bound()
949 struct gpu_local_array_info *local = &kernel->array[i]; in check_shared_memory_bound()
981 static void mark_global_arrays(struct ppcg_kernel *kernel) in mark_global_arrays() argument
985 for (i = 0; i < kernel->n_array; ++i) { in mark_global_arrays()
986 struct gpu_local_array_info *local = &kernel->array[i]; in mark_global_arrays()
1003 static void compute_group_tilings(struct ppcg_kernel *kernel) in compute_group_tilings() argument
1007 for (i = 0; i < kernel->n_array; ++i) { in compute_group_tilings()
1008 struct gpu_local_array_info *array = &kernel->array[i]; in compute_group_tilings()
1035 struct ppcg_kernel *kernel, __isl_take isl_union_set *domain) in extract_grid_size() argument
1043 isl_union_set_copy(kernel->block_filter)); in extract_grid_size()
1046 grid = isl_set_add_dims(grid, isl_dim_set, kernel->n_grid); in extract_grid_size()
1047 for (i = 0; i < kernel->n_grid; ++i) { in extract_grid_size()
1051 id = isl_id_list_get_id(kernel->block_ids, i); in extract_grid_size()
1061 context = isl_set_params(isl_set_copy(kernel->context)); in extract_grid_size()
1112 static isl_stat extract_block_size(struct ppcg_kernel *kernel, in extract_block_size() argument
1120 isl_union_set_copy(kernel->thread_filter)); in extract_block_size()
1123 block = isl_set_add_dims(block, isl_dim_set, kernel->n_block); in extract_block_size()
1124 for (i = 0; i < kernel->n_block; ++i) { in extract_block_size()
1131 id = isl_id_list_get_id(kernel->thread_ids, i); in extract_block_size()
1147 extract_fixed_size(block, kernel->block_dim); in extract_block_size()
1152 struct ppcg_kernel *ppcg_kernel_free(struct ppcg_kernel *kernel) in ppcg_kernel_free() argument
1156 if (!kernel) in ppcg_kernel_free()
1159 isl_id_list_free(kernel->block_ids); in ppcg_kernel_free()
1160 isl_id_list_free(kernel->thread_ids); in ppcg_kernel_free()
1161 isl_multi_pw_aff_free(kernel->grid_size); in ppcg_kernel_free()
1162 isl_ast_expr_free(kernel->grid_size_expr); in ppcg_kernel_free()
1163 isl_set_free(kernel->context); in ppcg_kernel_free()
1164 isl_union_set_free(kernel->core); in ppcg_kernel_free()
1165 isl_union_set_free(kernel->arrays); in ppcg_kernel_free()
1166 isl_union_pw_multi_aff_free(kernel->contraction); in ppcg_kernel_free()
1167 isl_union_set_free(kernel->expanded_domain); in ppcg_kernel_free()
1168 isl_space_free(kernel->space); in ppcg_kernel_free()
1169 isl_ast_node_free(kernel->tree); in ppcg_kernel_free()
1170 isl_union_set_free(kernel->block_filter); in ppcg_kernel_free()
1171 isl_union_set_free(kernel->thread_filter); in ppcg_kernel_free()
1172 isl_union_pw_multi_aff_free(kernel->copy_schedule); in ppcg_kernel_free()
1173 isl_union_set_free(kernel->sync_writes); in ppcg_kernel_free()
1175 for (i = 0; i < kernel->n_array; ++i) { in ppcg_kernel_free()
1176 struct gpu_local_array_info *array = &kernel->array[i]; in ppcg_kernel_free()
1185 free(kernel->array); in ppcg_kernel_free()
1187 for (i = 0; i < kernel->n_var; ++i) { in ppcg_kernel_free()
1188 free(kernel->var[i].name); in ppcg_kernel_free()
1189 isl_vec_free(kernel->var[i].size); in ppcg_kernel_free()
1191 free(kernel->var); in ppcg_kernel_free()
1193 free(kernel); in ppcg_kernel_free()
1202 struct ppcg_kernel *kernel = user; in ppcg_kernel_free_wrap() local
1204 ppcg_kernel_free(kernel); in ppcg_kernel_free_wrap()
1231 static int create_kernel_vars(struct ppcg_kernel *kernel) in create_kernel_vars() argument
1236 for (i = 0; i < kernel->n_array; ++i) { in create_kernel_vars()
1237 struct gpu_local_array_info *array = &kernel->array[i]; in create_kernel_vars()
1249 kernel->n_var = n; in create_kernel_vars()
1250 kernel->var = isl_calloc_array(kernel->ctx, struct ppcg_kernel_var, n); in create_kernel_vars()
1251 if (!kernel->var) in create_kernel_vars()
1255 for (i = 0; i < kernel->n_array; ++i) { in create_kernel_vars()
1256 struct gpu_local_array_info *array = &kernel->array[i]; in create_kernel_vars()
1265 create_kernel_var(kernel->ctx, group, &kernel->var[n]); in create_kernel_vars()
1303 static void localize_bounds(struct ppcg_kernel *kernel, in localize_bounds() argument
1312 for (i = 0; i < kernel->n_array; ++i) { in localize_bounds()
1313 struct gpu_local_array_info *local = &kernel->array[i]; in localize_bounds()
1350 struct ppcg_kernel *kernel, struct gpu_prog *prog) in ppcg_kernel_create_local_arrays() argument
1356 kernel->array = isl_calloc_array(ctx, in ppcg_kernel_create_local_arrays()
1358 if (!kernel->array) in ppcg_kernel_create_local_arrays()
1359 return ppcg_kernel_free(kernel); in ppcg_kernel_create_local_arrays()
1360 kernel->n_array = prog->n_array; in ppcg_kernel_create_local_arrays()
1363 kernel->array[i].array = &prog->array[i]; in ppcg_kernel_create_local_arrays()
1365 return kernel; in ppcg_kernel_create_local_arrays()
1372 int ppcg_kernel_requires_array_argument(struct ppcg_kernel *kernel, int i) in ppcg_kernel_requires_array_argument() argument
1374 return kernel->array[i].global; in ppcg_kernel_requires_array_argument()
1431 static int find_array_index(struct ppcg_kernel *kernel, const char *name) in find_array_index() argument
1435 for (i = 0; i < kernel->n_array; ++i) in find_array_index()
1436 if (!strcmp(name, kernel->array[i].array->name)) in find_array_index()
1463 struct ppcg_kernel *kernel; member
1616 if (!data->kernel) in transform_index()
1626 i = find_array_index(data->kernel, name); in transform_index()
1631 data->local_array = &data->kernel->array[i]; in transform_index()
1841 struct ppcg_kernel *kernel, __isl_take isl_ast_node *node, in create_domain_leaf() argument
1865 if (kernel) in create_domain_leaf()
1866 sched2copy = compute_sched_to_copy(kernel, in create_domain_leaf()
1874 data.kernel = kernel; in create_domain_leaf()
1924 static __isl_give isl_ast_node *create_access_leaf(struct ppcg_kernel *kernel, in create_access_leaf() argument
1937 stmt = isl_calloc_type(kernel->ctx, struct ppcg_kernel_stmt); in create_access_leaf()
1970 id = isl_id_alloc(kernel->ctx, "copy", stmt); in create_access_leaf()
1979 struct ppcg_kernel *kernel, __isl_take isl_ast_node *node, in create_sync_leaf() argument
1985 stmt = isl_calloc_type(kernel->ctx, struct ppcg_kernel_stmt); in create_sync_leaf()
1990 id = isl_id_alloc(kernel->ctx, "sync", stmt); in create_sync_leaf()
2051 struct ppcg_kernel *kernel; member
2090 is_sync = gpu_tree_id_is_sync(id, data->kernel); in at_domain()
2094 return create_domain_leaf(data->kernel, node, build, gpu_stmt, in at_domain()
2107 return create_access_leaf(data->kernel, group, node, build); in at_domain()
2113 return create_sync_leaf(data->kernel, node, build); in at_domain()
2313 struct ppcg_kernel *kernel, struct gpu_array_ref_group *group, in remove_local_accesses_group() argument
2325 return remove_local_accesses(kernel->prog, tagged, access, sched, read); in remove_local_accesses_group()
2331 static isl_stat build_grid_size(struct ppcg_kernel *kernel, in build_grid_size() argument
2336 size = isl_multi_pw_aff_copy(kernel->grid_size); in build_grid_size()
2338 kernel->grid_size_expr = ppcg_build_size_expr(size, build); in build_grid_size()
2340 if (!kernel->grid_size_expr) in build_grid_size()
2349 static isl_stat build_local_array_sizes(struct ppcg_kernel *kernel, in build_local_array_sizes() argument
2354 for (i = 0; i < kernel->n_array; ++i) { in build_local_array_sizes()
2355 struct gpu_local_array_info *local = &kernel->array[i]; in build_local_array_sizes()
2372 static isl_stat build_grid_and_local_array_sizes(struct ppcg_kernel *kernel, in build_grid_and_local_array_sizes() argument
2375 if (build_grid_size(kernel, build) < 0) in build_grid_and_local_array_sizes()
2377 if (build_local_array_sizes(kernel, build) < 0) in build_grid_and_local_array_sizes()
2397 data->kernel = isl_id_get_user(mark); in before_mark()
2398 if (build_grid_and_local_array_sizes(data->kernel, build) < 0) in before_mark()
2422 struct ppcg_kernel *kernel; in after_mark() local
2429 if (strcmp(isl_id_get_name(id), "kernel") || !data->kernel) { in after_mark()
2433 kernel = data->kernel; in after_mark()
2434 data->kernel = NULL; in after_mark()
2435 kernel->space = isl_ast_build_get_schedule_space(build); in after_mark()
2436 kernel->tree = isl_ast_node_mark_get_node(node); in after_mark()
2481 data.kernel = NULL; in generate_code()
2674 static int any_sync_writes_in_group(struct ppcg_kernel *kernel, in any_sync_writes_in_group() argument
2680 empty = isl_union_set_is_empty(kernel->sync_writes); in any_sync_writes_in_group()
2687 disjoint = isl_union_set_is_disjoint(kernel->sync_writes, writes); in any_sync_writes_in_group()
2702 struct ppcg_kernel *kernel) in collect_non_private_tagged_writes() argument
2707 writes = isl_union_set_empty(isl_union_set_get_space(kernel->arrays)); in collect_non_private_tagged_writes()
2709 for (i = 0; i < kernel->n_array; ++i) { in collect_non_private_tagged_writes()
2710 struct gpu_local_array_info *array = &kernel->array[i]; in collect_non_private_tagged_writes()
2733 static int any_global_or_shared_sync_writes(struct ppcg_kernel *kernel) in any_global_or_shared_sync_writes() argument
2738 empty = isl_union_set_is_empty(kernel->sync_writes); in any_global_or_shared_sync_writes()
2744 writes = collect_non_private_tagged_writes(kernel); in any_global_or_shared_sync_writes()
2745 disjoint = isl_union_set_is_disjoint(kernel->sync_writes, writes); in any_global_or_shared_sync_writes()
3054 static __isl_give isl_schedule_node *insert_context(struct ppcg_kernel *kernel, in insert_context() argument
3059 context = isl_set_universe(isl_set_get_space(kernel->context)); in insert_context()
3062 kernel->grid_size, kernel->block_ids); in insert_context()
3064 kernel->block_dim, kernel->thread_ids); in insert_context()
3111 static int kernel_requires_unroll(struct ppcg_kernel *kernel) in kernel_requires_unroll() argument
3115 for (i = 0; i < kernel->n_array; ++i) { in kernel_requires_unroll()
3116 struct gpu_local_array_info *array = &kernel->array[i]; in kernel_requires_unroll()
3160 static __isl_give isl_schedule_node *add_sync(struct ppcg_kernel *kernel, in add_sync() argument
3166 need_sync = any_global_or_shared_sync_writes(kernel); in add_sync()
3172 node = gpu_tree_move_down_to_thread(node, kernel->core); in add_sync()
3178 node = gpu_tree_move_down_to_depth(node, depth, kernel->core); in add_sync()
3179 node = gpu_tree_ensure_following_sync(node, kernel); in add_sync()
3197 struct ppcg_kernel *kernel, struct gpu_array_ref_group *group, in anchored_non_local_accesses() argument
3205 isl_union_pw_multi_aff_copy(kernel->contraction)); in anchored_non_local_accesses()
3207 access = remove_local_accesses_group(kernel, group, access, prefix, in anchored_non_local_accesses()
3259 __isl_take isl_schedule_node *node, struct ppcg_kernel *kernel, in add_group_write_sync() argument
3264 need_sync = any_sync_writes_in_group(kernel, group); in add_group_write_sync()
3271 node = gpu_tree_ensure_following_sync(node, kernel); in add_group_write_sync()
3279 kernel->core); in add_group_write_sync()
3280 node = gpu_tree_move_left_to_sync(node, kernel); in add_group_write_sync()
3332 struct ppcg_kernel *kernel, struct gpu_array_ref_group *group, in add_copies_group_private() argument
3350 node = gpu_tree_move_down_to_depth(node, tile->depth, kernel->core); in add_copies_group_private()
3352 access = anchored_non_local_accesses(kernel, group, node, read); in add_copies_group_private()
3364 from_access = create_from_access(kernel->ctx, group, read); in add_copies_group_private()
3368 filter = isl_union_set_copy(kernel->thread_filter); in add_copies_group_private()
3369 contraction = isl_union_pw_multi_aff_copy(kernel->contraction); in add_copies_group_private()
3399 node = add_group_write_sync(node, kernel, group, 0); in add_copies_group_private()
3486 struct ppcg_kernel *kernel, struct gpu_array_ref_group *group, in add_copies_group_shared() argument
3504 node = gpu_tree_move_down_to_depth(node, tile->depth, kernel->core); in add_copies_group_shared()
3506 access = anchored_non_local_accesses(kernel, group, node, read); in add_copies_group_shared()
3518 from_access = create_from_access(kernel->ctx, group, read); in add_copies_group_shared()
3544 if (kernel->options->unroll_copy_shared) in add_copies_group_shared()
3547 if (tile->n > kernel->n_block && kernel->n_block > 0) { in add_copies_group_shared()
3549 tile->n - kernel->n_block); in add_copies_group_shared()
3552 if (tile->n < kernel->n_block) in add_copies_group_shared()
3553 skip = kernel->n_block - tile->n; in add_copies_group_shared()
3556 filter = set_schedule_modulo(graft, kernel->thread_ids, in add_copies_group_shared()
3557 kernel->block_dim); in add_copies_group_shared()
3558 if (!kernel->options->wrap) in add_copies_group_shared()
3559 graft = snap_band_to_sizes(graft, kernel->block_dim + skip, in add_copies_group_shared()
3560 kernel->options); in add_copies_group_shared()
3561 if (tile->n > kernel->n_block && kernel->n_block > 0) in add_copies_group_shared()
3570 node = gpu_tree_ensure_sync_after_core(node, kernel); in add_copies_group_shared()
3571 node = gpu_tree_move_left_to_sync(node, kernel); in add_copies_group_shared()
3574 node = gpu_tree_move_right_to_sync(node, kernel); in add_copies_group_shared()
3577 node = add_group_write_sync(node, kernel, group, 1); in add_copies_group_shared()
3595 struct ppcg_kernel *kernel, struct gpu_array_ref_group *group, in add_copies_group() argument
3602 return add_copies_group_private(kernel, group, node, read); in add_copies_group()
3604 return add_copies_group_shared(kernel, group, node, read); in add_copies_group()
3615 static __isl_give isl_schedule_node *add_copies(struct ppcg_kernel *kernel, in add_copies() argument
3620 for (i = 0; i < kernel->n_array; ++i) { in add_copies()
3621 struct gpu_local_array_info *array = &kernel->array[i]; in add_copies()
3626 node = add_copies_group(kernel, group, node, 1); in add_copies()
3629 node = add_copies_group(kernel, group, node, 0); in add_copies()
3705 struct ppcg_kernel *kernel, __isl_keep isl_schedule_node *node) in compute_sync_writes() argument
3717 node = gpu_tree_move_down_to_thread(node, kernel->core); in compute_sync_writes()
3721 contraction = kernel->contraction; in compute_sync_writes()
3726 domain = isl_union_set_copy(kernel->expanded_domain); in compute_sync_writes()
3729 may_writes = isl_union_map_copy(kernel->prog->scop->tagged_may_writes); in compute_sync_writes()
3737 local = isl_union_map_copy(kernel->prog->scop->tagged_dep_flow); in compute_sync_writes()
3838 struct ppcg_kernel *kernel; in gpu_create_kernel() local
3851 kernel = isl_calloc_type(gen->ctx, struct ppcg_kernel); in gpu_create_kernel()
3852 kernel = ppcg_kernel_create_local_arrays(kernel, gen->prog); in gpu_create_kernel()
3853 if (!kernel) in gpu_create_kernel()
3859 kernel->ctx = gen->ctx; in gpu_create_kernel()
3860 kernel->prog = gen->prog; in gpu_create_kernel()
3861 kernel->options = gen->options; in gpu_create_kernel()
3862 kernel->context = extract_context(node, gen->prog); in gpu_create_kernel()
3863 kernel->core = isl_union_set_universe(isl_union_set_copy(domain)); in gpu_create_kernel()
3865 kernel->contraction = isl_union_pw_multi_aff_copy(contraction); in gpu_create_kernel()
3869 kernel->expanded_domain = isl_union_set_copy(expanded); in gpu_create_kernel()
3870 kernel->arrays = accessed_by_domain(expanded, gen->prog); in gpu_create_kernel()
3871 kernel->n_grid = n_outer_coincidence(node); in gpu_create_kernel()
3873 node_thread = gpu_tree_move_down_to_thread(node_thread, kernel->core); in gpu_create_kernel()
3875 kernel->n_block = n_outer_coincidence(node_thread); in gpu_create_kernel()
3877 kernel->id = gen->kernel_id++; in gpu_create_kernel()
3878 read_grid_and_block_sizes(kernel, gen); in gpu_create_kernel()
3880 kernel->sync_writes = compute_sync_writes(kernel, node); in gpu_create_kernel()
3888 id = isl_id_alloc(gen->ctx, "kernel", kernel); in gpu_create_kernel()
3893 node = group_statements(node, kernel->id); in gpu_create_kernel()
3896 node = split_band(node, kernel->n_grid); in gpu_create_kernel()
3897 kernel->block_ids = ppcg_scop_generate_names(gen->prog->scop, in gpu_create_kernel()
3898 kernel->n_grid, "b"); in gpu_create_kernel()
3899 kernel->block_filter = set_schedule_modulo(node, kernel->block_ids, in gpu_create_kernel()
3900 kernel->grid_dim); in gpu_create_kernel()
3901 kernel->grid_size = extract_grid_size(kernel, in gpu_create_kernel()
3903 if (!kernel->options->wrap) in gpu_create_kernel()
3904 node = snap_band_to_sizes(node, kernel->grid_dim, in gpu_create_kernel()
3905 kernel->options); in gpu_create_kernel()
3911 node = insert_guard(node, kernel->context, kernel->grid_size, in gpu_create_kernel()
3913 node = gpu_tree_move_down_to_thread(node, kernel->core); in gpu_create_kernel()
3915 node = split_band(node, kernel->n_block); in gpu_create_kernel()
3916 kernel->thread_ids = ppcg_scop_generate_names(gen->prog->scop, in gpu_create_kernel()
3917 kernel->n_block, "t"); in gpu_create_kernel()
3918 kernel->thread_filter = set_schedule_modulo(node, kernel->thread_ids, in gpu_create_kernel()
3919 kernel->block_dim); in gpu_create_kernel()
3920 if (extract_block_size(kernel, domain) < 0) in gpu_create_kernel()
3925 node = insert_context(kernel, node); in gpu_create_kernel()
3928 isl_union_set_copy(kernel->block_filter)); in gpu_create_kernel()
3932 if (gpu_group_references(kernel, node) < 0) in gpu_create_kernel()
3934 localize_bounds(kernel, host_domain); in gpu_create_kernel()
3937 check_shared_memory_bound(kernel); in gpu_create_kernel()
3938 mark_global_arrays(kernel); in gpu_create_kernel()
3939 compute_group_tilings(kernel); in gpu_create_kernel()
3941 node = gpu_tree_move_down_to_thread(node, kernel->core); in gpu_create_kernel()
3943 if (!kernel->options->wrap) in gpu_create_kernel()
3944 node = snap_band_to_sizes(node, kernel->block_dim, in gpu_create_kernel()
3945 kernel->options); in gpu_create_kernel()
3947 isl_union_set_copy(kernel->thread_filter)); in gpu_create_kernel()
3948 if (kernel_requires_unroll(kernel)) { in gpu_create_kernel()
3954 kernel->copy_schedule_dim = isl_schedule_node_get_schedule_depth(node); in gpu_create_kernel()
3955 kernel->copy_schedule = in gpu_create_kernel()
3957 contraction = isl_union_pw_multi_aff_copy(kernel->contraction); in gpu_create_kernel()
3958 kernel->copy_schedule = in gpu_create_kernel()
3960 kernel->copy_schedule, contraction); in gpu_create_kernel()
3964 node = add_sync(kernel, node); in gpu_create_kernel()
3965 node = add_copies(kernel, node); in gpu_create_kernel()
3967 node = gpu_tree_move_down_to_shared(node, kernel->core); in gpu_create_kernel()
3970 node = gpu_tree_move_down_to_thread(node, kernel->core); in gpu_create_kernel()
3975 if (create_kernel_vars(kernel) < 0) in gpu_create_kernel()