Lines Matching refs:affinity
67 __kmp_get_affinity_env_var(const kmp_affinity_t &affinity, in __kmp_get_affinity_env_var() argument
69 if (affinity.flags.omp_places) { in __kmp_get_affinity_env_var()
74 return affinity.env_var; in __kmp_get_affinity_env_var()
848 void kmp_topology_t::set_granularity(kmp_affinity_t &affinity) const { in set_granularity()
849 const char *env_var = __kmp_get_affinity_env_var(affinity); in set_granularity()
854 if (affinity.core_attr_gran.valid) { in set_granularity()
858 affinity, AffIgnoringNonHybrid, env_var, in set_granularity()
860 affinity.gran = KMP_HW_CORE; in set_granularity()
861 affinity.gran_levels = -1; in set_granularity()
862 affinity.core_attr_gran = KMP_AFFINITY_ATTRS_UNKNOWN; in set_granularity()
863 affinity.flags.core_types_gran = affinity.flags.core_effs_gran = 0; in set_granularity()
864 } else if (affinity.flags.core_types_gran || in set_granularity()
865 affinity.flags.core_effs_gran) { in set_granularity()
867 if (affinity.flags.omp_places) { in set_granularity()
869 affinity, AffIgnoringNonHybrid, env_var, in set_granularity()
873 KMP_AFF_WARNING(affinity, AffGranularityBad, env_var, in set_granularity()
877 affinity.gran = KMP_HW_CORE; in set_granularity()
878 affinity.gran_levels = -1; in set_granularity()
879 affinity.core_attr_gran = KMP_AFFINITY_ATTRS_UNKNOWN; in set_granularity()
880 affinity.flags.core_types_gran = affinity.flags.core_effs_gran = 0; in set_granularity()
884 if (affinity.gran_levels < 0) { in set_granularity()
885 kmp_hw_t gran_type = get_equivalent_type(affinity.gran); in set_granularity()
898 KMP_AFF_WARNING(affinity, AffGranularityBad, env_var, in set_granularity()
899 __kmp_hw_get_catalog_string(affinity.gran), in set_granularity()
901 affinity.gran = gran_type; in set_granularity()
916 KMP_AFF_WARNING(affinity, AffGranTooCoarseProcGroup, env_var, in set_granularity()
917 __kmp_hw_get_catalog_string(affinity.gran)); in set_granularity()
918 affinity.gran = gran_type = KMP_HW_PROC_GROUP; in set_granularity()
922 affinity.gran_levels = 0; in set_granularity()
924 affinity.gran_levels++; in set_granularity()
3595 kmp_affinity_t &affinity, in __kmp_create_os_id_masks() argument
3602 const char *env_var = __kmp_get_affinity_env_var(affinity); in __kmp_create_os_id_masks()
3621 affinity.num_os_id_masks = maxOsId + 1; in __kmp_create_os_id_masks()
3622 KMP_CPU_ALLOC_ARRAY(affinity.os_id_masks, affinity.num_os_id_masks); in __kmp_create_os_id_masks()
3623 KMP_ASSERT(affinity.gran_levels >= 0); in __kmp_create_os_id_masks()
3624 if (affinity.flags.verbose && (affinity.gran_levels > 0)) { in __kmp_create_os_id_masks()
3625 KMP_INFORM(ThreadsMigrate, env_var, affinity.gran_levels); in __kmp_create_os_id_masks()
3627 if (affinity.gran_levels >= (int)depth) { in __kmp_create_os_id_masks()
3628 KMP_AFF_WARNING(affinity, AffThreadsMayMigrate); in __kmp_create_os_id_masks()
3649 if (__kmp_topology->is_close(leader, i, affinity)) { in __kmp_create_os_id_masks()
3659 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.os_id_masks, osId); in __kmp_create_os_id_masks()
3677 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.os_id_masks, osId); in __kmp_create_os_id_masks()
3686 if (full_mask.restrict_to_mask() && affinity.flags.verbose) { in __kmp_create_os_id_masks()
3723 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, _osId); \
3731 static void __kmp_affinity_process_proclist(kmp_affinity_t &affinity) { in __kmp_affinity_process_proclist() argument
3733 kmp_affin_mask_t **out_masks = &affinity.masks; in __kmp_affinity_process_proclist()
3734 unsigned *out_numMasks = &affinity.num_masks; in __kmp_affinity_process_proclist()
3735 const char *proclist = affinity.proclist; in __kmp_affinity_process_proclist()
3736 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks; in __kmp_affinity_process_proclist()
3737 int maxOsId = affinity.num_os_id_masks - 1; in __kmp_affinity_process_proclist()
3775 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num); in __kmp_affinity_process_proclist()
3807 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num); in __kmp_affinity_process_proclist()
3942 kmp_affinity_t &affinity, int maxOsId, in __kmp_process_subplace_list() argument
3946 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks; in __kmp_process_subplace_list()
3965 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start); in __kmp_process_subplace_list()
3994 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start); in __kmp_process_subplace_list()
4041 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, start); in __kmp_process_subplace_list()
4060 static void __kmp_process_place(const char **scan, kmp_affinity_t &affinity, in __kmp_process_place() argument
4064 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks; in __kmp_process_place()
4070 __kmp_process_subplace_list(scan, affinity, maxOsId, tempMask, setSize); in __kmp_process_place()
4075 __kmp_process_place(scan, affinity, maxOsId, tempMask, setSize); in __kmp_process_place()
4084 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, num); in __kmp_process_place()
4096 void __kmp_affinity_process_placelist(kmp_affinity_t &affinity) { in __kmp_affinity_process_placelist() argument
4098 kmp_affin_mask_t **out_masks = &affinity.masks; in __kmp_affinity_process_placelist()
4099 unsigned *out_numMasks = &affinity.num_masks; in __kmp_affinity_process_placelist()
4100 const char *placelist = affinity.proclist; in __kmp_affinity_process_placelist()
4101 kmp_affin_mask_t *osId2Mask = affinity.os_id_masks; in __kmp_affinity_process_placelist()
4102 int maxOsId = affinity.num_os_id_masks - 1; in __kmp_affinity_process_placelist()
4122 __kmp_process_place(&scan, affinity, maxOsId, tempMask, &setSize); in __kmp_affinity_process_placelist()
4203 KMP_AFF_WARNING(affinity, AffIgnoreInvalidProcID, j + stride); in __kmp_affinity_process_placelist()
4363 static void __kmp_affinity_get_topology_info(kmp_affinity_t &affinity) { in __kmp_affinity_get_topology_info() argument
4366 if (affinity.type != affinity_none) { in __kmp_affinity_get_topology_info()
4367 KMP_ASSERT(affinity.num_os_id_masks); in __kmp_affinity_get_topology_info()
4368 KMP_ASSERT(affinity.os_id_masks); in __kmp_affinity_get_topology_info()
4370 KMP_ASSERT(affinity.num_masks); in __kmp_affinity_get_topology_info()
4371 KMP_ASSERT(affinity.masks); in __kmp_affinity_get_topology_info()
4378 if (!affinity.ids) { in __kmp_affinity_get_topology_info()
4379 affinity.ids = (kmp_affinity_ids_t *)__kmp_allocate( in __kmp_affinity_get_topology_info()
4380 sizeof(kmp_affinity_ids_t) * affinity.num_masks); in __kmp_affinity_get_topology_info()
4382 if (!affinity.attrs) { in __kmp_affinity_get_topology_info()
4383 affinity.attrs = (kmp_affinity_attrs_t *)__kmp_allocate( in __kmp_affinity_get_topology_info()
4384 sizeof(kmp_affinity_attrs_t) * affinity.num_masks); in __kmp_affinity_get_topology_info()
4399 for (unsigned i = 0; i < affinity.num_masks; ++i) { in __kmp_affinity_get_topology_info()
4400 kmp_affinity_ids_t &ids = affinity.ids[i]; in __kmp_affinity_get_topology_info()
4401 kmp_affinity_attrs_t &attrs = affinity.attrs[i]; in __kmp_affinity_get_topology_info()
4402 kmp_affin_mask_t *mask = KMP_CPU_INDEX(affinity.masks, i); in __kmp_affinity_get_topology_info()
4408 static void __kmp_aux_affinity_initialize_other_data(kmp_affinity_t &affinity) { in __kmp_aux_affinity_initialize_other_data() argument
4412 __kmp_affinity_get_topology_info(affinity); in __kmp_aux_affinity_initialize_other_data()
4421 static void __kmp_create_affinity_none_places(kmp_affinity_t &affinity) { in __kmp_create_affinity_none_places() argument
4423 KMP_ASSERT(affinity.type == affinity_none); in __kmp_create_affinity_none_places()
4425 affinity.num_masks = 1; in __kmp_create_affinity_none_places()
4426 KMP_CPU_ALLOC_ARRAY(affinity.masks, affinity.num_masks); in __kmp_create_affinity_none_places()
4427 kmp_affin_mask_t *dest = KMP_CPU_INDEX(affinity.masks, 0); in __kmp_create_affinity_none_places()
4429 __kmp_aux_affinity_initialize_other_data(affinity); in __kmp_create_affinity_none_places()
4432 static void __kmp_aux_affinity_initialize_masks(kmp_affinity_t &affinity) { in __kmp_aux_affinity_initialize_masks() argument
4437 int verbose = affinity.flags.verbose; in __kmp_aux_affinity_initialize_masks()
4438 const char *env_var = affinity.env_var; in __kmp_aux_affinity_initialize_masks()
4454 if (affinity.flags.respect) { in __kmp_aux_affinity_initialize_masks()
4465 KMP_AFF_WARNING(affinity, ErrorInitializeAffinity); in __kmp_aux_affinity_initialize_masks()
4466 affinity.type = affinity_none; in __kmp_aux_affinity_initialize_masks()
4499 static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) { in __kmp_aux_affinity_initialize_topology() argument
4501 const char *env_var = affinity.env_var; in __kmp_aux_affinity_initialize_topology()
4503 int verbose = affinity.flags.verbose; in __kmp_aux_affinity_initialize_topology()
4639 KMP_AFF_WARNING(affinity, ErrorInitializeAffinity); in __kmp_aux_affinity_initialize_topology()
4663 static void __kmp_aux_affinity_initialize(kmp_affinity_t &affinity) { in __kmp_aux_affinity_initialize() argument
4664 bool is_regular_affinity = (&affinity == &__kmp_affinity); in __kmp_aux_affinity_initialize()
4665 bool is_hidden_helper_affinity = (&affinity == &__kmp_hh_affinity); in __kmp_aux_affinity_initialize()
4666 const char *env_var = __kmp_get_affinity_env_var(affinity); in __kmp_aux_affinity_initialize()
4668 if (affinity.flags.initialized) { in __kmp_aux_affinity_initialize()
4674 __kmp_aux_affinity_initialize_masks(affinity); in __kmp_aux_affinity_initialize()
4677 bool success = __kmp_aux_affinity_initialize_topology(affinity); in __kmp_aux_affinity_initialize()
4681 affinity.type = affinity_none; in __kmp_aux_affinity_initialize()
4689 if (affinity.type == affinity_none) { in __kmp_aux_affinity_initialize()
4690 __kmp_create_affinity_none_places(affinity); in __kmp_aux_affinity_initialize()
4694 affinity.flags.initialized = TRUE; in __kmp_aux_affinity_initialize()
4698 __kmp_topology->set_granularity(affinity); in __kmp_aux_affinity_initialize()
4706 if (affinity.core_attr_gran.valid) { in __kmp_aux_affinity_initialize()
4707 __kmp_create_os_id_masks(&numUnique, affinity, [&](int idx) { in __kmp_aux_affinity_initialize()
4710 if (__kmp_topology->at(i).attrs.contains(affinity.core_attr_gran)) in __kmp_aux_affinity_initialize()
4714 if (!affinity.os_id_masks) { in __kmp_aux_affinity_initialize()
4716 if (affinity.core_attr_gran.core_eff != kmp_hw_attr_t::UNKNOWN_CORE_EFF) in __kmp_aux_affinity_initialize()
4720 KMP_AFF_WARNING(affinity, AffIgnoringNotAvailable, env_var, in __kmp_aux_affinity_initialize()
4727 if (!affinity.os_id_masks) { in __kmp_aux_affinity_initialize()
4728 __kmp_create_os_id_masks(&numUnique, affinity, [](int idx) { in __kmp_aux_affinity_initialize()
4733 if (affinity.gran_levels == 0) { in __kmp_aux_affinity_initialize()
4737 switch (affinity.type) { in __kmp_aux_affinity_initialize()
4740 KMP_DEBUG_ASSERT(affinity.proclist != NULL); in __kmp_aux_affinity_initialize()
4743 __kmp_affinity_process_proclist(affinity); in __kmp_aux_affinity_initialize()
4745 __kmp_affinity_process_placelist(affinity); in __kmp_aux_affinity_initialize()
4747 if (affinity.num_masks == 0) { in __kmp_aux_affinity_initialize()
4748 KMP_AFF_WARNING(affinity, AffNoValidProcID); in __kmp_aux_affinity_initialize()
4749 affinity.type = affinity_none; in __kmp_aux_affinity_initialize()
4750 __kmp_create_affinity_none_places(affinity); in __kmp_aux_affinity_initialize()
4751 affinity.flags.initialized = TRUE; in __kmp_aux_affinity_initialize()
4761 affinity.compact = 0; in __kmp_aux_affinity_initialize()
4762 if (affinity.offset) { in __kmp_aux_affinity_initialize()
4763 affinity.offset = in __kmp_aux_affinity_initialize()
4764 __kmp_nThreadsPerCore * affinity.offset % __kmp_avail_proc; in __kmp_aux_affinity_initialize()
4770 affinity.compact = 1; in __kmp_aux_affinity_initialize()
4771 if (affinity.compact >= depth) { in __kmp_aux_affinity_initialize()
4772 affinity.compact = 0; in __kmp_aux_affinity_initialize()
4775 affinity.compact = 0; in __kmp_aux_affinity_initialize()
4777 if (affinity.offset) { in __kmp_aux_affinity_initialize()
4778 affinity.offset = in __kmp_aux_affinity_initialize()
4779 __kmp_nThreadsPerCore * affinity.offset % __kmp_avail_proc; in __kmp_aux_affinity_initialize()
4784 if (affinity.compact >= depth) { in __kmp_aux_affinity_initialize()
4785 affinity.compact = 0; in __kmp_aux_affinity_initialize()
4787 affinity.compact = depth - 1 - affinity.compact; in __kmp_aux_affinity_initialize()
4792 if (affinity.compact >= depth) { in __kmp_aux_affinity_initialize()
4793 affinity.compact = depth - 1; in __kmp_aux_affinity_initialize()
4799 KMP_AFF_WARNING(affinity, AffBalancedNotAvail, env_var); in __kmp_aux_affinity_initialize()
4800 affinity.type = affinity_none; in __kmp_aux_affinity_initialize()
4801 __kmp_create_affinity_none_places(affinity); in __kmp_aux_affinity_initialize()
4802 affinity.flags.initialized = TRUE; in __kmp_aux_affinity_initialize()
4817 KMP_AFF_WARNING(affinity, AffBalancedNotAvail, env_var); in __kmp_aux_affinity_initialize()
4818 affinity.type = affinity_none; in __kmp_aux_affinity_initialize()
4819 __kmp_create_affinity_none_places(affinity); in __kmp_aux_affinity_initialize()
4820 affinity.flags.initialized = TRUE; in __kmp_aux_affinity_initialize()
4845 if (affinity.compact >= depth) { in __kmp_aux_affinity_initialize()
4846 affinity.compact = depth - 1; in __kmp_aux_affinity_initialize()
4851 if (affinity.flags.dups) { in __kmp_aux_affinity_initialize()
4852 affinity.num_masks = __kmp_avail_proc; in __kmp_aux_affinity_initialize()
4854 affinity.num_masks = numUnique; in __kmp_aux_affinity_initialize()
4859 ((unsigned)__kmp_affinity_num_places < affinity.num_masks) && in __kmp_aux_affinity_initialize()
4861 affinity.num_masks = __kmp_affinity_num_places; in __kmp_aux_affinity_initialize()
4864 KMP_CPU_ALLOC_ARRAY(affinity.masks, affinity.num_masks); in __kmp_aux_affinity_initialize()
4868 __kmp_topology->sort_compact(affinity); in __kmp_aux_affinity_initialize()
4875 if ((!affinity.flags.dups) && (!__kmp_topology->at(i).leader)) { in __kmp_aux_affinity_initialize()
4880 kmp_affin_mask_t *src = KMP_CPU_INDEX(affinity.os_id_masks, osId); in __kmp_aux_affinity_initialize()
4881 kmp_affin_mask_t *dest = KMP_CPU_INDEX(affinity.masks, j); in __kmp_aux_affinity_initialize()
4885 if (++j >= affinity.num_masks) { in __kmp_aux_affinity_initialize()
4889 KMP_DEBUG_ASSERT(j == affinity.num_masks); in __kmp_aux_affinity_initialize()
4891 if (full_mask.restrict_to_mask() && affinity.flags.verbose) { in __kmp_aux_affinity_initialize()
4902 __kmp_aux_affinity_initialize_other_data(affinity); in __kmp_aux_affinity_initialize()
4903 affinity.flags.initialized = TRUE; in __kmp_aux_affinity_initialize()
4906 void __kmp_affinity_initialize(kmp_affinity_t &affinity) { in __kmp_affinity_initialize() argument
4915 int disabled = (affinity.type == affinity_disabled); in __kmp_affinity_initialize()
4919 affinity.type = affinity_none; in __kmp_affinity_initialize()
4920 __kmp_aux_affinity_initialize(affinity); in __kmp_affinity_initialize()
4922 affinity.type = affinity_disabled; in __kmp_affinity_initialize()
4926 for (kmp_affinity_t *affinity : __kmp_affinities) { in __kmp_affinity_uninitialize()
4927 if (affinity->masks != NULL) in __kmp_affinity_uninitialize()
4928 KMP_CPU_FREE_ARRAY(affinity->masks, affinity->num_masks); in __kmp_affinity_uninitialize()
4929 if (affinity->os_id_masks != NULL) in __kmp_affinity_uninitialize()
4930 KMP_CPU_FREE_ARRAY(affinity->os_id_masks, affinity->num_os_id_masks); in __kmp_affinity_uninitialize()
4931 if (affinity->proclist != NULL) in __kmp_affinity_uninitialize()
4932 __kmp_free(affinity->proclist); in __kmp_affinity_uninitialize()
4933 if (affinity->ids != NULL) in __kmp_affinity_uninitialize()
4934 __kmp_free(affinity->ids); in __kmp_affinity_uninitialize()
4935 if (affinity->attrs != NULL) in __kmp_affinity_uninitialize()
4936 __kmp_free(affinity->attrs); in __kmp_affinity_uninitialize()
4937 *affinity = KMP_AFFINITY_INIT(affinity->env_var); in __kmp_affinity_uninitialize()
4977 static void __kmp_select_mask_by_gtid(int gtid, const kmp_affinity_t *affinity, in __kmp_select_mask_by_gtid() argument
4987 KMP_DEBUG_ASSERT(affinity->num_masks > 0); in __kmp_select_mask_by_gtid()
4988 *place = (mask_idx + affinity->offset) % affinity->num_masks; in __kmp_select_mask_by_gtid()
4989 *mask = KMP_CPU_INDEX(affinity->masks, *place); in __kmp_select_mask_by_gtid()
5020 const kmp_affinity_t *affinity; in __kmp_affinity_set_init_mask() local
5024 affinity = &__kmp_hh_affinity; in __kmp_affinity_set_init_mask()
5026 affinity = &__kmp_affinity; in __kmp_affinity_set_init_mask()
5029 if ((affinity->type == affinity_none) || in __kmp_affinity_set_init_mask()
5030 (affinity->type == affinity_balanced) || in __kmp_affinity_set_init_mask()
5041 __kmp_select_mask_by_gtid(gtid, affinity, &i, &mask); in __kmp_affinity_set_init_mask()
5054 __kmp_select_mask_by_gtid(gtid, affinity, &i, &mask); in __kmp_affinity_set_init_mask()
5062 th->th.th_last_place = affinity->num_masks - 1; in __kmp_affinity_set_init_mask()
5067 th->th.th_last_place = affinity->num_masks - 1; in __kmp_affinity_set_init_mask()
5091 const kmp_affinity_t *affinity; in __kmp_affinity_bind_init_mask() local
5096 affinity = &__kmp_hh_affinity; in __kmp_affinity_bind_init_mask()
5098 affinity = &__kmp_affinity; in __kmp_affinity_bind_init_mask()
5099 env_var = __kmp_get_affinity_env_var(*affinity, /*for_binding=*/true); in __kmp_affinity_bind_init_mask()
5101 if (affinity->flags.verbose && (affinity->type == affinity_none || in __kmp_affinity_bind_init_mask()
5103 affinity->type != affinity_balanced)) && in __kmp_affinity_bind_init_mask()
5116 if (affinity->type == affinity_none) { in __kmp_affinity_bind_init_mask()