178 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()  argument
181 	*(void __percpu **)(l->key + roundup(key_size, 8)) = pptr;  in htab_elem_set_ptr()
184 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()  argument
186 	return *(void __percpu **)(l->key + roundup(key_size, 8));  in htab_elem_get_ptr()
191 	return *(void **)(l->key + roundup(map->key_size, 8));  in fd_htab_map_get_ptr()
218 					   elem->key + round_up(htab->map.key_size, 8));  in htab_free_prealloced_timers_and_wq()
221 					       elem->key + round_up(htab->map.key_size, 8));  in htab_free_prealloced_timers_and_wq()
240 			void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);  in htab_free_prealloced_fields()
248 			bpf_obj_free_fields(htab->map.record, elem->key + round_up(htab->map.key_size, 8));  in htab_free_prealloced_fields()
266 					 htab->map.key_size);  in htab_free_elems()
294 		memcpy(l->key, key, htab->map.key_size);  in prealloc_lru_pop()
325 		htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size,  in prealloc_init()
433 	if (attr->max_entries == 0 || attr->key_size == 0 ||  in htab_map_alloc_check()
437 	if ((u64)attr->key_size + attr->value_size >= KMALLOC_MAX_SIZE -  in htab_map_alloc_check()
496 			  round_up(htab->map.key_size, 8);  in htab_map_alloc()
608 					 void *key, u32 key_size)  in lookup_elem_raw()  argument
614 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_elem_raw()
626 					       u32 key_size, u32 n_buckets)  in lookup_nulls_elem_raw()  argument
633 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_nulls_elem_raw()
652 	u32 hash, key_size;  in __htab_map_lookup_elem()  local
657 	key_size = map->key_size;  in __htab_map_lookup_elem()
659 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_map_lookup_elem()
663 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in __htab_map_lookup_elem()
673 		return l->key + round_up(map->key_size, 8);  in htab_map_lookup_elem()
700 				round_up(map->key_size, 8));  in htab_map_gen_lookup()
712 		return l->key + round_up(map->key_size, 8);  in __htab_lru_map_lookup_elem()
749 				round_up(map->key_size, 8));  in htab_lru_map_gen_lookup()
760 		void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);  in check_and_free_fields()
766 		void *map_value = elem->key + round_up(htab->map.key_size, 8);  in check_and_free_fields()
813 	u32 hash, key_size;  in htab_map_get_next_key()  local
818 	key_size = map->key_size;  in htab_map_get_next_key()
823 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_get_next_key()
828 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in htab_map_get_next_key()
839 		memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
857 			memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
976 					 void *value, u32 key_size, u32 hash,  in alloc_htab_elem()  argument
1019 	memcpy(l_new->key, key, key_size);  in alloc_htab_elem()
1022 			pptr = htab_elem_get_ptr(l_new, key_size);  in alloc_htab_elem()
1039 			htab_elem_set_ptr(l_new, key_size, pptr);  in alloc_htab_elem()
1042 		memcpy(l_new->key + round_up(key_size, 8), value, size);  in alloc_htab_elem()
1045 			       l_new->key + round_up(key_size, 8),  in alloc_htab_elem()
1080 	u32 key_size, hash;  in htab_map_update_elem()  local
1090 	key_size = map->key_size;  in htab_map_update_elem()
1092 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_update_elem()
1101 		l_old = lookup_nulls_elem_raw(head, hash, key, key_size,  in htab_map_update_elem()
1109 					      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
1123 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_map_update_elem()
1137 				      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
1143 	l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,  in htab_map_update_elem()
1198 	u32 key_size, hash;  in htab_lru_map_update_elem()  local
1208 	key_size = map->key_size;  in htab_lru_map_update_elem()
1210 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_update_elem()
1224 		       l_new->key + round_up(map->key_size, 8), value);  in htab_lru_map_update_elem()
1230 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_update_elem()
1267 	u32 key_size, hash;  in __htab_percpu_map_update_elem()  local
1277 	key_size = map->key_size;  in __htab_percpu_map_update_elem()
1279 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_percpu_map_update_elem()
1288 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_percpu_map_update_elem()
1296 		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),  in __htab_percpu_map_update_elem()
1299 		l_new = alloc_htab_elem(htab, key, value, key_size,  in __htab_percpu_map_update_elem()
1322 	u32 key_size, hash;  in __htab_lru_percpu_map_update_elem()  local
1332 	key_size = map->key_size;  in __htab_lru_percpu_map_update_elem()
1334 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_lru_percpu_map_update_elem()
1354 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_lru_percpu_map_update_elem()
1364 		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),  in __htab_lru_percpu_map_update_elem()
1367 		pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),  in __htab_lru_percpu_map_update_elem()
1404 	u32 hash, key_size;  in htab_map_delete_elem()  local
1410 	key_size = map->key_size;  in htab_map_delete_elem()
1412 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_delete_elem()
1420 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_map_delete_elem()
1440 	u32 hash, key_size;  in htab_lru_map_delete_elem()  local
1446 	key_size = map->key_size;  in htab_lru_map_delete_elem()
1448 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_delete_elem()
1456 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_delete_elem()
1503 						   l->key + round_up(htab->map.key_size, 8));  in htab_free_malloced_timers_and_wq()
1506 						       l->key + round_up(htab->map.key_size, 8));  in htab_free_malloced_timers_and_wq()
1586 	u32 hash, key_size;  in __htab_map_lookup_and_delete_elem()  local
1590 	key_size = map->key_size;  in __htab_map_lookup_and_delete_elem()
1592 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_map_lookup_and_delete_elem()
1600 	l = lookup_elem_raw(head, hash, key, key_size);  in __htab_map_lookup_and_delete_elem()
1611 		pptr = htab_elem_get_ptr(l, key_size);  in __htab_map_lookup_and_delete_elem()
1618 		u32 roundup_key_size = round_up(map->key_size, 8);  in __htab_map_lookup_and_delete_elem()
1683 	u32 bucket_cnt, total, key_size, value_size, roundup_key_size;  in __htab_map_lookup_and_delete_batch()  local
1722 	key_size = htab->map.key_size;  in __htab_map_lookup_and_delete_batch()
1723 	roundup_key_size = round_up(htab->map.key_size, 8);  in __htab_map_lookup_and_delete_batch()
1738 	keys = kvmalloc_array(key_size, bucket_size, GFP_USER | __GFP_NOWARN);  in __htab_map_lookup_and_delete_batch()
1802 		memcpy(dst_key, l->key, key_size);  in __htab_map_lookup_and_delete_batch()
1808 			pptr = htab_elem_get_ptr(l, map->key_size);  in __htab_map_lookup_and_delete_batch()
1848 		dst_key += key_size;  in __htab_map_lookup_and_delete_batch()
1875 	if (bucket_cnt && (copy_to_user(ukeys + total * key_size, keys,  in __htab_map_lookup_and_delete_batch()
1876 	    key_size * bucket_cnt) ||  in __htab_map_lookup_and_delete_batch()
2080 			roundup_key_size = round_up(map->key_size, 8);  in __bpf_hash_map_seq_show()
2086 				pptr = htab_elem_get_ptr(elem, map->key_size);  in __bpf_hash_map_seq_show()
2183 	roundup_key_size = round_up(map->key_size, 8);  in bpf_for_each_hash_elem()
2196 				pptr = htab_elem_get_ptr(elem, map->key_size);  in bpf_for_each_hash_elem()
2304 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_percpu_map_lookup_elem()
2322 				offsetof(struct htab_elem, key) + roundup(map->key_size, 8));  in htab_percpu_map_gen_lookup()
2338 		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);  in htab_percpu_map_lookup_percpu_elem()
2349 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_lru_percpu_map_lookup_elem()
2365 		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);  in htab_lru_percpu_map_lookup_percpu_elem()
2391 	pptr = htab_elem_get_ptr(l, map->key_size);  in bpf_percpu_hash_copy()
2438 	pptr = htab_elem_get_ptr(l, map->key_size);  in htab_percpu_map_seq_show_elem()
2605 				round_up(map->key_size, 8));  in htab_of_map_gen_lookup()