168 	struct rte_table_hash *t;  in rte_table_hash_ext_create()  local
220 	t = rte_zmalloc_socket(p->name,  in rte_table_hash_ext_create()
224 	if (t == NULL) {  in rte_table_hash_ext_create()
235 	t->key_size = p->key_size;  in rte_table_hash_ext_create()
236 	t->entry_size = entry_size;  in rte_table_hash_ext_create()
237 	t->n_keys = p->n_keys;  in rte_table_hash_ext_create()
238 	t->n_buckets = p->n_buckets;  in rte_table_hash_ext_create()
239 	t->n_buckets_ext = n_buckets_ext;  in rte_table_hash_ext_create()
240 	t->f_hash = p->f_hash;  in rte_table_hash_ext_create()
241 	t->seed = p->seed;  in rte_table_hash_ext_create()
242 	t->key_offset = p->key_offset;  in rte_table_hash_ext_create()
245 	t->bucket_mask = t->n_buckets - 1;  in rte_table_hash_ext_create()
246 	t->key_size_shl = __builtin_ctzl(p->key_size);  in rte_table_hash_ext_create()
247 	t->data_size_shl = __builtin_ctzl(entry_size);  in rte_table_hash_ext_create()
258 	t->key_mask = (uint64_t *) &t->memory[key_mask_offset];  in rte_table_hash_ext_create()
259 	t->buckets = (struct bucket *) &t->memory[bucket_offset];  in rte_table_hash_ext_create()
260 	t->buckets_ext = (struct bucket *) &t->memory[bucket_ext_offset];  in rte_table_hash_ext_create()
261 	t->key_mem = &t->memory[key_offset];  in rte_table_hash_ext_create()
262 	t->key_stack = (uint32_t *) &t->memory[key_stack_offset];  in rte_table_hash_ext_create()
263 	t->bkt_ext_stack = (uint32_t *) &t->memory[bkt_ext_stack_offset];  in rte_table_hash_ext_create()
264 	t->data_mem = &t->memory[data_offset];  in rte_table_hash_ext_create()
268 		memset(t->key_mask, 0xFF, p->key_size);  in rte_table_hash_ext_create()
270 		memcpy(t->key_mask, p->key_mask, p->key_size);  in rte_table_hash_ext_create()
273 	for (i = 0; i < t->n_keys; i++)  in rte_table_hash_ext_create()
274 		t->key_stack[i] = t->n_keys - 1 - i;  in rte_table_hash_ext_create()
275 	t->key_stack_tos = t->n_keys;  in rte_table_hash_ext_create()
278 	for (i = 0; i < t->n_buckets_ext; i++)  in rte_table_hash_ext_create()
279 		t->bkt_ext_stack[i] = t->n_buckets_ext - 1 - i;  in rte_table_hash_ext_create()
280 	t->bkt_ext_stack_tos = t->n_buckets_ext;  in rte_table_hash_ext_create()
282 	return t;  in rte_table_hash_ext_create()
288 	struct rte_table_hash *t = table;  in rte_table_hash_ext_free()  local
291 	if (t == NULL)  in rte_table_hash_ext_free()
294 	rte_free(t);  in rte_table_hash_ext_free()
302 	struct rte_table_hash *t = table;  in rte_table_hash_ext_entry_add()  local
307 	sig = t->f_hash(key, t->key_mask, t->key_size, t->seed);  in rte_table_hash_ext_entry_add()
308 	bkt_index = sig & t->bucket_mask;  in rte_table_hash_ext_entry_add()
309 	bkt0 = &t->buckets[bkt_index];  in rte_table_hash_ext_entry_add()
318 				&t->key_mem[bkt_key_index << t->key_size_shl];  in rte_table_hash_ext_entry_add()
320 			if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask,  in rte_table_hash_ext_entry_add()
321 				t->key_size) == 0)) {  in rte_table_hash_ext_entry_add()
322 				uint8_t *data = &t->data_mem[bkt_key_index <<  in rte_table_hash_ext_entry_add()
323 					t->data_size_shl];  in rte_table_hash_ext_entry_add()
325 				memcpy(data, entry, t->entry_size);  in rte_table_hash_ext_entry_add()
343 				if (t->key_stack_tos == 0) /* No free keys */  in rte_table_hash_ext_entry_add()
346 				bkt_key_index = t->key_stack[  in rte_table_hash_ext_entry_add()
347 					--t->key_stack_tos];  in rte_table_hash_ext_entry_add()
350 				bkt_key = &t->key_mem[bkt_key_index <<  in rte_table_hash_ext_entry_add()
351 					t->key_size_shl];  in rte_table_hash_ext_entry_add()
352 				data = &t->data_mem[bkt_key_index <<  in rte_table_hash_ext_entry_add()
353 					t->data_size_shl];  in rte_table_hash_ext_entry_add()
357 				keycpy(bkt_key, key, t->key_mask, t->key_size);  in rte_table_hash_ext_entry_add()
358 				memcpy(data, entry, t->entry_size);  in rte_table_hash_ext_entry_add()
367 	if ((t->bkt_ext_stack_tos > 0) && (t->key_stack_tos > 0)) {  in rte_table_hash_ext_entry_add()
372 		bkt_index = t->bkt_ext_stack[--t->bkt_ext_stack_tos];  in rte_table_hash_ext_entry_add()
373 		bkt = &t->buckets_ext[bkt_index];  in rte_table_hash_ext_entry_add()
380 		bkt_key_index = t->key_stack[--t->key_stack_tos];  in rte_table_hash_ext_entry_add()
381 		bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl];  in rte_table_hash_ext_entry_add()
383 		data = &t->data_mem[bkt_key_index << t->data_size_shl];  in rte_table_hash_ext_entry_add()
388 		keycpy(bkt_key, key, t->key_mask, t->key_size);  in rte_table_hash_ext_entry_add()
389 		memcpy(data, entry, t->entry_size);  in rte_table_hash_ext_entry_add()
403 	struct rte_table_hash *t = table;  in rte_table_hash_ext_entry_delete()  local
408 	sig = t->f_hash(key, t->key_mask, t->key_size, t->seed);  in rte_table_hash_ext_entry_delete()
409 	bkt_index = sig & t->bucket_mask;  in rte_table_hash_ext_entry_delete()
410 	bkt0 = &t->buckets[bkt_index];  in rte_table_hash_ext_entry_delete()
419 			uint8_t *bkt_key = &t->key_mem[bkt_key_index <<  in rte_table_hash_ext_entry_delete()
420 				t->key_size_shl];  in rte_table_hash_ext_entry_delete()
422 			if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask,  in rte_table_hash_ext_entry_delete()
423 				t->key_size) == 0)) {  in rte_table_hash_ext_entry_delete()
424 				uint8_t *data = &t->data_mem[bkt_key_index <<  in rte_table_hash_ext_entry_delete()
425 					t->data_size_shl];  in rte_table_hash_ext_entry_delete()
431 					memcpy(entry, data, t->entry_size);  in rte_table_hash_ext_entry_delete()
434 				t->key_stack[t->key_stack_tos++] =  in rte_table_hash_ext_entry_delete()
448 					bkt_index = bkt - t->buckets_ext;  in rte_table_hash_ext_entry_delete()
449 					t->bkt_ext_stack[t->bkt_ext_stack_tos++]  in rte_table_hash_ext_entry_delete()
469 	struct rte_table_hash *t = (struct rte_table_hash *) table;  in rte_table_hash_ext_lookup_unoptimized()  local
486 		key = RTE_MBUF_METADATA_UINT8_PTR(pkt, t->key_offset);  in rte_table_hash_ext_lookup_unoptimized()
487 		sig = (uint64_t) t->f_hash(key, t->key_mask, t->key_size, t->seed);  in rte_table_hash_ext_lookup_unoptimized()
489 		bkt_index = sig & t->bucket_mask;  in rte_table_hash_ext_lookup_unoptimized()
490 		bkt0 = &t->buckets[bkt_index];  in rte_table_hash_ext_lookup_unoptimized()
498 				uint8_t *bkt_key = &t->key_mem[bkt_key_index <<  in rte_table_hash_ext_lookup_unoptimized()
499 					t->key_size_shl];  in rte_table_hash_ext_lookup_unoptimized()
502 					t->key_mask, t->key_size) == 0)) {  in rte_table_hash_ext_lookup_unoptimized()
503 					uint8_t *data = &t->data_mem[  in rte_table_hash_ext_lookup_unoptimized()
504 					bkt_key_index << t->data_size_shl];  in rte_table_hash_ext_lookup_unoptimized()
668 #define lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index)	\  argument
672 	uint32_t key_offset = t->key_offset;			\
688 #define lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask, pkt00_index, \  argument
693 	uint32_t key_offset = t->key_offset;			\
711 #define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index)	\  argument
716 	struct bucket *bkt10, *bkt11, *buckets = t->buckets;		\
718 	uint64_t bucket_mask = t->bucket_mask;				\
719 	rte_table_hash_op_hash f_hash = t->f_hash;			\
720 	uint64_t seed = t->seed;					\
721 	uint32_t key_size = t->key_size;				\
722 	uint32_t key_offset = t->key_offset;				\
726 	sig10 = (uint64_t) f_hash(key10, t->key_mask, key_size, seed);	\
732 	sig11 = (uint64_t) f_hash(key11, t->key_mask, key_size, seed);	\
748 #define lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many)\  argument
753 	uint8_t *key20, *key21, *key_mem = t->key_mem;			\
756 	uint32_t key20_index, key21_index, key_size_shl = t->key_size_shl;\
792 #define lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, \  argument
797 	uint8_t *key30, *key31, *key_mem = t->key_mem;			\
798 	uint8_t *data30, *data31, *data_mem = t->data_mem;		\
801 	uint32_t key_size_shl = t->key_size_shl;			\
802 	uint32_t data_size_shl = t->data_size_shl;			\
809 	lookup_cmp_key(mbuf30, key30, match_key30, t);			\
820 	lookup_cmp_key(mbuf31, key31, match_key31, t);			\
856 	struct rte_table_hash *t = (struct rte_table_hash *) table;  in rte_table_hash_ext_lookup()  local
857 	struct grinder *g = t->grinders;  in rte_table_hash_ext_lookup()
864 	RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(t, n_pkts_in);  in rte_table_hash_ext_lookup()
870 		RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in -  in rte_table_hash_ext_lookup()
876 	lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);  in rte_table_hash_ext_lookup()
883 	lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);  in rte_table_hash_ext_lookup()
886 	lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);  in rte_table_hash_ext_lookup()
895 	lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);  in rte_table_hash_ext_lookup()
898 	lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);  in rte_table_hash_ext_lookup()
901 	lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);  in rte_table_hash_ext_lookup()
917 		lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask,  in rte_table_hash_ext_lookup()
921 		lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);  in rte_table_hash_ext_lookup()
924 		lookup2_stage2(t, g, pkt20_index, pkt21_index,  in rte_table_hash_ext_lookup()
928 		lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index,  in rte_table_hash_ext_lookup()
941 	lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);  in rte_table_hash_ext_lookup()
944 	lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);  in rte_table_hash_ext_lookup()
947 	lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,  in rte_table_hash_ext_lookup()
957 	lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);  in rte_table_hash_ext_lookup()
960 	lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,  in rte_table_hash_ext_lookup()
968 	lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,  in rte_table_hash_ext_lookup()
982 	RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));  in rte_table_hash_ext_lookup()
989 	struct rte_table_hash *t = table;  in rte_table_hash_ext_stats_read()  local
992 		memcpy(stats, &t->stats, sizeof(t->stats));  in rte_table_hash_ext_stats_read()
995 		memset(&t->stats, 0, sizeof(t->stats));  in rte_table_hash_ext_stats_read()