30 #define EFD_KEY(key_idx, table) (table->keys + ((key_idx) * table->key_len))  argument
32 #define EFD_HASH(key, table) \  argument
33 	(uint32_t)(rte_jhash(key, table->key_len, 0xbc9f1d34))
35 #define EFD_HASHFUNCA(key, table) \  argument
36 	(uint32_t)(rte_hash_crc(key, table->key_len, 0xbc9f1d35))
38 #define EFD_HASHFUNCB(key, table) \  argument
39 	(uint32_t)(rte_hash_crc(key, table->key_len, 0xbc9f1d36))
273 efd_get_chunk_id(const struct rte_efd_table * const table,  in efd_get_chunk_id()  argument
276 	return hashed_key & (table->num_chunks - 1);  in efd_get_chunk_id()
290 efd_get_bin_id(const struct rte_efd_table * const table,  in efd_get_bin_id()  argument
293 	return (hashed_key >> table->num_chunks_shift) & (EFD_CHUNK_NUM_BINS - 1);  in efd_get_bin_id()
312 efd_get_choice(const struct rte_efd_table * const table,  in efd_get_choice()  argument
316 	struct efd_online_chunk *chunk = &table->chunks[socket_id][chunk_id];  in efd_get_choice()
349 efd_compute_ids(const struct rte_efd_table * const table,  in efd_compute_ids()  argument
353 	uint32_t h = EFD_HASH(key, table);  in efd_compute_ids()
356 	*chunk_id = efd_get_chunk_id(table, h);  in efd_compute_ids()
362 	*bin_id = efd_get_bin_id(table, h);  in efd_compute_ids()
369 efd_search_hash(struct rte_efd_table * const table,  in efd_search_hash()  argument
390 		void *key_stored = EFD_KEY(off_group->key_idx[i], table);  in efd_search_hash()
391 		hash_val_b[i] = EFD_HASHFUNCB(key_stored, table);  in efd_search_hash()
392 		hash_val_a[i] = EFD_HASHFUNCA(key_stored, table);  in efd_search_hash()
497 	struct rte_efd_table *table = NULL;  in rte_efd_create()  local
542 		table = (struct rte_efd_table *) te->data;  in rte_efd_create()
543 		if (strncmp(name, table->name, RTE_EFD_NAMESIZE) == 0)  in rte_efd_create()
547 	table = NULL;  in rte_efd_create()
561 	table = rte_zmalloc_socket(NULL,  in rte_efd_create()
565 	if (table == NULL) {  in rte_efd_create()
576 	table->max_num_rules = num_chunks * EFD_TARGET_CHUNK_MAX_NUM_RULES;  in rte_efd_create()
577 	table->num_rules = 0;  in rte_efd_create()
578 	table->num_chunks = num_chunks;  in rte_efd_create()
579 	table->num_chunks_shift = num_chunks_shift;  in rte_efd_create()
580 	table->key_len = key_len;  in rte_efd_create()
584 			table->max_num_rules * table->key_len,  in rte_efd_create()
593 	table->keys = key_array;  in rte_efd_create()
594 	strlcpy(table->name, name, sizeof(table->name));  in rte_efd_create()
598 			num_chunks, table->max_num_rules);  in rte_efd_create()
602 		table->chunks[socket_id] = NULL;  in rte_efd_create()
603 	table->offline_chunks = NULL;  in rte_efd_create()
618 			table->chunks[socket_id] =  in rte_efd_create()
624 			if (table->chunks[socket_id] == NULL) {  in rte_efd_create()
649 		table->lookup_fn = EFD_LOOKUP_AVX2;  in rte_efd_create()
660 		table->lookup_fn = EFD_LOOKUP_NEON;  in rte_efd_create()
663 		table->lookup_fn = EFD_LOOKUP_SCALAR;  in rte_efd_create()
671 	table->offline_chunks =  in rte_efd_create()
676 	if (table->offline_chunks == NULL) {  in rte_efd_create()
688 	te->data = (void *) table;  in rte_efd_create()
692 	snprintf(ring_name, sizeof(ring_name), "HT_%s", table->name);  in rte_efd_create()
694 	r = rte_ring_create(ring_name, rte_align32pow2(table->max_num_rules),  in rte_efd_create()
698 		rte_efd_free(table);  in rte_efd_create()
703 	for (i = 0; i < table->max_num_rules; i++)  in rte_efd_create()
706 	table->free_slots = r;  in rte_efd_create()
707 	return table;  in rte_efd_create()
712 	rte_efd_free(table);  in rte_efd_create()
720 	struct rte_efd_table *table = NULL;  in rte_efd_find_existing()  local
730 		table = (struct rte_efd_table *) te->data;  in rte_efd_find_existing()
731 		if (strncmp(name, table->name, RTE_EFD_NAMESIZE) == 0)  in rte_efd_find_existing()
740 	return table;  in rte_efd_find_existing()
744 rte_efd_free(struct rte_efd_table *table)  in rte_efd_free()  argument
750 	if (table == NULL)  in rte_efd_free()
754 		rte_free(table->chunks[socket_id]);  in rte_efd_free()
760 		if (te->data == (void *) table) {  in rte_efd_free()
768 	rte_ring_free(table->free_slots);  in rte_efd_free()
769 	rte_free(table->offline_chunks);  in rte_efd_free()
770 	rte_free(table->keys);  in rte_efd_free()
771 	rte_free(table);  in rte_efd_free()
796 efd_apply_update(struct rte_efd_table * const table, const unsigned int socket_id,  in efd_apply_update()  argument
802 	struct efd_online_chunk *chunk = &table->chunks[socket_id][chunk_id];  in efd_apply_update()
822 		if (table->chunks[i] != NULL) {  in efd_apply_update()
823 			memcpy(&(table->chunks[i][chunk_id].groups[group_id]),  in efd_apply_update()
826 			table->chunks[i][chunk_id].bin_choice_list[bin_index] =  in efd_apply_update()
957 efd_compute_update(struct rte_efd_table * const table,  in efd_compute_update()  argument
971 	efd_compute_ids(table, key, chunk_id, bin_id);  in efd_compute_update()
974 			&table->offline_chunks[*chunk_id];  in efd_compute_update()
977 	uint8_t current_choice = efd_get_choice(table, socket_id,  in efd_compute_update()
994 		void *key_stored = EFD_KEY(current_group->key_idx[i], table);  in efd_compute_update()
996 				table->key_len) == 0)) {  in efd_compute_update()
1034 		if (rte_ring_sc_dequeue(table->free_slots, &slot_id) != 0)  in efd_compute_update()
1037 		new_k = RTE_PTR_ADD(table->keys, (uintptr_t) slot_id *  in efd_compute_update()
1038 					table->key_len);  in efd_compute_update()
1042 		rte_memcpy(EFD_KEY(new_idx, table), key, table->key_len);  in efd_compute_update()
1047 		table->num_rules++;  in efd_compute_update()
1129 		ret = efd_search_hash(table, new_group, entry);  in efd_compute_update()
1152 		table->num_rules--;  in efd_compute_update()
1160 rte_efd_update(struct rte_efd_table * const table, const unsigned int socket_id,  in rte_efd_update()  argument
1167 	int status = efd_compute_update(table, socket_id, key, value,  in rte_efd_update()
1177 	efd_apply_update(table, socket_id, chunk_id, group_id, bin_id,  in rte_efd_update()
1183 rte_efd_delete(struct rte_efd_table * const table, const unsigned int socket_id,  in rte_efd_delete()  argument
1190 	efd_compute_ids(table, key, &chunk_id, &bin_id);  in rte_efd_delete()
1193 			&table->offline_chunks[chunk_id];  in rte_efd_delete()
1195 	uint8_t current_choice = efd_get_choice(table, socket_id,  in rte_efd_delete()
1208 			if (memcmp(EFD_KEY(current_group->key_idx[i], table),  in rte_efd_delete()
1209 					key, table->key_len) == 0) {  in rte_efd_delete()
1215 				rte_ring_sp_enqueue(table->free_slots,  in rte_efd_delete()
1232 		table->num_rules--;  in rte_efd_delete()
1299 rte_efd_lookup(const struct rte_efd_table * const table,  in rte_efd_lookup()  argument
1305 	const struct efd_online_chunk * const chunks = table->chunks[socket_id];  in rte_efd_lookup()
1308 	efd_compute_ids(table, key, &chunk_id, &bin_id);  in rte_efd_lookup()
1309 	bin_choice = efd_get_choice(table, socket_id, chunk_id, bin_id);  in rte_efd_lookup()
1314 			EFD_HASHFUNCA(key, table),  in rte_efd_lookup()
1315 			EFD_HASHFUNCB(key, table),  in rte_efd_lookup()
1316 			table->lookup_fn);  in rte_efd_lookup()
1319 void rte_efd_lookup_bulk(const struct rte_efd_table * const table,  in rte_efd_lookup_bulk()  argument
1330 	struct efd_online_chunk *chunks = table->chunks[socket_id];  in rte_efd_lookup_bulk()
1333 		efd_compute_ids(table, key_list[i], &chunk_id_list[i],  in rte_efd_lookup_bulk()
1339 		bin_choice_list[i] = efd_get_choice(table, socket_id,  in rte_efd_lookup_bulk()
1350 				EFD_HASHFUNCA(key_list[i], table),  in rte_efd_lookup_bulk()
1351 				EFD_HASHFUNCB(key_list[i], table),  in rte_efd_lookup_bulk()
1352 				table->lookup_fn);  in rte_efd_lookup_bulk()