33 #define EFD_KEY(key_idx, table) (table->keys + ((key_idx) * table->key_len))  argument
35 #define EFD_HASH(key, table) \  argument
36 	(uint32_t)(rte_jhash(key, table->key_len, 0xbc9f1d34))
38 #define EFD_HASHFUNCA(key, table) \  argument
39 	(uint32_t)(rte_hash_crc(key, table->key_len, 0xbc9f1d35))
41 #define EFD_HASHFUNCB(key, table) \  argument
42 	(uint32_t)(rte_hash_crc(key, table->key_len, 0xbc9f1d36))
276 efd_get_chunk_id(const struct rte_efd_table * const table,  in efd_get_chunk_id()  argument
279 	return hashed_key & (table->num_chunks - 1);  in efd_get_chunk_id()
293 efd_get_bin_id(const struct rte_efd_table * const table,  in efd_get_bin_id()  argument
296 	return (hashed_key >> table->num_chunks_shift) & (EFD_CHUNK_NUM_BINS - 1);  in efd_get_bin_id()
315 efd_get_choice(const struct rte_efd_table * const table,  in efd_get_choice()  argument
319 	struct efd_online_chunk *chunk = &table->chunks[socket_id][chunk_id];  in efd_get_choice()
352 efd_compute_ids(const struct rte_efd_table * const table,  in efd_compute_ids()  argument
356 	uint32_t h = EFD_HASH(key, table);  in efd_compute_ids()
359 	*chunk_id = efd_get_chunk_id(table, h);  in efd_compute_ids()
365 	*bin_id = efd_get_bin_id(table, h);  in efd_compute_ids()
372 efd_search_hash(struct rte_efd_table * const table,  in efd_search_hash()  argument
393 		void *key_stored = EFD_KEY(off_group->key_idx[i], table);  in efd_search_hash()
394 		hash_val_b[i] = EFD_HASHFUNCB(key_stored, table);  in efd_search_hash()
395 		hash_val_a[i] = EFD_HASHFUNCA(key_stored, table);  in efd_search_hash()
500 	struct rte_efd_table *table = NULL;  in rte_efd_create()  local
545 		table = (struct rte_efd_table *) te->data;  in rte_efd_create()
546 		if (strncmp(name, table->name, RTE_EFD_NAMESIZE) == 0)  in rte_efd_create()
550 	table = NULL;  in rte_efd_create()
564 	table = rte_zmalloc_socket(NULL,  in rte_efd_create()
568 	if (table == NULL) {  in rte_efd_create()
579 	table->max_num_rules = num_chunks * EFD_TARGET_CHUNK_MAX_NUM_RULES;  in rte_efd_create()
580 	table->num_rules = 0;  in rte_efd_create()
581 	table->num_chunks = num_chunks;  in rte_efd_create()
582 	table->num_chunks_shift = num_chunks_shift;  in rte_efd_create()
583 	table->key_len = key_len;  in rte_efd_create()
587 			table->max_num_rules * table->key_len,  in rte_efd_create()
596 	table->keys = key_array;  in rte_efd_create()
597 	strlcpy(table->name, name, sizeof(table->name));  in rte_efd_create()
601 			num_chunks, table->max_num_rules);  in rte_efd_create()
605 		table->chunks[socket_id] = NULL;  in rte_efd_create()
606 	table->offline_chunks = NULL;  in rte_efd_create()
621 			table->chunks[socket_id] =  in rte_efd_create()
627 			if (table->chunks[socket_id] == NULL) {  in rte_efd_create()
652 		table->lookup_fn = EFD_LOOKUP_AVX2;  in rte_efd_create()
663 		table->lookup_fn = EFD_LOOKUP_NEON;  in rte_efd_create()
666 		table->lookup_fn = EFD_LOOKUP_SCALAR;  in rte_efd_create()
674 	table->offline_chunks =  in rte_efd_create()
679 	if (table->offline_chunks == NULL) {  in rte_efd_create()
691 	te->data = (void *) table;  in rte_efd_create()
695 	snprintf(ring_name, sizeof(ring_name), "HT_%s", table->name);  in rte_efd_create()
697 	r = rte_ring_create(ring_name, rte_align32pow2(table->max_num_rules),  in rte_efd_create()
701 		rte_efd_free(table);  in rte_efd_create()
706 	for (i = 0; i < table->max_num_rules; i++)  in rte_efd_create()
709 	table->free_slots = r;  in rte_efd_create()
710 	return table;  in rte_efd_create()
715 	rte_efd_free(table);  in rte_efd_create()
723 	struct rte_efd_table *table = NULL;  in rte_efd_find_existing()  local
733 		table = (struct rte_efd_table *) te->data;  in rte_efd_find_existing()
734 		if (strncmp(name, table->name, RTE_EFD_NAMESIZE) == 0)  in rte_efd_find_existing()
743 	return table;  in rte_efd_find_existing()
747 rte_efd_free(struct rte_efd_table *table)  in rte_efd_free()  argument
753 	if (table == NULL)  in rte_efd_free()
757 		rte_free(table->chunks[socket_id]);  in rte_efd_free()
763 		if (te->data == (void *) table) {  in rte_efd_free()
771 	rte_ring_free(table->free_slots);  in rte_efd_free()
772 	rte_free(table->offline_chunks);  in rte_efd_free()
773 	rte_free(table->keys);  in rte_efd_free()
774 	rte_free(table);  in rte_efd_free()
799 efd_apply_update(struct rte_efd_table * const table, const unsigned int socket_id,  in efd_apply_update()  argument
805 	struct efd_online_chunk *chunk = &table->chunks[socket_id][chunk_id];  in efd_apply_update()
825 		if (table->chunks[i] != NULL) {  in efd_apply_update()
826 			memcpy(&(table->chunks[i][chunk_id].groups[group_id]),  in efd_apply_update()
829 			table->chunks[i][chunk_id].bin_choice_list[bin_index] =  in efd_apply_update()
960 efd_compute_update(struct rte_efd_table * const table,  in efd_compute_update()  argument
974 	efd_compute_ids(table, key, chunk_id, bin_id);  in efd_compute_update()
977 			&table->offline_chunks[*chunk_id];  in efd_compute_update()
980 	uint8_t current_choice = efd_get_choice(table, socket_id,  in efd_compute_update()
997 		void *key_stored = EFD_KEY(current_group->key_idx[i], table);  in efd_compute_update()
999 				table->key_len) == 0)) {  in efd_compute_update()
1037 		if (rte_ring_sc_dequeue(table->free_slots, &slot_id) != 0)  in efd_compute_update()
1040 		new_k = RTE_PTR_ADD(table->keys, (uintptr_t) slot_id *  in efd_compute_update()
1041 					table->key_len);  in efd_compute_update()
1045 		rte_memcpy(EFD_KEY(new_idx, table), key, table->key_len);  in efd_compute_update()
1050 		table->num_rules++;  in efd_compute_update()
1132 		ret = efd_search_hash(table, new_group, entry);  in efd_compute_update()
1155 		table->num_rules--;  in efd_compute_update()
1163 rte_efd_update(struct rte_efd_table * const table, const unsigned int socket_id,  in rte_efd_update()  argument
1170 	int status = efd_compute_update(table, socket_id, key, value,  in rte_efd_update()
1180 	efd_apply_update(table, socket_id, chunk_id, group_id, bin_id,  in rte_efd_update()
1186 rte_efd_delete(struct rte_efd_table * const table, const unsigned int socket_id,  in rte_efd_delete()  argument
1193 	efd_compute_ids(table, key, &chunk_id, &bin_id);  in rte_efd_delete()
1196 			&table->offline_chunks[chunk_id];  in rte_efd_delete()
1198 	uint8_t current_choice = efd_get_choice(table, socket_id,  in rte_efd_delete()
1211 			if (memcmp(EFD_KEY(current_group->key_idx[i], table),  in rte_efd_delete()
1212 					key, table->key_len) == 0) {  in rte_efd_delete()
1218 				rte_ring_sp_enqueue(table->free_slots,  in rte_efd_delete()
1235 		table->num_rules--;  in rte_efd_delete()
1302 rte_efd_lookup(const struct rte_efd_table * const table,  in rte_efd_lookup()  argument
1308 	const struct efd_online_chunk * const chunks = table->chunks[socket_id];  in rte_efd_lookup()
1311 	efd_compute_ids(table, key, &chunk_id, &bin_id);  in rte_efd_lookup()
1312 	bin_choice = efd_get_choice(table, socket_id, chunk_id, bin_id);  in rte_efd_lookup()
1317 			EFD_HASHFUNCA(key, table),  in rte_efd_lookup()
1318 			EFD_HASHFUNCB(key, table),  in rte_efd_lookup()
1319 			table->lookup_fn);  in rte_efd_lookup()
1322 void rte_efd_lookup_bulk(const struct rte_efd_table * const table,  in rte_efd_lookup_bulk()  argument
1333 	struct efd_online_chunk *chunks = table->chunks[socket_id];  in rte_efd_lookup_bulk()
1336 		efd_compute_ids(table, key_list[i], &chunk_id_list[i],  in rte_efd_lookup_bulk()
1342 		bin_choice_list[i] = efd_get_choice(table, socket_id,  in rte_efd_lookup_bulk()
1353 				EFD_HASHFUNCA(key_list[i], table),  in rte_efd_lookup_bulk()
1354 				EFD_HASHFUNCB(key_list[i], table),  in rte_efd_lookup_bulk()
1355 				table->lookup_fn);  in rte_efd_lookup_bulk()