101 void __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node);
119 				  struct rte_node *node, uint16_t req_size);
137 	struct rte_node *node;  in rte_graph_walk()  local
158 		node = (struct rte_node *)RTE_PTR_ADD(graph, cir_start[(int32_t)head++]);  in rte_graph_walk()
159 		RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);  in rte_graph_walk()
160 		objs = node->objs;  in rte_graph_walk()
165 			rc = node->process(graph, node, objs, node->idx);  in rte_graph_walk()
166 			node->total_cycles += rte_rdtsc() - start;  in rte_graph_walk()
167 			node->total_calls++;  in rte_graph_walk()
168 			node->total_objs += rc;  in rte_graph_walk()
170 			node->process(graph, node, objs, node->idx);  in rte_graph_walk()
172 		node->idx = 0;  in rte_graph_walk()
191 __rte_node_enqueue_tail_update(struct rte_graph *graph, struct rte_node *node)  in __rte_node_enqueue_tail_update()  argument
196 	graph->cir_start[tail++] = node->off;  in __rte_node_enqueue_tail_update()
218 __rte_node_enqueue_prologue(struct rte_graph *graph, struct rte_node *node,  in __rte_node_enqueue_prologue()  argument
224 		__rte_node_enqueue_tail_update(graph, node);  in __rte_node_enqueue_prologue()
226 	if (unlikely(node->size < (idx + space)))  in __rte_node_enqueue_prologue()
227 		__rte_node_stream_alloc(graph, node);  in __rte_node_enqueue_prologue()
244 __rte_node_next_node_get(struct rte_node *node, rte_edge_t next)  in __rte_node_next_node_get()  argument
246 	RTE_ASSERT(next < node->nb_edges);  in __rte_node_next_node_get()
247 	RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);  in __rte_node_next_node_get()
248 	node = node->nodes[next];  in __rte_node_next_node_get()
249 	RTE_ASSERT(node->fence == RTE_GRAPH_FENCE);  in __rte_node_next_node_get()
251 	return node;  in __rte_node_next_node_get()
271 rte_node_enqueue(struct rte_graph *graph, struct rte_node *node,  in rte_node_enqueue()  argument
274 	node = __rte_node_next_node_get(node, next);  in rte_node_enqueue()
275 	const uint16_t idx = node->idx;  in rte_node_enqueue()
277 	__rte_node_enqueue_prologue(graph, node, idx, nb_objs);  in rte_node_enqueue()
279 	rte_memcpy(&node->objs[idx], objs, nb_objs * sizeof(void *));  in rte_node_enqueue()
280 	node->idx = idx + nb_objs;  in rte_node_enqueue()
298 rte_node_enqueue_x1(struct rte_graph *graph, struct rte_node *node,  in rte_node_enqueue_x1()  argument
301 	node = __rte_node_next_node_get(node, next);  in rte_node_enqueue_x1()
302 	uint16_t idx = node->idx;  in rte_node_enqueue_x1()
304 	__rte_node_enqueue_prologue(graph, node, idx, 1);  in rte_node_enqueue_x1()
306 	node->objs[idx++] = obj;  in rte_node_enqueue_x1()
307 	node->idx = idx;  in rte_node_enqueue_x1()
328 rte_node_enqueue_x2(struct rte_graph *graph, struct rte_node *node,  in rte_node_enqueue_x2()  argument
331 	node = __rte_node_next_node_get(node, next);  in rte_node_enqueue_x2()
332 	uint16_t idx = node->idx;  in rte_node_enqueue_x2()
334 	__rte_node_enqueue_prologue(graph, node, idx, 2);  in rte_node_enqueue_x2()
336 	node->objs[idx++] = obj0;  in rte_node_enqueue_x2()
337 	node->objs[idx++] = obj1;  in rte_node_enqueue_x2()
338 	node->idx = idx;  in rte_node_enqueue_x2()
363 rte_node_enqueue_x4(struct rte_graph *graph, struct rte_node *node,  in rte_node_enqueue_x4()  argument
367 	node = __rte_node_next_node_get(node, next);  in rte_node_enqueue_x4()
368 	uint16_t idx = node->idx;  in rte_node_enqueue_x4()
370 	__rte_node_enqueue_prologue(graph, node, idx, 4);  in rte_node_enqueue_x4()
372 	node->objs[idx++] = obj0;  in rte_node_enqueue_x4()
373 	node->objs[idx++] = obj1;  in rte_node_enqueue_x4()
374 	node->objs[idx++] = obj2;  in rte_node_enqueue_x4()
375 	node->objs[idx++] = obj3;  in rte_node_enqueue_x4()
376 	node->idx = idx;  in rte_node_enqueue_x4()
397 rte_node_enqueue_next(struct rte_graph *graph, struct rte_node *node,  in rte_node_enqueue_next()  argument
403 		rte_node_enqueue_x1(graph, node, nexts[i], objs[i]);  in rte_node_enqueue_next()
427 rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node,  in rte_node_next_stream_get()  argument
430 	node = __rte_node_next_node_get(node, next);  in rte_node_next_stream_get()
431 	const uint16_t idx = node->idx;  in rte_node_next_stream_get()
432 	uint16_t free_space = node->size - idx;  in rte_node_next_stream_get()
435 		__rte_node_stream_alloc_size(graph, node, nb_objs);  in rte_node_next_stream_get()
437 	return &node->objs[idx];  in rte_node_next_stream_get()
458 rte_node_next_stream_put(struct rte_graph *graph, struct rte_node *node,  in rte_node_next_stream_put()  argument
464 	node = __rte_node_next_node_get(node, next);  in rte_node_next_stream_put()
465 	if (node->idx == 0)  in rte_node_next_stream_put()
466 		__rte_node_enqueue_tail_update(graph, node);  in rte_node_next_stream_put()
468 	node->idx += idx;  in rte_node_next_stream_put()