27 	struct rb_node node;  member
33 #define fetch_node(x) rb_entry(READ_ONCE(x), typeof(struct active_node), node)
48 static inline struct llist_node *barrier_to_ll(struct active_node *node)  in barrier_to_ll()  argument
50 	GEM_BUG_ON(!is_barrier(&node->base));  in barrier_to_ll()
51 	return (struct llist_node *)&node->base.cb.node;  in barrier_to_ll()
55 __barrier_to_engine(struct active_node *node)  in __barrier_to_engine()  argument
57 	return (struct intel_engine_cs *)READ_ONCE(node->base.cb.node.prev);  in __barrier_to_engine()
61 barrier_to_engine(struct active_node *node)  in barrier_to_engine()  argument
63 	GEM_BUG_ON(!is_barrier(&node->base));  in barrier_to_engine()
64 	return __barrier_to_engine(node);  in barrier_to_engine()
70 			    struct active_node, base.cb.node);  in barrier_from_ll()
148 		rb_erase(&ref->cache->node, &ref->tree);  in __active_retire()
152 		rb_link_node(&ref->cache->node, NULL, &ref->tree.rb_node);  in __active_retire()
153 		rb_insert_color(&ref->cache->node, &ref->tree);  in __active_retire()
154 		GEM_BUG_ON(ref->tree.rb_node != &ref->cache->node);  in __active_retire()
170 	rbtree_postorder_for_each_entry_safe(it, n, &root, node) {  in __active_retire()
267 	BUILD_BUG_ON(offsetof(typeof(*it), node));  in __active_lookup()
275 			it = fetch_node(it->node.rb_right);  in __active_lookup()
277 			it = fetch_node(it->node.rb_left);  in __active_lookup()
291 	struct active_node *node;  in active_instance()  local
294 	node = __active_lookup(ref, idx);  in active_instance()
295 	if (likely(node))  in active_instance()
296 		return &node->base;  in active_instance()
306 		node = rb_entry(parent, struct active_node, node);  in active_instance()
307 		if (node->timeline == idx)  in active_instance()
310 		if (node->timeline < idx)  in active_instance()
320 	node = kmem_cache_alloc(slab_cache, GFP_ATOMIC);  in active_instance()
321 	if (!node)  in active_instance()
324 	__i915_active_fence_init(&node->base, NULL, node_retire);  in active_instance()
325 	node->ref = ref;  in active_instance()
326 	node->timeline = idx;  in active_instance()
328 	rb_link_node(&node->node, parent, p);  in active_instance()
329 	rb_insert_color(&node->node, &ref->tree);  in active_instance()
332 	WRITE_ONCE(ref->cache, node);  in active_instance()
335 	return &node->base;  in active_instance()
366 				   struct active_node *node,  in ____active_del_barrier()  argument
373 	GEM_BUG_ON(node->timeline != engine->kernel_context->timeline->fence_context);  in ____active_del_barrier()
391 		if (node == barrier_from_ll(pos)) {  in ____active_del_barrier()
392 			node = NULL;  in ____active_del_barrier()
404 	return !node;  in ____active_del_barrier()
408 __active_del_barrier(struct i915_active *ref, struct active_node *node)  in __active_del_barrier()  argument
410 	return ____active_del_barrier(ref, node, barrier_to_engine(node));  in __active_del_barrier()
572 	rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {  in flush_lazy_signals()
693 		rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {  in await_active()
751 static inline bool is_idle_barrier(struct active_node *node, u64 idx)  in is_idle_barrier()  argument
753 	return node->timeline == idx && !i915_active_fence_isset(&node->base);  in is_idle_barrier()
773 		p = &ref->cache->node;  in reuse_idle_barrier()
780 		struct active_node *node =  in reuse_idle_barrier()  local
781 			rb_entry(p, struct active_node, node);  in reuse_idle_barrier()
783 		if (is_idle_barrier(node, idx))  in reuse_idle_barrier()
787 		if (node->timeline < idx)  in reuse_idle_barrier()
800 		struct active_node *node =  in reuse_idle_barrier()  local
801 			rb_entry(p, struct active_node, node);  in reuse_idle_barrier()
804 		if (node->timeline > idx)  in reuse_idle_barrier()
807 		if (node->timeline < idx)  in reuse_idle_barrier()
810 		if (is_idle_barrier(node, idx))  in reuse_idle_barrier()
820 		engine = __barrier_to_engine(node);  in reuse_idle_barrier()
822 		if (is_barrier(&node->base) &&  in reuse_idle_barrier()
823 		    ____active_del_barrier(ref, node, engine))  in reuse_idle_barrier()
832 	if (p == &ref->cache->node)  in reuse_idle_barrier()
836 	return rb_entry(p, struct active_node, node);  in reuse_idle_barrier()
862 		struct active_node *node;  in i915_active_acquire_preallocate_barrier()  local
865 		node = reuse_idle_barrier(ref, idx);  in i915_active_acquire_preallocate_barrier()
867 		if (!node) {  in i915_active_acquire_preallocate_barrier()
868 			node = kmem_cache_alloc(slab_cache, GFP_KERNEL);  in i915_active_acquire_preallocate_barrier()
869 			if (!node)  in i915_active_acquire_preallocate_barrier()
872 			RCU_INIT_POINTER(node->base.fence, NULL);  in i915_active_acquire_preallocate_barrier()
873 			node->base.cb.func = node_retire;  in i915_active_acquire_preallocate_barrier()
874 			node->timeline = idx;  in i915_active_acquire_preallocate_barrier()
875 			node->ref = ref;  in i915_active_acquire_preallocate_barrier()
878 		if (!i915_active_fence_isset(&node->base)) {  in i915_active_acquire_preallocate_barrier()
888 			RCU_INIT_POINTER(node->base.fence, ERR_PTR(-EAGAIN));  in i915_active_acquire_preallocate_barrier()
889 			node->base.cb.node.prev = (void *)engine;  in i915_active_acquire_preallocate_barrier()
892 		GEM_BUG_ON(rcu_access_pointer(node->base.fence) != ERR_PTR(-EAGAIN));  in i915_active_acquire_preallocate_barrier()
894 		GEM_BUG_ON(barrier_to_engine(node) != engine);  in i915_active_acquire_preallocate_barrier()
895 		first = barrier_to_ll(node);  in i915_active_acquire_preallocate_barrier()
909 		struct active_node *node = barrier_from_ll(first);  in i915_active_acquire_preallocate_barrier()  local
914 		intel_engine_pm_put(barrier_to_engine(node));  in i915_active_acquire_preallocate_barrier()
916 		kmem_cache_free(slab_cache, node);  in i915_active_acquire_preallocate_barrier()
935 		struct active_node *node = barrier_from_ll(pos);  in i915_active_acquire_barrier()  local
936 		struct intel_engine_cs *engine = barrier_to_engine(node);  in i915_active_acquire_barrier()
948 			it = rb_entry(parent, struct active_node, node);  in i915_active_acquire_barrier()
949 			if (it->timeline < node->timeline)  in i915_active_acquire_barrier()
954 		rb_link_node(&node->node, parent, p);  in i915_active_acquire_barrier()
955 		rb_insert_color(&node->node, &ref->tree);  in i915_active_acquire_barrier()
959 		llist_add(barrier_to_ll(node), &engine->barrier_tasks);  in i915_active_acquire_barrier()
964 static struct dma_fence **ll_to_fence_slot(struct llist_node *node)  in ll_to_fence_slot()  argument
966 	return __active_fence_slot(&barrier_from_ll(node)->base);  in ll_to_fence_slot()
972 	struct llist_node *node, *next;  in i915_request_add_active_barriers()  local
979 	node = llist_del_all(&engine->barrier_tasks);  in i915_request_add_active_barriers()
980 	if (!node)  in i915_request_add_active_barriers()
988 	llist_for_each_safe(node, next, node) {  in i915_request_add_active_barriers()
990 		smp_store_mb(*ll_to_fence_slot(node), &rq->fence);  in i915_request_add_active_barriers()
991 		list_add_tail((struct list_head *)node, &rq->fence.cb_list);  in i915_request_add_active_barriers()
1091 		__list_del_entry(&active->cb.node);  in __i915_active_fence_set()
1094 	list_add_tail(&active->cb.node, &fence->cb_list);  in __i915_active_fence_set()