23 static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,
72 static inline struct mm_struct *mm_from_sdma_node(struct sdma_mmu_node *node)  in mm_from_sdma_node()  argument
74 	return node->rb.handler->mn.mm;  in mm_from_sdma_node()
77 static void free_system_node(struct sdma_mmu_node *node)  in free_system_node()  argument
79 	if (node->npages) {  in free_system_node()
80 		unpin_vector_pages(mm_from_sdma_node(node), node->pages, 0,  in free_system_node()
81 				   node->npages);  in free_system_node()
82 		atomic_sub(node->npages, &node->pq->n_locked);  in free_system_node()
84 	kfree(node);  in free_system_node()
116 			    struct sdma_mmu_node *node, int npages)  in pin_system_pages()  argument
137 		 start_address, node->npages, npages);  in pin_system_pages()
147 		unpin_vector_pages(current->mm, pages, node->npages, pinned);  in pin_system_pages()
151 	node->rb.addr = start_address;  in pin_system_pages()
152 	node->rb.len = length;  in pin_system_pages()
153 	node->pages = pages;  in pin_system_pages()
154 	node->npages = npages;  in pin_system_pages()
173 	struct sdma_mmu_node *node;  in add_system_pinning()  local
176 	node = kzalloc(sizeof(*node), GFP_KERNEL);  in add_system_pinning()
177 	if (!node)  in add_system_pinning()
181 	kref_init(&node->rb.refcount);  in add_system_pinning()
184 	kref_get(&node->rb.refcount);  in add_system_pinning()
186 	node->pq = pq;  in add_system_pinning()
187 	ret = pin_system_pages(req, start, len, node, PFN_DOWN(len));  in add_system_pinning()
189 		ret = hfi1_mmu_rb_insert(pq->handler, &node->rb);  in add_system_pinning()
191 			free_system_node(node);  in add_system_pinning()
193 			*node_p = node;  in add_system_pinning()
198 	kfree(node);  in add_system_pinning()
221 		struct sdma_mmu_node *node =  in get_system_cache_entry()  local
225 		SDMA_DBG(req, "node %p start %llx end %llu", node, start, end);  in get_system_cache_entry()
226 		if (!node) {  in get_system_cache_entry()
239 		if (node->rb.addr <= start) {  in get_system_cache_entry()
244 			*node_p = node;  in get_system_cache_entry()
249 			 node->rb.addr, kref_read(&node->rb.refcount));  in get_system_cache_entry()
250 		prepend_len = node->rb.addr - start;  in get_system_cache_entry()
256 		kref_put(&node->rb.refcount, hfi1_mmu_rb_release);  in get_system_cache_entry()
270 	struct mmu_rb_node *node = ctx;  in sdma_mmu_rb_node_get()  local
272 	kref_get(&node->refcount);  in sdma_mmu_rb_node_get()
277 	struct sdma_mmu_node *node = ctx;  in sdma_mmu_rb_node_put()  local
279 	kref_put(&node->rb.refcount, hfi1_mmu_rb_release);  in sdma_mmu_rb_node_put()
440 static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,  in sdma_rb_filter()  argument
443 	return (bool)(node->addr == addr);  in sdma_rb_filter()
454 	struct sdma_mmu_node *node =  in sdma_rb_evict()  local
459 	evict_data->cleared += node->npages;  in sdma_rb_evict()
470 	struct sdma_mmu_node *node =  in sdma_rb_remove()  local
473 	free_system_node(node);  in sdma_rb_remove()