85 ioat_test_transaction_destroy(struct test_transaction *tx)  in ioat_test_transaction_destroy()  argument
90 	test = tx->test;  in ioat_test_transaction_destroy()
93 		if (tx->buf[i] != NULL) {  in ioat_test_transaction_destroy()
95 				free(tx->buf[i], M_IOAT_TEST);  in ioat_test_transaction_destroy()
97 				contigfree(tx->buf[i], tx->length, M_IOAT_TEST);  in ioat_test_transaction_destroy()
98 			tx->buf[i] = NULL;  in ioat_test_transaction_destroy()
102 	free(tx, M_IOAT_TEST);  in ioat_test_transaction_destroy()
109 	struct test_transaction *tx;  in ioat_test_transaction_create()  local
112 	tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO);  in ioat_test_transaction_create()
113 	if (tx == NULL)  in ioat_test_transaction_create()
116 	tx->length = test->buffer_size;  in ioat_test_transaction_create()
120 			tx->buf[i] = malloc(test->buffer_size, M_IOAT_TEST,  in ioat_test_transaction_create()
123 			tx->buf[i] = contigmalloc(test->buffer_size,  in ioat_test_transaction_create()
127 		if (tx->buf[i] == NULL) {  in ioat_test_transaction_create()
128 			ioat_test_transaction_destroy(tx);  in ioat_test_transaction_create()
132 	return (tx);  in ioat_test_transaction_create()
148 ioat_compare_ok(struct test_transaction *tx)  in ioat_compare_ok()  argument
154 	test = tx->test;  in ioat_compare_ok()
156 	for (i = 0; i < tx->depth; i++) {  in ioat_compare_ok()
157 		dst = tx->buf[2 * i + 1];  in ioat_compare_ok()
158 		src = tx->buf[2 * i];  in ioat_compare_ok()
161 			for (j = 0; j < tx->length; j += sizeof(uint64_t)) {  in ioat_compare_ok()
163 					MIN(sizeof(uint64_t), tx->length - j))  in ioat_compare_ok()
168 			if (memcmp(src, dst, tx->length) != 0)  in ioat_compare_ok()
173 			dump_hex(dst, tx->length / 32);  in ioat_compare_ok()
182 	struct test_transaction *tx;  in ioat_dma_test_callback()  local
188 	tx = arg;  in ioat_dma_test_callback()
189 	test = tx->test;  in ioat_dma_test_callback()
191 	if (test->verify && !ioat_compare_ok(tx)) {  in ioat_dma_test_callback()
193 		atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth);  in ioat_dma_test_callback()
195 		atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth);  in ioat_dma_test_callback()
198 	TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_dma_test_callback()
199 	TAILQ_INSERT_TAIL(&test->free_q, tx, entry);  in ioat_dma_test_callback()
208 	struct test_transaction *tx;  in ioat_test_prealloc_memory()  local
211 		tx = ioat_test_transaction_create(test, test->chain_depth * 2);  in ioat_test_prealloc_memory()
212 		if (tx == NULL) {  in ioat_test_prealloc_memory()
218 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_prealloc_memory()
220 		tx->test = test;  in ioat_test_prealloc_memory()
221 		tx->depth = test->chain_depth;  in ioat_test_prealloc_memory()
224 		for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) {  in ioat_test_prealloc_memory()
228 				((uint32_t *)tx->buf[2*k])[j] = ~val;  in ioat_test_prealloc_memory()
229 				((uint32_t *)tx->buf[2*k+1])[j] = val;  in ioat_test_prealloc_memory()
239 	struct test_transaction *tx, *s;  in ioat_test_release_memory()  local
241 	TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s)  in ioat_test_release_memory()
242 		ioat_test_transaction_destroy(tx);  in ioat_test_release_memory()
245 	TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s)  in ioat_test_release_memory()
246 		ioat_test_transaction_destroy(tx);  in ioat_test_release_memory()
253 	struct test_transaction *tx;  in ioat_test_submit_1_tx()  local
266 	tx = TAILQ_FIRST(&test->free_q);  in ioat_test_submit_1_tx()
267 	TAILQ_REMOVE(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()
268 	TAILQ_INSERT_HEAD(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
273 	for (i = 0; i < tx->depth; i++) {  in ioat_test_submit_1_tx()
275 			memcpy(tx->buf[2 * i + 1], tx->buf[2 * i], tx->length);  in ioat_test_submit_1_tx()
276 			if (i == tx->depth - 1)  in ioat_test_submit_1_tx()
277 				ioat_dma_test_callback(tx, 0);  in ioat_test_submit_1_tx()
281 		src = vtophys((vm_offset_t)tx->buf[2*i]);  in ioat_test_submit_1_tx()
282 		dest = vtophys((vm_offset_t)tx->buf[2*i+1]);  in ioat_test_submit_1_tx()
291 		if (i == tx->depth - 1) {  in ioat_test_submit_1_tx()
301 			desc = ioat_copy(dma, dest, src, tx->length, cb, tx,  in ioat_test_submit_1_tx()
304 			fillpattern = *(uint64_t *)tx->buf[2*i];  in ioat_test_submit_1_tx()
306 			    tx->length, cb, tx, flags);  in ioat_test_submit_1_tx()
310 			src2 = vtophys((vm_offset_t)tx->buf[2*i] + PAGE_SIZE);  in ioat_test_submit_1_tx()
311 			dst2 = vtophys((vm_offset_t)tx->buf[2*i+1] + PAGE_SIZE);  in ioat_test_submit_1_tx()
314 			    cb, tx, flags);  in ioat_test_submit_1_tx()
328 	if (desc == NULL && tx->depth > 0) {  in ioat_test_submit_1_tx()
329 		atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth);  in ioat_test_submit_1_tx()
331 		TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
332 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()