68 	struct ioat_test	*test;  member
87 	struct ioat_test *test;  in ioat_test_transaction_destroy()  local
90 	test = tx->test;  in ioat_test_transaction_destroy()
94 			if (test->testkind == IOAT_TEST_DMA_8K)  in ioat_test_transaction_destroy()
106 test_transaction *ioat_test_transaction_create(struct ioat_test *test,  in ioat_test_transaction_create()  argument
116 	tx->length = test->buffer_size;  in ioat_test_transaction_create()
119 		if (test->testkind == IOAT_TEST_DMA_8K)  in ioat_test_transaction_create()
120 			tx->buf[i] = malloc(test->buffer_size, M_IOAT_TEST,  in ioat_test_transaction_create()
123 			tx->buf[i] = contigmalloc(test->buffer_size,  in ioat_test_transaction_create()
150 	struct ioat_test *test;  in ioat_compare_ok()  local
154 	test = tx->test;  in ioat_compare_ok()
160 		if (test->testkind == IOAT_TEST_FILL) {  in ioat_compare_ok()
167 		} else if (test->testkind == IOAT_TEST_DMA) {  in ioat_compare_ok()
170 		} else if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_compare_ok()
171 			if (test->raw_write)  in ioat_compare_ok()
172 				dst = test->raw_vtarget;  in ioat_compare_ok()
183 	struct ioat_test *test;  in ioat_dma_test_callback()  local
189 	test = tx->test;  in ioat_dma_test_callback()
191 	if (test->verify && !ioat_compare_ok(tx)) {  in ioat_dma_test_callback()
193 		atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth);  in ioat_dma_test_callback()
194 	} else if (!test->too_late)  in ioat_dma_test_callback()
195 		atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth);  in ioat_dma_test_callback()
198 	TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_dma_test_callback()
199 	TAILQ_INSERT_TAIL(&test->free_q, tx, entry);  in ioat_dma_test_callback()
200 	wakeup(&test->free_q);  in ioat_dma_test_callback()
205 ioat_test_prealloc_memory(struct ioat_test *test, int index)  in ioat_test_prealloc_memory()  argument
210 	for (i = 0; i < test->transactions; i++) {  in ioat_test_prealloc_memory()
211 		tx = ioat_test_transaction_create(test, test->chain_depth * 2);  in ioat_test_prealloc_memory()
214 			test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_test_prealloc_memory()
218 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_prealloc_memory()
220 		tx->test = test;  in ioat_test_prealloc_memory()
221 		tx->depth = test->chain_depth;  in ioat_test_prealloc_memory()
227 			for (k = 0; k < test->chain_depth; k++) {  in ioat_test_prealloc_memory()
237 ioat_test_release_memory(struct ioat_test *test)  in ioat_test_release_memory()  argument
241 	TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s)  in ioat_test_release_memory()
243 	TAILQ_INIT(&test->free_q);  in ioat_test_release_memory()
245 	TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s)  in ioat_test_release_memory()
247 	TAILQ_INIT(&test->pend_q);  in ioat_test_release_memory()
251 ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma)  in ioat_test_submit_1_tx()  argument
263 	while (TAILQ_EMPTY(&test->free_q))  in ioat_test_submit_1_tx()
264 		msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0);  in ioat_test_submit_1_tx()
266 	tx = TAILQ_FIRST(&test->free_q);  in ioat_test_submit_1_tx()
267 	TAILQ_REMOVE(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()
268 	TAILQ_INSERT_HEAD(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
271 	if (test->testkind != IOAT_TEST_MEMCPY)  in ioat_test_submit_1_tx()
274 		if (test->testkind == IOAT_TEST_MEMCPY) {  in ioat_test_submit_1_tx()
284 		if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_test_submit_1_tx()
285 			if (test->raw_write)  in ioat_test_submit_1_tx()
286 				dest = test->raw_target;  in ioat_test_submit_1_tx()
288 				src = test->raw_target;  in ioat_test_submit_1_tx()
299 		if (test->testkind == IOAT_TEST_DMA ||  in ioat_test_submit_1_tx()
300 		    test->testkind == IOAT_TEST_RAW_DMA)  in ioat_test_submit_1_tx()
303 		else if (test->testkind == IOAT_TEST_FILL) {  in ioat_test_submit_1_tx()
307 		} else if (test->testkind == IOAT_TEST_DMA_8K) {  in ioat_test_submit_1_tx()
319 	if (test->testkind == IOAT_TEST_MEMCPY)  in ioat_test_submit_1_tx()
329 		atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth);  in ioat_test_submit_1_tx()
331 		TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
332 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()
341 	struct ioat_test *test;  in ioat_dma_test()  local
346 	test = arg;  in ioat_dma_test()
347 	memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status));  in ioat_dma_test()
349 	if (test->testkind == IOAT_TEST_DMA_8K &&  in ioat_dma_test()
350 	    test->buffer_size != 2 * PAGE_SIZE) {  in ioat_dma_test()
352 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
356 	if (test->buffer_size > 1024 * 1024) {  in ioat_dma_test()
358 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
362 	if (test->chain_depth * 2 > IOAT_MAX_BUFS) {  in ioat_dma_test()
365 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
369 	if (btoc((uint64_t)test->buffer_size * test->chain_depth *  in ioat_dma_test()
370 	    test->transactions) > (physmem / 4)) {  in ioat_dma_test()
373 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
377 	if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) {  in ioat_dma_test()
380 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
384 	if (test->testkind >= IOAT_NUM_TESTKINDS) {  in ioat_dma_test()
386 		    (unsigned)test->testkind);  in ioat_dma_test()
387 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
391 	dmaengine = ioat_get_dmaengine(test->channel_index, M_NOWAIT);  in ioat_dma_test()
394 		test->status[IOAT_TEST_NO_DMA_ENGINE]++;  in ioat_dma_test()
399 	if (test->testkind == IOAT_TEST_FILL &&  in ioat_dma_test()
404 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
408 	if (test->coalesce_period > ioat->intrdelay_max) {  in ioat_dma_test()
411 		    (unsigned)test->coalesce_period);  in ioat_dma_test()
412 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
415 	error = ioat_set_interrupt_coalesce(dmaengine, test->coalesce_period);  in ioat_dma_test()
416 	if (error == ENODEV && test->coalesce_period == 0)  in ioat_dma_test()
420 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
424 	if (test->zero_stats)  in ioat_dma_test()
427 	if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_dma_test()
428 		if (test->raw_is_virtual) {  in ioat_dma_test()
429 			test->raw_vtarget = (void *)test->raw_target;  in ioat_dma_test()
430 			test->raw_target = vtophys(test->raw_vtarget);  in ioat_dma_test()
432 			test->raw_vtarget = pmap_mapdev(test->raw_target,  in ioat_dma_test()
433 			    test->buffer_size);  in ioat_dma_test()
438 	TAILQ_INIT(&test->free_q);  in ioat_dma_test()
439 	TAILQ_INIT(&test->pend_q);  in ioat_dma_test()
441 	if (test->duration == 0)  in ioat_dma_test()
443 		    index, test->transactions);  in ioat_dma_test()
447 	rc = ioat_test_prealloc_memory(test, index);  in ioat_dma_test()
454 	test->too_late = false;  in ioat_dma_test()
456 	end = start + (((sbintime_t)test->duration * hz) / 1000);  in ioat_dma_test()
459 		if (test->duration == 0 && loops >= test->transactions)  in ioat_dma_test()
461 		else if (test->duration != 0 && time_after(ticks, end)) {  in ioat_dma_test()
462 			test->too_late = true;  in ioat_dma_test()
466 		ioat_test_submit_1_tx(test, dmaengine);  in ioat_dma_test()
473 	while (!TAILQ_EMPTY(&test->pend_q))  in ioat_dma_test()
474 		msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz);  in ioat_dma_test()
480 	ioat_test_release_memory(test);  in ioat_dma_test()
482 	if (test->testkind == IOAT_TEST_RAW_DMA && !test->raw_is_virtual)  in ioat_dma_test()
483 		pmap_unmapdev((vm_offset_t)test->raw_vtarget,  in ioat_dma_test()
484 		    test->buffer_size);  in ioat_dma_test()