48 static int pt_blk_status(const struct pt_block_decoder *decoder, int flags)  in pt_blk_status()  argument
52 	if (!decoder)  in pt_blk_status()
55 	status = decoder->status;  in pt_blk_status()
62 	if (!decoder->enabled)  in pt_blk_status()
69 	if ((status & pts_eos) && !decoder->process_event)  in pt_blk_status()
75 static void pt_blk_reset(struct pt_block_decoder *decoder)  in pt_blk_reset()  argument
77 	if (!decoder)  in pt_blk_reset()
80 	decoder->mode = ptem_unknown;  in pt_blk_reset()
81 	decoder->ip = 0ull;  in pt_blk_reset()
82 	decoder->status = 0;  in pt_blk_reset()
83 	decoder->enabled = 0;  in pt_blk_reset()
84 	decoder->process_event = 0;  in pt_blk_reset()
85 	decoder->speculative = 0;  in pt_blk_reset()
86 	decoder->process_insn = 0;  in pt_blk_reset()
87 	decoder->bound_paging = 0;  in pt_blk_reset()
88 	decoder->bound_vmcs = 0;  in pt_blk_reset()
89 	decoder->bound_ptwrite = 0;  in pt_blk_reset()
91 	memset(&decoder->event, 0, sizeof(decoder->event));  in pt_blk_reset()
92 	pt_retstack_init(&decoder->retstack);  in pt_blk_reset()
93 	pt_asid_init(&decoder->asid);  in pt_blk_reset()
109 int pt_blk_decoder_init(struct pt_block_decoder *decoder,  in pt_blk_decoder_init()  argument
115 	if (!decoder)  in pt_blk_decoder_init()
123 	decoder->flags = config.flags;  in pt_blk_decoder_init()
126 	errcode = pt_blk_init_qry_flags(&config.flags, &decoder->flags);  in pt_blk_decoder_init()
130 	errcode = pt_qry_decoder_init(&decoder->query, &config);  in pt_blk_decoder_init()
134 	pt_image_init(&decoder->default_image, NULL);  in pt_blk_decoder_init()
135 	decoder->image = &decoder->default_image;  in pt_blk_decoder_init()
137 	errcode = pt_msec_cache_init(&decoder->scache);  in pt_blk_decoder_init()
141 	pt_blk_reset(decoder);  in pt_blk_decoder_init()
146 void pt_blk_decoder_fini(struct pt_block_decoder *decoder)  in pt_blk_decoder_fini()  argument
148 	if (!decoder)  in pt_blk_decoder_fini()
151 	pt_msec_cache_fini(&decoder->scache);  in pt_blk_decoder_fini()
152 	pt_image_fini(&decoder->default_image);  in pt_blk_decoder_fini()
153 	pt_qry_decoder_fini(&decoder->query);  in pt_blk_decoder_fini()
159 	struct pt_block_decoder *decoder;  in pt_blk_alloc_decoder()  local
162 	decoder = malloc(sizeof(*decoder));  in pt_blk_alloc_decoder()
163 	if (!decoder)  in pt_blk_alloc_decoder()
166 	errcode = pt_blk_decoder_init(decoder, config);  in pt_blk_alloc_decoder()
168 		free(decoder);  in pt_blk_alloc_decoder()
172 	return decoder;  in pt_blk_alloc_decoder()
175 void pt_blk_free_decoder(struct pt_block_decoder *decoder)  in pt_blk_free_decoder()  argument
177 	if (!decoder)  in pt_blk_free_decoder()
180 	pt_blk_decoder_fini(decoder);  in pt_blk_free_decoder()
181 	free(decoder);  in pt_blk_free_decoder()
193 static int pt_blk_tick(struct pt_block_decoder *decoder, uint64_t ip)  in pt_blk_tick()  argument
200 	if (!decoder)  in pt_blk_tick()
204 	if (!decoder->enabled)  in pt_blk_tick()
218 	if (decoder->process_event)  in pt_blk_tick()
221 	errcode = pt_qry_time(&decoder->query, &tsc, &lost_mtc, &lost_cyc);  in pt_blk_tick()
228 	ev = &decoder->event;  in pt_blk_tick()
247 	decoder->process_event = 1;  in pt_blk_tick()
256 static int pt_blk_indirect_branch(struct pt_block_decoder *decoder,  in pt_blk_indirect_branch()  argument
262 	if (!decoder)  in pt_blk_indirect_branch()
265 	evip = decoder->ip;  in pt_blk_indirect_branch()
267 	status = pt_qry_indirect_branch(&decoder->query, ip);  in pt_blk_indirect_branch()
271 	if (decoder->flags.variant.block.enable_tick_events) {  in pt_blk_indirect_branch()
272 		errcode = pt_blk_tick(decoder, evip);  in pt_blk_indirect_branch()
284 static int pt_blk_cond_branch(struct pt_block_decoder *decoder, int *taken)  in pt_blk_cond_branch()  argument
288 	if (!decoder)  in pt_blk_cond_branch()
291 	status = pt_qry_cond_branch(&decoder->query, taken);  in pt_blk_cond_branch()
295 	if (decoder->flags.variant.block.enable_tick_events) {  in pt_blk_cond_branch()
296 		errcode = pt_blk_tick(decoder, decoder->ip);  in pt_blk_cond_branch()
304 static int pt_blk_start(struct pt_block_decoder *decoder, int status)  in pt_blk_start()  argument
306 	if (!decoder)  in pt_blk_start()
312 	decoder->status = status;  in pt_blk_start()
314 		decoder->enabled = 1;  in pt_blk_start()
324 	return pt_blk_proceed_trailing_event(decoder, NULL);  in pt_blk_start()
327 static int pt_blk_sync_reset(struct pt_block_decoder *decoder)  in pt_blk_sync_reset()  argument
329 	if (!decoder)  in pt_blk_sync_reset()
332 	pt_blk_reset(decoder);  in pt_blk_sync_reset()
337 int pt_blk_sync_forward(struct pt_block_decoder *decoder)  in pt_blk_sync_forward()  argument
341 	if (!decoder)  in pt_blk_sync_forward()
344 	errcode = pt_blk_sync_reset(decoder);  in pt_blk_sync_forward()
348 	status = pt_qry_sync_forward(&decoder->query, &decoder->ip);  in pt_blk_sync_forward()
350 	return pt_blk_start(decoder, status);  in pt_blk_sync_forward()
353 int pt_blk_sync_backward(struct pt_block_decoder *decoder)  in pt_blk_sync_backward()  argument
357 	if (!decoder)  in pt_blk_sync_backward()
360 	errcode = pt_blk_sync_reset(decoder);  in pt_blk_sync_backward()
364 	status = pt_qry_sync_backward(&decoder->query, &decoder->ip);  in pt_blk_sync_backward()
366 	return pt_blk_start(decoder, status);  in pt_blk_sync_backward()
369 int pt_blk_sync_set(struct pt_block_decoder *decoder, uint64_t offset)  in pt_blk_sync_set()  argument
373 	if (!decoder)  in pt_blk_sync_set()
376 	errcode = pt_blk_sync_reset(decoder);  in pt_blk_sync_set()
380 	status = pt_qry_sync_set(&decoder->query, &decoder->ip, offset);  in pt_blk_sync_set()
382 	return pt_blk_start(decoder, status);  in pt_blk_sync_set()
385 int pt_blk_get_offset(const struct pt_block_decoder *decoder, uint64_t *offset)  in pt_blk_get_offset()  argument
387 	if (!decoder)  in pt_blk_get_offset()
390 	return pt_qry_get_offset(&decoder->query, offset);  in pt_blk_get_offset()
393 int pt_blk_get_sync_offset(const struct pt_block_decoder *decoder,  in pt_blk_get_sync_offset()  argument
396 	if (!decoder)  in pt_blk_get_sync_offset()
399 	return pt_qry_get_sync_offset(&decoder->query, offset);  in pt_blk_get_sync_offset()
402 struct pt_image *pt_blk_get_image(struct pt_block_decoder *decoder)  in pt_blk_get_image()  argument
404 	if (!decoder)  in pt_blk_get_image()
407 	return decoder->image;  in pt_blk_get_image()
410 int pt_blk_set_image(struct pt_block_decoder *decoder, struct pt_image *image)  in pt_blk_set_image()  argument
412 	if (!decoder)  in pt_blk_set_image()
416 		image = &decoder->default_image;  in pt_blk_set_image()
418 	decoder->image = image;  in pt_blk_set_image()
423 pt_blk_get_config(const struct pt_block_decoder *decoder)  in pt_blk_get_config()  argument
425 	if (!decoder)  in pt_blk_get_config()
428 	return pt_qry_get_config(&decoder->query);  in pt_blk_get_config()
431 int pt_blk_time(struct pt_block_decoder *decoder, uint64_t *time,  in pt_blk_time()  argument
434 	if (!decoder || !time)  in pt_blk_time()
437 	return pt_qry_time(&decoder->query, time, lost_mtc, lost_cyc);  in pt_blk_time()
440 int pt_blk_core_bus_ratio(struct pt_block_decoder *decoder, uint32_t *cbr)  in pt_blk_core_bus_ratio()  argument
442 	if (!decoder || !cbr)  in pt_blk_core_bus_ratio()
445 	return pt_qry_core_bus_ratio(&decoder->query, cbr);  in pt_blk_core_bus_ratio()
448 int pt_blk_asid(const struct pt_block_decoder *decoder, struct pt_asid *asid,  in pt_blk_asid()  argument
451 	if (!decoder || !asid)  in pt_blk_asid()
454 	return pt_asid_to_user(asid, &decoder->asid, size);  in pt_blk_asid()
466 static inline int pt_blk_fetch_event(struct pt_block_decoder *decoder)  in pt_blk_fetch_event()  argument
470 	if (!decoder)  in pt_blk_fetch_event()
473 	if (decoder->process_event)  in pt_blk_fetch_event()
476 	if (!(decoder->status & pts_event_pending))  in pt_blk_fetch_event()
479 	status = pt_qry_event(&decoder->query, &decoder->event,  in pt_blk_fetch_event()
480 			      sizeof(decoder->event));  in pt_blk_fetch_event()
484 	decoder->process_event = 1;  in pt_blk_fetch_event()
485 	decoder->status = status;  in pt_blk_fetch_event()
546 static int pt_blk_next_ip(uint64_t *pip, struct pt_block_decoder *decoder,  in pt_blk_next_ip()  argument
552 	if (!pip || !decoder || !insn || !iext)  in pt_blk_next_ip()
565 		status = pt_blk_cond_branch(decoder, &taken);  in pt_blk_next_ip()
581 		status = pt_blk_cond_branch(decoder, &taken);  in pt_blk_next_ip()
595 		errcode = pt_retstack_pop(&decoder->retstack, pip);  in pt_blk_next_ip()
628 	return pt_blk_indirect_branch(decoder, pip);  in pt_blk_next_ip()
638 static int pt_blk_proceed_with_trace(struct pt_block_decoder *decoder,  in pt_blk_proceed_with_trace()  argument
644 	if (!decoder)  in pt_blk_proceed_with_trace()
647 	status = pt_blk_next_ip(&decoder->ip, decoder, insn, iext);  in pt_blk_proceed_with_trace()
654 	decoder->status = status;  in pt_blk_proceed_with_trace()
699 static inline int pt_blk_log_call(struct pt_block_decoder *decoder,  in pt_blk_log_call()  argument
703 	if (!decoder || !insn || !iext)  in pt_blk_log_call()
716 	return pt_retstack_push(&decoder->retstack, insn->ip + insn->size);  in pt_blk_log_call()
735 static int pt_blk_proceed_one_insn(struct pt_block_decoder *decoder,  in pt_blk_proceed_one_insn()  argument
745 	if (!decoder || !block || !pinsn || !piext)  in pt_blk_proceed_one_insn()
760 	insn.mode = decoder->mode;  in pt_blk_proceed_one_insn()
761 	insn.ip = decoder->ip;  in pt_blk_proceed_one_insn()
763 	status = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);  in pt_blk_proceed_one_insn()
785 	status = pt_blk_log_call(decoder, &insn, &iext);  in pt_blk_proceed_one_insn()
821 static int pt_blk_proceed_to_insn(struct pt_block_decoder *decoder,  in pt_blk_proceed_to_insn()  argument
830 	if (!decoder || !insn || !predicate)  in pt_blk_proceed_to_insn()
834 		status = pt_blk_proceed_one_insn(decoder, block, insn, iext);  in pt_blk_proceed_to_insn()
846 		status = pt_insn_next_ip(&decoder->ip, insn, iext);  in pt_blk_proceed_to_insn()
856 		if ((decoder->flags.variant.block.end_on_call &&  in pt_blk_proceed_to_insn()
858 		    (decoder->flags.variant.block.end_on_jump &&  in pt_blk_proceed_to_insn()
883 static int pt_blk_proceed_to_ip(struct pt_block_decoder *decoder,  in pt_blk_proceed_to_ip()  argument
889 	if (!decoder || !insn)  in pt_blk_proceed_to_ip()
896 		if (decoder->ip == ip)  in pt_blk_proceed_to_ip()
899 		status = pt_blk_proceed_one_insn(decoder, block, insn, iext);  in pt_blk_proceed_to_ip()
904 		status = pt_insn_next_ip(&decoder->ip, insn, iext);  in pt_blk_proceed_to_ip()
917 		if ((decoder->flags.variant.block.end_on_call &&  in pt_blk_proceed_to_ip()
919 		    (decoder->flags.variant.block.end_on_jump &&  in pt_blk_proceed_to_ip()
921 			return (decoder->ip == ip ? 1 : 0);  in pt_blk_proceed_to_ip()
942 static int pt_blk_proceed_to_ip_with_trace(struct pt_block_decoder *decoder,  in pt_blk_proceed_to_ip_with_trace()  argument
955 	status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext, ip);  in pt_blk_proceed_to_ip_with_trace()
962 	return pt_blk_proceed_with_trace(decoder, &insn, &iext);  in pt_blk_proceed_to_ip_with_trace()
1001 static int pt_blk_proceed_skl014(struct pt_block_decoder *decoder,  in pt_blk_proceed_skl014()  argument
1008 	if (!decoder || !block || !insn || !iext)  in pt_blk_proceed_skl014()
1011 	addr_filter = &decoder->query.config.addr_filter;  in pt_blk_proceed_skl014()
1015 		status = pt_blk_proceed_to_insn(decoder, block, insn, iext,  in pt_blk_proceed_skl014()
1047 		decoder->ip = ip;  in pt_blk_proceed_skl014()
1055 		if ((decoder->flags.variant.block.end_on_call &&  in pt_blk_proceed_skl014()
1057 		    (decoder->flags.variant.block.end_on_jump &&  in pt_blk_proceed_skl014()
1076 static int pt_blk_proceed_to_disabled(struct pt_block_decoder *decoder,  in pt_blk_proceed_to_disabled()  argument
1082 	if (!decoder || !block || !ev)  in pt_blk_proceed_to_disabled()
1094 		if (decoder->query.config.addr_filter.config.addr_cfg &&  in pt_blk_proceed_to_disabled()
1095 		    decoder->query.config.errata.skl014)  in pt_blk_proceed_to_disabled()
1096 			return pt_blk_proceed_skl014(decoder, block, insn,  in pt_blk_proceed_to_disabled()
1105 		return pt_blk_proceed_to_insn(decoder, block, insn, iext,  in pt_blk_proceed_to_disabled()
1108 		return pt_blk_proceed_to_ip(decoder, block, insn, iext,  in pt_blk_proceed_to_disabled()
1122 static int pt_blk_set_disable_resume_ip(struct pt_block_decoder *decoder,  in pt_blk_set_disable_resume_ip()  argument
1125 	if (!decoder || !insn)  in pt_blk_set_disable_resume_ip()
1131 		decoder->ip = insn->ip + insn->size;  in pt_blk_set_disable_resume_ip()
1135 		decoder->ip = 0ull;  in pt_blk_set_disable_resume_ip()
1152 static int pt_blk_proceed_to_async_paging(struct pt_block_decoder *decoder,  in pt_blk_proceed_to_async_paging()  argument
1158 	if (!decoder || !ev)  in pt_blk_proceed_to_async_paging()
1165 	status = pt_blk_proceed_to_ip_with_trace(decoder, block,  in pt_blk_proceed_to_async_paging()
1171 	return (decoder->ip == ev->variant.async_paging.ip ? 1 : 0);  in pt_blk_proceed_to_async_paging()
1184 static int pt_blk_proceed_to_async_vmcs(struct pt_block_decoder *decoder,  in pt_blk_proceed_to_async_vmcs()  argument
1190 	if (!decoder || !ev)  in pt_blk_proceed_to_async_vmcs()
1197 	status = pt_blk_proceed_to_ip_with_trace(decoder, block,  in pt_blk_proceed_to_async_vmcs()
1203 	return (decoder->ip == ev->variant.async_vmcs.ip ? 1 : 0);  in pt_blk_proceed_to_async_vmcs()
1216 static int pt_blk_proceed_to_exec_mode(struct pt_block_decoder *decoder,  in pt_blk_proceed_to_exec_mode()  argument
1222 	if (!decoder || !ev)  in pt_blk_proceed_to_exec_mode()
1229 	status = pt_blk_proceed_to_ip_with_trace(decoder, block,  in pt_blk_proceed_to_exec_mode()
1235 	return (decoder->ip == ev->variant.exec_mode.ip ? 1 : 0);  in pt_blk_proceed_to_exec_mode()
1253 static int pt_blk_proceed_to_ptwrite(struct pt_block_decoder *decoder,  in pt_blk_proceed_to_ptwrite()  argument
1278 		status = pt_blk_proceed_to_insn(decoder, block, insn, iext,  in pt_blk_proceed_to_ptwrite()
1290 		status = pt_blk_proceed_to_ip(decoder, block, insn, iext,  in pt_blk_proceed_to_ptwrite()
1302 		status = pt_blk_proceed_one_insn(decoder, block, insn, iext);  in pt_blk_proceed_to_ptwrite()
1319 static int pt_blk_handle_erratum_skd022(struct pt_block_decoder *decoder,  in pt_blk_handle_erratum_skd022()  argument
1326 	if (!decoder || !ev)  in pt_blk_handle_erratum_skd022()
1329 	insn.mode = decoder->mode;  in pt_blk_handle_erratum_skd022()
1332 	errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);  in pt_blk_handle_erratum_skd022()
1364 static int pt_blk_postpone_insn(struct pt_block_decoder *decoder,  in pt_blk_postpone_insn()  argument
1368 	if (!decoder || !insn || !iext)  in pt_blk_postpone_insn()
1372 	if (decoder->process_insn)  in pt_blk_postpone_insn()
1375 	decoder->process_insn = 1;  in pt_blk_postpone_insn()
1376 	decoder->insn = *insn;  in pt_blk_postpone_insn()
1377 	decoder->iext = *iext;  in pt_blk_postpone_insn()
1379 	return pt_blk_status(decoder, pts_event_pending);  in pt_blk_postpone_insn()
1386 static int pt_blk_clear_postponed_insn(struct pt_block_decoder *decoder)  in pt_blk_clear_postponed_insn()  argument
1388 	if (!decoder)  in pt_blk_clear_postponed_insn()
1391 	decoder->process_insn = 0;  in pt_blk_clear_postponed_insn()
1392 	decoder->bound_paging = 0;  in pt_blk_clear_postponed_insn()
1393 	decoder->bound_vmcs = 0;  in pt_blk_clear_postponed_insn()
1394 	decoder->bound_ptwrite = 0;  in pt_blk_clear_postponed_insn()
1405 static int pt_blk_proceed_postponed_insn(struct pt_block_decoder *decoder)  in pt_blk_proceed_postponed_insn()  argument
1409 	if (!decoder)  in pt_blk_proceed_postponed_insn()
1413 	if (!decoder->process_insn)  in pt_blk_proceed_postponed_insn()
1417 	if (!decoder->enabled)  in pt_blk_proceed_postponed_insn()
1418 		return pt_blk_clear_postponed_insn(decoder);  in pt_blk_proceed_postponed_insn()
1420 	status = pt_insn_next_ip(&decoder->ip, &decoder->insn, &decoder->iext);  in pt_blk_proceed_postponed_insn()
1425 		status = pt_blk_proceed_with_trace(decoder, &decoder->insn,  in pt_blk_proceed_postponed_insn()
1426 						   &decoder->iext);  in pt_blk_proceed_postponed_insn()
1431 	return pt_blk_clear_postponed_insn(decoder);  in pt_blk_proceed_postponed_insn()
1452 static int pt_blk_proceed_event(struct pt_block_decoder *decoder,  in pt_blk_proceed_event()  argument
1460 	if (!decoder || !decoder->process_event || !block)  in pt_blk_proceed_event()
1463 	ev = &decoder->event;  in pt_blk_proceed_event()
1469 		status = pt_blk_proceed_to_disabled(decoder, block, &insn,  in pt_blk_proceed_event()
1479 			status = pt_blk_set_disable_resume_ip(decoder, &insn);  in pt_blk_proceed_event()
1487 		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,  in pt_blk_proceed_event()
1492 		if (decoder->query.config.errata.skd022) {  in pt_blk_proceed_event()
1493 			status = pt_blk_handle_erratum_skd022(decoder, ev);  in pt_blk_proceed_event()
1501 				return pt_blk_proceed_event(decoder, block);  in pt_blk_proceed_event()
1508 		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,  in pt_blk_proceed_event()
1516 		if (!decoder->enabled)  in pt_blk_proceed_event()
1519 		status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,  in pt_blk_proceed_event()
1527 		decoder->bound_paging = 1;  in pt_blk_proceed_event()
1529 		return pt_blk_postpone_insn(decoder, &insn, &iext);  in pt_blk_proceed_event()
1532 		status = pt_blk_proceed_to_async_paging(decoder, block, ev);  in pt_blk_proceed_event()
1539 		if (!decoder->enabled)  in pt_blk_proceed_event()
1542 		status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,  in pt_blk_proceed_event()
1550 		decoder->bound_vmcs = 1;  in pt_blk_proceed_event()
1552 		return pt_blk_postpone_insn(decoder, &insn, &iext);  in pt_blk_proceed_event()
1555 		status = pt_blk_proceed_to_async_vmcs(decoder, block, ev);  in pt_blk_proceed_event()
1565 		status = pt_blk_proceed_to_exec_mode(decoder, block, ev);  in pt_blk_proceed_event()
1575 		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,  in pt_blk_proceed_event()
1586 		if (!decoder->enabled || ev->ip_suppressed)  in pt_blk_proceed_event()
1589 		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,  in pt_blk_proceed_event()
1597 		if (!decoder->enabled || ev->ip_suppressed)  in pt_blk_proceed_event()
1600 		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,  in pt_blk_proceed_event()
1612 		if (!decoder->enabled)  in pt_blk_proceed_event()
1615 		status = pt_blk_proceed_to_ptwrite(decoder, block, &insn,  in pt_blk_proceed_event()
1623 		decoder->bound_ptwrite = 1;  in pt_blk_proceed_event()
1625 		return pt_blk_postpone_insn(decoder, &insn, &iext);  in pt_blk_proceed_event()
1633 	return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_event()
1649 static int pt_blk_proceed_no_event_uncached(struct pt_block_decoder *decoder,  in pt_blk_proceed_no_event_uncached()  argument
1656 	if (!decoder || !block)  in pt_blk_proceed_no_event_uncached()
1666 	status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,  in pt_blk_proceed_no_event_uncached()
1672 		return pt_blk_proceed_with_trace(decoder, &insn, &iext);  in pt_blk_proceed_no_event_uncached()
1772 pt_blk_proceed_no_event_fill_cache(struct pt_block_decoder *decoder,  in pt_blk_proceed_no_event_fill_cache()  argument
1785 	if (!decoder || !steps)  in pt_blk_proceed_no_event_fill_cache()
1793 	status = pt_blk_proceed_one_insn(decoder, block, &insn, &iext);  in pt_blk_proceed_no_event_fill_cache()
1803 	status = pt_insn_next_ip(&decoder->ip, &insn, &iext);  in pt_blk_proceed_no_event_fill_cache()
1868 		return pt_blk_proceed_with_trace(decoder, &insn, &iext);  in pt_blk_proceed_no_event_fill_cache()
1872 	nip = decoder->ip;  in pt_blk_proceed_no_event_fill_cache()
1942 		    decoder->flags.variant.block.end_on_jump)  in pt_blk_proceed_no_event_fill_cache()
1978 		status = pt_blk_proceed_no_event_fill_cache(decoder, block,  in pt_blk_proceed_no_event_fill_cache()
2058 static int pt_blk_proceed_truncated(struct pt_block_decoder *decoder,  in pt_blk_proceed_truncated()  argument
2065 	if (!decoder || !block)  in pt_blk_proceed_truncated()
2071 	insn.mode = decoder->mode;  in pt_blk_proceed_truncated()
2072 	insn.ip = decoder->ip;  in pt_blk_proceed_truncated()
2074 	errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);  in pt_blk_proceed_truncated()
2089 	errcode = pt_blk_log_call(decoder, &insn, &iext);  in pt_blk_proceed_truncated()
2098 	errcode = pt_insn_next_ip(&decoder->ip, &insn, &iext);  in pt_blk_proceed_truncated()
2103 		return pt_blk_proceed_with_trace(decoder, &insn, &iext);  in pt_blk_proceed_truncated()
2125 static int pt_blk_proceed_no_event_cached(struct pt_block_decoder *decoder,  in pt_blk_proceed_no_event_cached()  argument
2135 	if (!decoder || !block)  in pt_blk_proceed_no_event_cached()
2138 	offset = pt_msec_unmap(msec, decoder->ip);  in pt_blk_proceed_no_event_cached()
2145 		return pt_blk_proceed_no_event_fill_cache(decoder, block,  in pt_blk_proceed_no_event_cached()
2169 	nip = decoder->ip + bce.displacement;  in pt_blk_proceed_no_event_cached()
2171 		return pt_blk_proceed_no_event_uncached(decoder, block);  in pt_blk_proceed_no_event_cached()
2192 	decoder->ip = nip;  in pt_blk_proceed_no_event_cached()
2215 		return pt_blk_proceed_no_event_cached(decoder, block, bcache,  in pt_blk_proceed_no_event_cached()
2240 			status = pt_blk_cond_branch(decoder, &taken);  in pt_blk_proceed_no_event_cached()
2247 			decoder->status = status;  in pt_blk_proceed_no_event_cached()
2249 			ip = decoder->ip;  in pt_blk_proceed_no_event_cached()
2268 			decoder->ip = ip + bce.isize;  in pt_blk_proceed_no_event_cached()
2288 		insn.ip = decoder->ip;  in pt_blk_proceed_no_event_cached()
2295 			return pt_blk_proceed_truncated(decoder, block);  in pt_blk_proceed_no_event_cached()
2302 		status = pt_blk_log_call(decoder, &insn, &iext);  in pt_blk_proceed_no_event_cached()
2311 		status = pt_insn_next_ip(&decoder->ip, &insn, &iext);  in pt_blk_proceed_no_event_cached()
2319 			return pt_blk_proceed_with_trace(decoder, &insn, &iext);  in pt_blk_proceed_no_event_cached()
2328 		if ((decoder->flags.variant.block.end_on_call &&  in pt_blk_proceed_no_event_cached()
2330 		    (decoder->flags.variant.block.end_on_jump &&  in pt_blk_proceed_no_event_cached()
2339 		if (!pt_blk_is_in_section(msec, decoder->ip))  in pt_blk_proceed_no_event_cached()
2342 		return pt_blk_proceed_no_event_cached(decoder, block, bcache,  in pt_blk_proceed_no_event_cached()
2355 		ip = decoder->ip;  in pt_blk_proceed_no_event_cached()
2379 		status = pt_retstack_push(&decoder->retstack, ip);  in pt_blk_proceed_no_event_cached()
2383 		status = pt_blk_indirect_branch(decoder, &decoder->ip);  in pt_blk_proceed_no_event_cached()
2390 		decoder->status = status;  in pt_blk_proceed_no_event_cached()
2401 		status = pt_blk_cond_branch(decoder, &taken);  in pt_blk_proceed_no_event_cached()
2409 			status = pt_blk_indirect_branch(decoder, &decoder->ip);  in pt_blk_proceed_no_event_cached()
2416 			decoder->status = status;  in pt_blk_proceed_no_event_cached()
2423 		decoder->status = status;  in pt_blk_proceed_no_event_cached()
2431 		return pt_retstack_pop(&decoder->retstack, &decoder->ip);  in pt_blk_proceed_no_event_cached()
2450 		status = pt_blk_indirect_branch(decoder, &decoder->ip);  in pt_blk_proceed_no_event_cached()
2457 		decoder->status = status;  in pt_blk_proceed_no_event_cached()
2464 static int pt_blk_msec_fill(struct pt_block_decoder *decoder,  in pt_blk_msec_fill()  argument
2471 	if (!decoder || !pmsec)  in pt_blk_msec_fill()
2474 	isid = pt_msec_cache_fill(&decoder->scache, &msec,  decoder->image,  in pt_blk_msec_fill()
2475 				  &decoder->asid, decoder->ip);  in pt_blk_msec_fill()
2492 static inline int pt_blk_msec_lookup(struct pt_block_decoder *decoder,  in pt_blk_msec_lookup()  argument
2497 	if (!decoder)  in pt_blk_msec_lookup()
2500 	isid = pt_msec_cache_read(&decoder->scache, pmsec, decoder->image,  in pt_blk_msec_lookup()
2501 				  decoder->ip);  in pt_blk_msec_lookup()
2506 		return pt_blk_msec_fill(decoder, pmsec);  in pt_blk_msec_lookup()
2525 static int pt_blk_proceed_no_event(struct pt_block_decoder *decoder,  in pt_blk_proceed_no_event()  argument
2533 	if (!decoder || !block)  in pt_blk_proceed_no_event()
2536 	isid = pt_blk_msec_lookup(decoder, &msec);  in pt_blk_proceed_no_event()
2544 		return pt_blk_proceed_no_event_uncached(decoder, block);  in pt_blk_proceed_no_event()
2561 		return pt_blk_proceed_no_event_uncached(decoder, block);  in pt_blk_proceed_no_event()
2563 	return pt_blk_proceed_no_event_cached(decoder, block, bcache, msec);  in pt_blk_proceed_no_event()
2571 static int pt_blk_proceed(struct pt_block_decoder *decoder,  in pt_blk_proceed()  argument
2576 	status = pt_blk_fetch_event(decoder);  in pt_blk_proceed()
2581 		return pt_blk_proceed_event(decoder, block);  in pt_blk_proceed()
2587 	if (!decoder->enabled) {  in pt_blk_proceed()
2588 		if (decoder->status & pts_eos)  in pt_blk_proceed()
2594 	status = pt_blk_proceed_no_event(decoder, block);  in pt_blk_proceed()
2598 	return pt_blk_proceed_trailing_event(decoder, block);  in pt_blk_proceed()
2617 static int pt_blk_handle_erratum_bdm64(struct pt_block_decoder *decoder,  in pt_blk_handle_erratum_bdm64()  argument
2625 	if (!decoder || !block || !ev)  in pt_blk_handle_erratum_bdm64()
2643 	status = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);  in pt_blk_handle_erratum_bdm64()
2655 	status = pt_insn_range_is_contiguous(decoder->ip, ev->variant.tsx.ip,  in pt_blk_handle_erratum_bdm64()
2656 					     decoder->mode, decoder->image,  in pt_blk_handle_erratum_bdm64()
2657 					     &decoder->asid, bdm64_max_steps);  in pt_blk_handle_erratum_bdm64()
2667 	decoder->ip = ev->variant.tsx.ip;  in pt_blk_handle_erratum_bdm64()
2680 static inline int pt_blk_postpone_trailing_tsx(struct pt_block_decoder *decoder,  in pt_blk_postpone_trailing_tsx()  argument
2686 	if (!decoder || !ev)  in pt_blk_postpone_trailing_tsx()
2692 	if (block && decoder->query.config.errata.bdm64) {  in pt_blk_postpone_trailing_tsx()
2693 		status = pt_blk_handle_erratum_bdm64(decoder, block, ev);  in pt_blk_postpone_trailing_tsx()
2698 	if (decoder->ip != ev->variant.tsx.ip)  in pt_blk_postpone_trailing_tsx()
2718 static int pt_blk_proceed_trailing_event(struct pt_block_decoder *decoder,  in pt_blk_proceed_trailing_event()  argument
2724 	if (!decoder)  in pt_blk_proceed_trailing_event()
2727 	status = pt_blk_fetch_event(decoder);  in pt_blk_proceed_trailing_event()
2732 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2736 		return pt_blk_status(decoder, 0);  in pt_blk_proceed_trailing_event()
2739 	ev = &decoder->event;  in pt_blk_proceed_trailing_event()
2745 		if (!decoder->process_insn)  in pt_blk_proceed_trailing_event()
2750 		    pt_insn_changes_cr3(&decoder->insn, &decoder->iext))  in pt_blk_proceed_trailing_event()
2751 			return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2759 		status = pt_insn_next_ip(&decoder->ip, &decoder->insn,  in pt_blk_proceed_trailing_event()
2760 					 &decoder->iext);  in pt_blk_proceed_trailing_event()
2765 			status = pt_blk_set_disable_resume_ip(decoder,  in pt_blk_proceed_trailing_event()
2766 							      &decoder->insn);  in pt_blk_proceed_trailing_event()
2770 			return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2774 		status = pt_blk_clear_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2780 		    decoder->ip == ev->variant.disabled.ip)  in pt_blk_proceed_trailing_event()
2781 			return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2787 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2791 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2795 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2799 		if (decoder->ip != ev->variant.async_disabled.at)  in pt_blk_proceed_trailing_event()
2802 		if (decoder->query.config.errata.skd022) {  in pt_blk_proceed_trailing_event()
2803 			status = pt_blk_handle_erratum_skd022(decoder, ev);  in pt_blk_proceed_trailing_event()
2817 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2821 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2825 		if (decoder->ip != ev->variant.async_branch.from)  in pt_blk_proceed_trailing_event()
2828 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2832 		if (!decoder->enabled)  in pt_blk_proceed_trailing_event()
2833 			return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2841 		if (!decoder->process_insn || decoder->bound_paging)  in pt_blk_proceed_trailing_event()
2848 		if (!pt_insn_binds_to_pip(&decoder->insn, &decoder->iext))  in pt_blk_proceed_trailing_event()
2854 		decoder->bound_paging = 1;  in pt_blk_proceed_trailing_event()
2856 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2860 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2865 		    decoder->ip != ev->variant.async_paging.ip)  in pt_blk_proceed_trailing_event()
2868 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2872 		if (!decoder->enabled)  in pt_blk_proceed_trailing_event()
2873 			return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2881 		if (!decoder->process_insn || decoder->bound_vmcs)  in pt_blk_proceed_trailing_event()
2888 		if (!pt_insn_binds_to_vmcs(&decoder->insn, &decoder->iext))  in pt_blk_proceed_trailing_event()
2894 		decoder->bound_vmcs = 1;  in pt_blk_proceed_trailing_event()
2896 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2900 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2905 		    decoder->ip != ev->variant.async_vmcs.ip)  in pt_blk_proceed_trailing_event()
2908 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2912 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2916 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2920 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2925 		    decoder->ip != ev->variant.exec_mode.ip)  in pt_blk_proceed_trailing_event()
2928 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2932 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2936 		status = pt_blk_postpone_trailing_tsx(decoder, block, ev);  in pt_blk_proceed_trailing_event()
2944 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2948 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2952 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2956 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2960 		if (!ev->ip_suppressed && decoder->enabled &&  in pt_blk_proceed_trailing_event()
2961 		    decoder->ip != ev->variant.exstop.ip)  in pt_blk_proceed_trailing_event()
2964 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2968 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2972 		if (!ev->ip_suppressed && decoder->enabled &&  in pt_blk_proceed_trailing_event()
2973 		    decoder->ip != ev->variant.mwait.ip)  in pt_blk_proceed_trailing_event()
2976 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2981 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
2985 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2989 		if (!decoder->enabled)  in pt_blk_proceed_trailing_event()
2990 			return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
2997 		if (!decoder->process_insn || decoder->bound_ptwrite)  in pt_blk_proceed_trailing_event()
3005 		    !pt_insn_is_ptwrite(&decoder->insn, &decoder->iext))  in pt_blk_proceed_trailing_event()
3011 		decoder->bound_ptwrite = 1;  in pt_blk_proceed_trailing_event()
3013 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
3019 		status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
3023 		return pt_blk_status(decoder, pts_event_pending);  in pt_blk_proceed_trailing_event()
3027 	status = pt_blk_proceed_postponed_insn(decoder);  in pt_blk_proceed_trailing_event()
3031 	return pt_blk_status(decoder, 0);  in pt_blk_proceed_trailing_event()
3034 int pt_blk_next(struct pt_block_decoder *decoder, struct pt_block *ublock,  in pt_blk_next()  argument
3040 	if (!decoder || !ublock)  in pt_blk_next()
3056 	pblock->ip = decoder->ip;  in pt_blk_next()
3057 	pblock->mode = decoder->mode;  in pt_blk_next()
3058 	if (decoder->speculative)  in pt_blk_next()
3062 	status = pt_blk_proceed(decoder, pblock);  in pt_blk_next()
3075 static int pt_blk_process_enabled(struct pt_block_decoder *decoder,  in pt_blk_process_enabled()  argument
3078 	if (!decoder || !ev)  in pt_blk_process_enabled()
3090 	if (decoder->enabled)  in pt_blk_process_enabled()
3093 	decoder->ip = ev->variant.enabled.ip;  in pt_blk_process_enabled()
3094 	decoder->enabled = 1;  in pt_blk_process_enabled()
3095 	decoder->process_event = 0;  in pt_blk_process_enabled()
3104 static int pt_blk_process_disabled(struct pt_block_decoder *decoder,  in pt_blk_process_disabled()  argument
3107 	if (!decoder || !ev)  in pt_blk_process_disabled()
3115 	if (!decoder->enabled)  in pt_blk_process_disabled()
3122 	decoder->enabled = 0;  in pt_blk_process_disabled()
3123 	decoder->process_event = 0;  in pt_blk_process_disabled()
3132 static int pt_blk_process_async_branch(struct pt_block_decoder *decoder,  in pt_blk_process_async_branch()  argument
3135 	if (!decoder || !ev)  in pt_blk_process_async_branch()
3143 	if (!decoder->enabled)  in pt_blk_process_async_branch()
3149 	decoder->ip = ev->variant.async_branch.to;  in pt_blk_process_async_branch()
3150 	decoder->process_event = 0;  in pt_blk_process_async_branch()
3159 static int pt_blk_process_paging(struct pt_block_decoder *decoder,  in pt_blk_process_paging()  argument
3165 	if (!decoder || !ev)  in pt_blk_process_paging()
3169 	if (decoder->asid.cr3 != cr3) {  in pt_blk_process_paging()
3170 		errcode = pt_msec_cache_invalidate(&decoder->scache);  in pt_blk_process_paging()
3174 		decoder->asid.cr3 = cr3;  in pt_blk_process_paging()
3177 	decoder->process_event = 0;  in pt_blk_process_paging()
3186 static int pt_blk_process_vmcs(struct pt_block_decoder *decoder,  in pt_blk_process_vmcs()  argument
3192 	if (!decoder || !ev)  in pt_blk_process_vmcs()
3196 	if (decoder->asid.vmcs != vmcs) {  in pt_blk_process_vmcs()
3197 		errcode = pt_msec_cache_invalidate(&decoder->scache);  in pt_blk_process_vmcs()
3201 		decoder->asid.vmcs = vmcs;  in pt_blk_process_vmcs()
3204 	decoder->process_event = 0;  in pt_blk_process_vmcs()
3213 static int pt_blk_process_overflow(struct pt_block_decoder *decoder,  in pt_blk_process_overflow()  argument
3216 	if (!decoder || !ev)  in pt_blk_process_overflow()
3232 		decoder->enabled = 0;  in pt_blk_process_overflow()
3233 		decoder->ip = 0ull;  in pt_blk_process_overflow()
3238 		decoder->enabled = 1;  in pt_blk_process_overflow()
3239 		decoder->ip = ev->variant.overflow.ip;  in pt_blk_process_overflow()
3248 	decoder->speculative = 0;  in pt_blk_process_overflow()
3249 	decoder->process_event = 0;  in pt_blk_process_overflow()
3258 static int pt_blk_process_exec_mode(struct pt_block_decoder *decoder,  in pt_blk_process_exec_mode()  argument
3263 	if (!decoder || !ev)  in pt_blk_process_exec_mode()
3268 	if (ev->status_update && decoder->enabled &&  in pt_blk_process_exec_mode()
3269 	    decoder->mode != ptem_unknown && decoder->mode != mode)  in pt_blk_process_exec_mode()
3272 	decoder->mode = mode;  in pt_blk_process_exec_mode()
3273 	decoder->process_event = 0;  in pt_blk_process_exec_mode()
3282 static int pt_blk_process_tsx(struct pt_block_decoder *decoder,  in pt_blk_process_tsx()  argument
3285 	if (!decoder || !ev)  in pt_blk_process_tsx()
3288 	decoder->speculative = ev->variant.tsx.speculative;  in pt_blk_process_tsx()
3289 	decoder->process_event = 0;  in pt_blk_process_tsx()
3298 static int pt_blk_process_stop(struct pt_block_decoder *decoder,  in pt_blk_process_stop()  argument
3301 	if (!decoder || !ev)  in pt_blk_process_stop()
3309 	if (decoder->enabled)  in pt_blk_process_stop()
3312 	decoder->process_event = 0;  in pt_blk_process_stop()
3317 int pt_blk_event(struct pt_block_decoder *decoder, struct pt_event *uevent,  in pt_blk_event()  argument
3323 	if (!decoder || !uevent)  in pt_blk_event()
3327 	if (!decoder->process_event)  in pt_blk_event()
3330 	ev = &decoder->event;  in pt_blk_event()
3337 		if (ev->variant.enabled.ip == decoder->ip)  in pt_blk_event()
3340 		status = pt_blk_process_enabled(decoder, ev);  in pt_blk_event()
3347 		if (decoder->ip != ev->variant.async_disabled.at)  in pt_blk_event()
3353 		status = pt_blk_process_disabled(decoder, ev);  in pt_blk_event()
3360 		if (decoder->ip != ev->variant.async_branch.from)  in pt_blk_event()
3363 		status = pt_blk_process_async_branch(decoder, ev);  in pt_blk_event()
3371 		    decoder->ip != ev->variant.async_paging.ip)  in pt_blk_event()
3376 		status = pt_blk_process_paging(decoder, ev);  in pt_blk_event()
3384 		    decoder->ip != ev->variant.async_vmcs.ip)  in pt_blk_event()
3389 		status = pt_blk_process_vmcs(decoder, ev);  in pt_blk_event()
3396 		status = pt_blk_process_overflow(decoder, ev);  in pt_blk_event()
3404 		    decoder->ip != ev->variant.exec_mode.ip)  in pt_blk_event()
3407 		status = pt_blk_process_exec_mode(decoder, ev);  in pt_blk_event()
3414 		if (!ev->ip_suppressed && decoder->ip != ev->variant.tsx.ip)  in pt_blk_event()
3417 		status = pt_blk_process_tsx(decoder, ev);  in pt_blk_event()
3424 		status = pt_blk_process_stop(decoder, ev);  in pt_blk_event()
3431 		if (!ev->ip_suppressed && decoder->enabled &&  in pt_blk_event()
3432 		    decoder->ip != ev->variant.exstop.ip)  in pt_blk_event()
3435 		decoder->process_event = 0;  in pt_blk_event()
3439 		if (!ev->ip_suppressed && decoder->enabled &&  in pt_blk_event()
3440 		    decoder->ip != ev->variant.mwait.ip)  in pt_blk_event()
3443 		decoder->process_event = 0;  in pt_blk_event()
3452 		decoder->process_event = 0;  in pt_blk_event()
3468 	return pt_blk_proceed_trailing_event(decoder, NULL);  in pt_blk_event()