185 #define ikm_require(kmsg) \  argument
186 	zone_id_require(ZONE_ID_IPC_KMSG, sizeof(struct ipc_kmsg), kmsg)
187 #define ikm_require_aligned(kmsg) \  argument
188 	zone_id_require_aligned(ZONE_ID_IPC_KMSG, kmsg)
200 ikm_is_linear(ipc_kmsg_t kmsg)  in ikm_is_linear()  argument
202 	return kmsg->ikm_type == IKM_TYPE_ALL_INLINED ||  in ikm_is_linear()
203 	       kmsg->ikm_type == IKM_TYPE_KDATA_OOL;  in ikm_is_linear()
246 	ipc_kmsg_t      kmsg,  in ikm_udata()  argument
250 	if (ikm_is_linear(kmsg)) {  in ikm_udata()
251 		mach_msg_header_t *hdr = ikm_header(kmsg);  in ikm_udata()
255 	return kmsg->ikm_udata;  in ikm_udata()
268 ikm_udata_from_header(ipc_kmsg_t kmsg)  in ikm_udata_from_header()  argument
270 	if (ikm_is_linear(kmsg)) {  in ikm_udata_from_header()
271 		mach_msg_header_t *hdr = ikm_header(kmsg);  in ikm_udata_from_header()
275 	return kmsg->ikm_udata;  in ikm_udata_from_header()
281 ikm_kdata_end(ipc_kmsg_t kmsg)  in ikm_kdata_end()  argument
283 	switch (kmsg->ikm_type) {  in ikm_kdata_end()
285 		return (vm_offset_t)kmsg->ikm_big_data + IKM_BIG_MSG_SIZE;  in ikm_kdata_end()
287 		return (vm_offset_t)kmsg->ikm_small_data + IKM_SMALL_MSG_SIZE;  in ikm_kdata_end()
289 		return (vm_offset_t)kmsg->ikm_kdata + kmsg->ikm_kdata_size;  in ikm_kdata_end()
299 	ipc_kmsg_t         kmsg)  in ikm_header()  argument
301 	switch (kmsg->ikm_type) {  in ikm_header()
303 		return (mach_msg_header_t *)kmsg->ikm_big_data;  in ikm_header()
305 		return (mach_msg_header_t *)kmsg->ikm_small_data;  in ikm_header()
307 		return (mach_msg_header_t *)kmsg->ikm_kdata;  in ikm_header()
313 	ipc_kmsg_t         kmsg)  in ikm_aux_header()  argument
315 	if (!kmsg->ikm_aux_size) {  in ikm_aux_header()
319 	assert(kmsg->ikm_aux_size >= sizeof(mach_msg_aux_header_t));  in ikm_aux_header()
321 	if (kmsg->ikm_type == IKM_TYPE_ALL_INLINED) {  in ikm_aux_header()
322 		return (mach_msg_aux_header_t *)((vm_offset_t)(kmsg + 1) -  in ikm_aux_header()
323 		       kmsg->ikm_aux_size);  in ikm_aux_header()
325 		assert(kmsg->ikm_type != IKM_TYPE_KDATA_OOL);  in ikm_aux_header()
326 		return (mach_msg_aux_header_t *)((vm_offset_t)kmsg->ikm_udata +  in ikm_aux_header()
327 		       kmsg->ikm_udata_size - kmsg->ikm_aux_size);  in ikm_aux_header()
394 	ipc_kmsg_t              kmsg)  in ipc_kmsg_get_trailer()  argument
396 	mach_msg_header_t *hdr = ikm_header(kmsg);  in ipc_kmsg_get_trailer()
400 	if (ikm_is_linear(kmsg)) {  in ipc_kmsg_get_trailer()
403 		base = (vm_offset_t)kmsg->ikm_udata;  in ipc_kmsg_get_trailer()
412 	ipc_kmsg_t           kmsg,  in ipc_kmsg_set_voucher_port()  argument
419 	kmsg->ikm_voucher_port = voucher_port;  in ipc_kmsg_set_voucher_port()
420 	kmsg->ikm_voucher_type = type;  in ipc_kmsg_set_voucher_port()
424 ipc_kmsg_get_voucher_port(ipc_kmsg_t kmsg)  in ipc_kmsg_get_voucher_port()  argument
426 	return kmsg->ikm_voucher_port;  in ipc_kmsg_get_voucher_port()
430 ipc_kmsg_clear_voucher_port(ipc_kmsg_t kmsg)  in ipc_kmsg_clear_voucher_port()  argument
432 	kmsg->ikm_voucher_port = IP_NULL;  in ipc_kmsg_clear_voucher_port()
433 	kmsg->ikm_voucher_type = MACH_MSGH_BITS_ZERO;  in ipc_kmsg_clear_voucher_port()
446 ipc_kmsg_get_thread_group(ipc_kmsg_t kmsg)  in ipc_kmsg_get_thread_group()  argument
451 	ipc_voucher_t voucher = convert_port_to_voucher(ipc_kmsg_get_voucher_port(kmsg));  in ipc_kmsg_get_thread_group()
463 __ikm_signature_check_panic(ipc_kmsg_t kmsg, uint32_t sig)  in __ikm_signature_check_panic()  argument
465 	mach_msg_header_t *hdr = ikm_header(kmsg);  in __ikm_signature_check_panic()
469 	    kmsg, hdr, hdr->msgh_id, sig, kmsg->ikm_signature);  in __ikm_signature_check_panic()
474 	ipc_kmsg_t              kmsg,  in __ipc_kmsg_sign()  argument
479 	mach_msg_header_t *hdr  = ikm_header(kmsg);  in __ipc_kmsg_sign()
503 		uintptr_t data = (uintptr_t)kmsg;  in __ipc_kmsg_sign()
513 	(void)kmsg;  in __ipc_kmsg_sign()
524 ipc_kmsg_sign(ipc_kmsg_t kmsg, mach_msg_max_trailer_t *trailer)  in ipc_kmsg_sign()  argument
526 	kmsg->ikm_signature = __ipc_kmsg_sign(kmsg, trailer, NULL);  in ipc_kmsg_sign()
537 	ipc_kmsg_t          kmsg,  in ipc_kmsg_init_trailer_and_sign()  argument
547 	mach_msg_max_trailer_t *trailer = ipc_kmsg_get_trailer(kmsg);  in ipc_kmsg_init_trailer_and_sign()
559 	ipc_kmsg_sign(kmsg, trailer);  in ipc_kmsg_init_trailer_and_sign()
568 	ipc_kmsg_t kmsg)  in ipc_kmsg_validate_signature()  argument
573 	ikm_require_aligned(kmsg);  in ipc_kmsg_validate_signature()
574 	sig = __ipc_kmsg_sign(kmsg, ipc_kmsg_get_trailer(kmsg), &dsc_count);  in ipc_kmsg_validate_signature()
575 	if (sig != kmsg->ikm_signature) {  in ipc_kmsg_validate_signature()
576 		__ikm_signature_check_panic(kmsg, sig);  in ipc_kmsg_validate_signature()
740 	ipc_kmsg_t kmsg;  in ipc_kmsg_alloc()  local
870 	kmsg = zalloc_id(ZONE_ID_IPC_KMSG, Z_WAITOK | Z_ZERO | Z_NOFAIL);  in ipc_kmsg_alloc()
871 	kmsg->ikm_type = kmsg_type;  in ipc_kmsg_alloc()
872 	kmsg->ikm_aux_size = (uint16_t)aux_size;  in ipc_kmsg_alloc()
876 		kmsg->ikm_keep_alive = IKM_KEEP_ALIVE_OWNED;  in ipc_kmsg_alloc()
884 			kmsg->ikm_kdata = kmsg->ikm_small_data;  in ipc_kmsg_alloc()
886 			kmsg->ikm_kdata = msg_kdata;  in ipc_kmsg_alloc()
888 		kmsg->ikm_udata = msg_udata;  in ipc_kmsg_alloc()
889 		kmsg->ikm_kdata_size = max_kdata_size;  in ipc_kmsg_alloc()
890 		kmsg->ikm_udata_size = max_udata_size;  in ipc_kmsg_alloc()
893 	return kmsg;  in ipc_kmsg_alloc()
914 ipc_kmsg_keep_alive_try_reusing(ipc_kmsg_t kmsg)  in ipc_kmsg_keep_alive_try_reusing()  argument
918 	v = os_atomic_or_orig(&kmsg->ikm_keep_alive,  in ipc_kmsg_keep_alive_try_reusing()
934 ipc_kmsg_keep_alive_done_using(ipc_kmsg_t kmsg)  in ipc_kmsg_keep_alive_done_using()  argument
936 	uintptr_t v = os_atomic_load(&kmsg->ikm_keep_alive, relaxed);  in ipc_kmsg_keep_alive_done_using()
943 	v = os_atomic_andnot_orig(&kmsg->ikm_keep_alive,  in ipc_kmsg_keep_alive_done_using()
964 	ipc_kmsg_t              kmsg)  in ipc_kmsg_keep_alive_abandon()  argument
968 	v = os_atomic_andnot_orig(&kmsg->ikm_keep_alive,  in ipc_kmsg_keep_alive_abandon()
976 		ipc_kmsg_free(kmsg);  in ipc_kmsg_keep_alive_abandon()
989 	ipc_kmsg_t              kmsg)  in ipc_kmsg_free_allocations()  argument
993 	switch (kmsg->ikm_type) {  in ipc_kmsg_free_allocations()
997 		kfree_data(kmsg->ikm_udata, kmsg->ikm_udata_size);  in ipc_kmsg_free_allocations()
1001 		ikm_free_kdata_ool(kmsg->ikm_kdata, kmsg->ikm_kdata_size);  in ipc_kmsg_free_allocations()
1005 		dsc_count = (kmsg->ikm_kdata_size - sizeof(mach_msg_base_t)) /  in ipc_kmsg_free_allocations()
1008 		    kmsg->ikm_kdata);  in ipc_kmsg_free_allocations()
1010 		kfree_data(kmsg->ikm_udata, kmsg->ikm_udata_size);  in ipc_kmsg_free_allocations()
1016 	kmsg->ikm_type = IKM_TYPE_ALL_INLINED;  in ipc_kmsg_free_allocations()
1019 	kmsg->ikm_udata = NULL;  in ipc_kmsg_free_allocations()
1020 	kmsg->ikm_kdata = NULL;  in ipc_kmsg_free_allocations()
1021 	kmsg->ikm_udata_size = 0;  in ipc_kmsg_free_allocations()
1022 	kmsg->ikm_kdata_size = 0;  in ipc_kmsg_free_allocations()
1023 	kmsg->ikm_aux_size = 0;  in ipc_kmsg_free_allocations()
1035 	ipc_kmsg_t      kmsg)  in ipc_kmsg_free()  argument
1037 	assert(!IP_VALID(ipc_kmsg_get_voucher_port(kmsg)));  in ipc_kmsg_free()
1040 	    VM_KERNEL_ADDRPERM((uintptr_t)kmsg),  in ipc_kmsg_free()
1047 	if (kmsg->ikm_type == IKM_TYPE_ALL_INLINED &&  in ipc_kmsg_free()
1048 	    !ipc_kmsg_keep_alive_done_using(kmsg)) {  in ipc_kmsg_free()
1052 	ipc_kmsg_free_allocations(kmsg);  in ipc_kmsg_free()
1053 	zfree_id(ZONE_ID_IPC_KMSG, kmsg);  in ipc_kmsg_free()
1066 	ipc_kmsg_t              kmsg)  in ipc_kmsg_clean_header()  argument
1069 	mach_msg_header_t *hdr = ikm_header(kmsg);  in ipc_kmsg_clean_header()
1073 	ipc_importance_clean(kmsg);  in ipc_kmsg_clean_header()
1085 	object = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));  in ipc_kmsg_clean_header()
1089 		ipc_kmsg_clear_voucher_port(kmsg);  in ipc_kmsg_clean_header()
1187 ipc_kmsg_clean(ipc_kmsg_t kmsg, mach_msg_size_t dsc_count)  in ipc_kmsg_clean()  argument
1189 	ipc_kmsg_clean_header(kmsg);  in ipc_kmsg_clean()
1192 		mach_msg_kbase_t *kbase = mach_msg_header_to_kbase(ikm_header(kmsg));  in ipc_kmsg_clean()
1221 	ipc_kmsg_t              kmsg)  in ipc_kmsg_enqueue_qos()  argument
1223 	mach_msg_qos_t qos_ovr = kmsg->ikm_qos_override;  in ipc_kmsg_enqueue_qos()
1226 	if (ipc_kmsg_enqueue(queue, kmsg)) {  in ipc_kmsg_enqueue_qos()
1231 	prev = ipc_kmsg_queue_element(kmsg->ikm_link.prev);  in ipc_kmsg_enqueue_qos()
1232 	while (prev != kmsg) {  in ipc_kmsg_enqueue_qos()
1259 	ipc_kmsg_t          kmsg,  in ipc_kmsg_override_qos()  argument
1263 	ipc_kmsg_t cur = kmsg;  in ipc_kmsg_override_qos()
1289 	ipc_kmsg_t                     kmsg,  in ipc_kmsg_destroy()  argument
1294 	mach_msg_header_t *hdr = ikm_header(kmsg);  in ipc_kmsg_destroy()
1309 		ipc_kmsg_sign(kmsg, ipc_kmsg_get_trailer(kmsg));  in ipc_kmsg_destroy()
1318 	if (ipc_kmsg_delayed_destroy(kmsg)) {  in ipc_kmsg_destroy()
1334 	ipc_kmsg_t kmsg)  in ipc_kmsg_delayed_destroy()  argument
1336 	return ipc_kmsg_enqueue(&current_thread()->ith_messages, kmsg);  in ipc_kmsg_delayed_destroy()
1368 	ipc_kmsg_t kmsg;  in ipc_kmsg_reap_delayed()  local
1374 	while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) {  in ipc_kmsg_reap_delayed()
1384 		ipc_kmsg_clean(kmsg, ipc_kmsg_validate_signature(kmsg));  in ipc_kmsg_reap_delayed()
1385 		ipc_kmsg_rmqueue(queue, kmsg);  in ipc_kmsg_reap_delayed()
1386 		ipc_kmsg_free(kmsg);  in ipc_kmsg_reap_delayed()
1407 	ipc_kmsg_t kmsg,  in ipc_kmsg_set_qos()  argument
1412 	mach_msg_header_t *hdr = ikm_header(kmsg);  in ipc_kmsg_set_qos()
1422 		kmsg->ikm_ppriority = _pthread_priority_make_from_thread_qos(qos, relpri, 0);  in ipc_kmsg_set_qos()
1423 		kmsg->ikm_qos_override = MAX(qos, ovr);  in ipc_kmsg_set_qos()
1426 		kr = ipc_get_pthpriority_from_kmsg_voucher(kmsg, &kmsg->ikm_ppriority);  in ipc_kmsg_set_qos()
1432 				kmsg->ikm_ppriority = ipc_get_current_thread_priority();  in ipc_kmsg_set_qos()
1434 				kmsg->ikm_ppriority = MACH_MSG_PRIORITY_UNSPECIFIED;  in ipc_kmsg_set_qos()
1439 			mach_msg_qos_t qos = _pthread_priority_thread_qos(kmsg->ikm_ppriority);  in ipc_kmsg_set_qos()
1441 			kmsg->ikm_qos_override = MAX(qos, ovr);  in ipc_kmsg_set_qos()
1443 			kmsg->ikm_qos_override = _pthread_priority_thread_qos(kmsg->ikm_ppriority);  in ipc_kmsg_set_qos()
1465 	ipc_kmsg_t kmsg)  in ipc_kmsg_set_qos_kernel()  argument
1467 	ipc_port_t dest_port = ikm_header(kmsg)->msgh_remote_port;  in ipc_kmsg_set_qos_kernel()
1468 	kmsg->ikm_qos_override = dest_port->ip_kernel_qos_override;  in ipc_kmsg_set_qos_kernel()
1469 	kmsg->ikm_ppriority = _pthread_priority_make_from_thread_qos(kmsg->ikm_qos_override, 0, 0);  in ipc_kmsg_set_qos_kernel()
1720 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyin_header()  argument
1725 	mach_msg_header_t *msg = ikm_header(kmsg);  in ipc_kmsg_copyin_header()
2068 		kmsg->ikm_flags |= IPC_OBJECT_COPYIN_FLAGS_ALLOW_IMMOVABLE_SEND;  in ipc_kmsg_copyin_header()
2146 		ipc_kmsg_set_voucher_port(kmsg, voucher_port, voucher_type);  in ipc_kmsg_copyin_header()
2158 	ipc_kmsg_set_qos(kmsg, options, priority);  in ipc_kmsg_copyin_header()
2161 	ipc_kmsg_init_trailer_and_sign(kmsg, current_task());  in ipc_kmsg_copyin_header()
2193 			ipc_kmsg_clean_header(kmsg);  in ipc_kmsg_copyin_header()
2287 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyin_port_descriptor()  argument
2299 		    0, NULL, kmsg->ikm_flags);  in ipc_kmsg_copyin_port_descriptor()
2310 			ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;  in ipc_kmsg_copyin_port_descriptor()
2495 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyin_ool_ports_descriptor()  argument
2535 		    0, NULL, kmsg->ikm_flags);  in ipc_kmsg_copyin_ool_ports_descriptor()
2555 			ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;  in ipc_kmsg_copyin_ool_ports_descriptor()
2622 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyin_guarded_port_descriptor()  argument
2635 		    dsc->u_context, &guard_flags, kmsg->ikm_flags);  in ipc_kmsg_copyin_guarded_port_descriptor()
2646 			ikm_header(kmsg)->msgh_bits |= MACH_MSGH_BITS_CIRCULAR;  in ipc_kmsg_copyin_guarded_port_descriptor()
2856 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyin_body()  argument
2865 	mach_msg_header_t      *hdr   = ikm_header(kmsg);  in ipc_kmsg_copyin_body()
2882 			ipc_kmsg_clean_header(kmsg);  in ipc_kmsg_copyin_body()
2894 	kmsg->ikm_flags |= IPC_OBJECT_COPYIN_FLAGS_ALLOW_CONN_IMMOVABLE_RECEIVE;  in ipc_kmsg_copyin_body()
2903 			    space, dest_port, kmsg, options);  in ipc_kmsg_copyin_body()
2912 			    map, space, dest_port, kmsg, options);  in ipc_kmsg_copyin_body()
2916 			    space, dest_port, kmsg, options);  in ipc_kmsg_copyin_body()
2924 			ipc_kmsg_clean_header(kmsg);  in ipc_kmsg_copyin_body()
2955 	ipc_kmsg_t              kmsg,  in ipc_kmsg_get_and_inflate_from_user()  argument
2995 	if (ikm_is_linear(kmsg)) {  in ipc_kmsg_get_and_inflate_from_user()
2999 		kbody = kmsg->ikm_udata;  in ipc_kmsg_get_and_inflate_from_user()
3019 		if (ikm_is_linear(kmsg)) {  in ipc_kmsg_get_and_inflate_from_user()
3024 		assert((vm_offset_t)kdesc + copyin_size <= ikm_kdata_end(kmsg));  in ipc_kmsg_get_and_inflate_from_user()
3073 		mach_msg_aux_header_t *aux_hdr  = ikm_aux_header(kmsg);  in ipc_kmsg_get_and_inflate_from_user()
3119 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyin_from_user()  argument
3127 	mach_msg_header_t  *hdr = ikm_header(kmsg);  in ipc_kmsg_copyin_from_user()
3136 	mr = ipc_kmsg_get_and_inflate_from_user(kmsg, send_uctx,  in ipc_kmsg_copyin_from_user()
3148 	mr = ipc_kmsg_copyin_header(kmsg, space, priority, option64p);  in ipc_kmsg_copyin_from_user()
3157 		ipc_kmsg_clean_header(kmsg);  in ipc_kmsg_copyin_from_user()
3162 	    VM_KERNEL_ADDRPERM((uintptr_t)kmsg),  in ipc_kmsg_copyin_from_user()
3165 	    VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(ipc_kmsg_get_voucher_port(kmsg))),  in ipc_kmsg_copyin_from_user()
3173 	    ipc_kmsg_get_voucher_port(kmsg),  in ipc_kmsg_copyin_from_user()
3177 		mr = ipc_kmsg_copyin_body(kmsg, send_uctx, space, map, options);  in ipc_kmsg_copyin_from_user()
3317 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyout_header()  argument
3332 	ipc_port_t voucher = ipc_kmsg_get_voucher_port(kmsg);  in ipc_kmsg_copyout_header()
3509 			ipc_kmsg_clear_voucher_port(kmsg);  in ipc_kmsg_copyout_header()
3671 		    VM_KERNEL_ADDRPERM((uintptr_t)kmsg),  in ipc_kmsg_copyout_header()
3677 		    VM_KERNEL_ADDRPERM((uintptr_t)kmsg),  in ipc_kmsg_copyout_header()
4289 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyout_size()  argument
4292 	mach_msg_header_t *hdr   = ikm_header(kmsg);  in ipc_kmsg_copyout_size()
4324 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyout()  argument
4329 	mach_msg_header_t *hdr = ikm_header(kmsg);  in ipc_kmsg_copyout()
4333 	dsc_count = ipc_kmsg_validate_signature(kmsg);  in ipc_kmsg_copyout()
4335 	mr = ipc_kmsg_copyout_header(kmsg, hdr, space, option);  in ipc_kmsg_copyout()
4372 	ipc_kmsg_t              kmsg,  in ipc_kmsg_copyout_pseudo()  argument
4376 	mach_msg_header_t *hdr = ikm_header(kmsg);  in ipc_kmsg_copyout_pseudo()
4380 	ipc_object_t voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));  in ipc_kmsg_copyout_pseudo()
4392 	dsc_count = ipc_kmsg_validate_signature(kmsg);  in ipc_kmsg_copyout_pseudo()
4402 	ipc_importance_clean(kmsg);  in ipc_kmsg_copyout_pseudo()
4405 	ipc_importance_assert_clean(kmsg);  in ipc_kmsg_copyout_pseudo()
4435 		if (kmsg->ikm_voucher_type == MACH_MSG_TYPE_MOVE_SEND) {  in ipc_kmsg_copyout_pseudo()
4439 			assert(kmsg->ikm_voucher_type == MACH_MSG_TYPE_COPY_SEND);  in ipc_kmsg_copyout_pseudo()
4444 		ipc_kmsg_clear_voucher_port(kmsg);  in ipc_kmsg_copyout_pseudo()
4472 	ipc_kmsg_t      kmsg,  in ipc_kmsg_copyout_dest_to_user()  argument
4488 	dsc_count = ipc_kmsg_validate_signature(kmsg);  in ipc_kmsg_copyout_dest_to_user()
4490 	hdr = ikm_header(kmsg);  in ipc_kmsg_copyout_dest_to_user()
4494 	voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));  in ipc_kmsg_copyout_dest_to_user()
4500 	aux_size = kmsg->ikm_aux_size;  in ipc_kmsg_copyout_dest_to_user()
4504 	ipc_importance_assert_clean(kmsg);  in ipc_kmsg_copyout_dest_to_user()
4527 		ipc_kmsg_clear_voucher_port(kmsg);  in ipc_kmsg_copyout_dest_to_user()
4537 	ipc_kmsg_free_allocations(kmsg);  in ipc_kmsg_copyout_dest_to_user()
4542 	*ikm_header(kmsg) = (mach_msg_header_t){  in ipc_kmsg_copyout_dest_to_user()
4550 	ipc_kmsg_init_trailer_and_sign(kmsg, TASK_NULL);  in ipc_kmsg_copyout_dest_to_user()
4554 		kmsg->ikm_aux_size = sizeof(mach_msg_aux_header_t);  in ipc_kmsg_copyout_dest_to_user()
4555 		*ikm_aux_header(kmsg) = (mach_msg_aux_header_t){  in ipc_kmsg_copyout_dest_to_user()
4576 	ipc_kmsg_t      kmsg,  in ipc_kmsg_copyout_dest_to_kernel()  argument
4586 	(void)ipc_kmsg_validate_signature(kmsg);  in ipc_kmsg_copyout_dest_to_kernel()
4588 	hdr = ikm_header(kmsg);  in ipc_kmsg_copyout_dest_to_kernel()
4617 		ipc_port_t voucher = ipc_kmsg_get_voucher_port(kmsg);  in ipc_kmsg_copyout_dest_to_kernel()
4629 		ipc_kmsg_clear_voucher_port(kmsg);  in ipc_kmsg_copyout_dest_to_kernel()
4725 	ipc_kmsg_t              kmsg,     /* scalar or vector */  in ipc_kmsg_deflate()  argument
4730 	mach_msg_header_t      *hdr  = ikm_header(kmsg);  in ipc_kmsg_deflate()
4735 		.dctx_aux_hdr    = ikm_aux_header(kmsg),  in ipc_kmsg_deflate()
4736 		.dctx_aux_size   = kmsg->ikm_aux_size,  in ipc_kmsg_deflate()
4746 		dctx.dctx_trailer      = ipc_kmsg_get_trailer(kmsg);  in ipc_kmsg_deflate()
4754 	if (!ikm_is_linear(kmsg)) {  in ipc_kmsg_deflate()
4759 			dctx.dctx_udata      = kmsg->ikm_udata;  in ipc_kmsg_deflate()
4821 	ipc_kmsg_t              kmsg,     /* scalar or vector */  in ipc_kmsg_put_to_user()  argument
4839 	dctx = ipc_kmsg_deflate(kmsg, msgr, options, map);  in ipc_kmsg_put_to_user()
4947 	    recv_bufs->recv_msg_addr, VM_KERNEL_ADDRPERM((uintptr_t)kmsg),  in ipc_kmsg_put_to_user()
4950 	ipc_kmsg_free(kmsg);  in ipc_kmsg_put_to_user()
4980 	ipc_kmsg_t         kmsg;  in ipc_kmsg_get_from_kernel()  local
5001 	kmsg = ipc_kmsg_alloc(size, 0, desc_count, IPC_KMSG_ALLOC_KERNEL);  in ipc_kmsg_get_from_kernel()
5004 	if (kmsg == IKM_NULL) {  in ipc_kmsg_get_from_kernel()
5008 	hdr = ikm_header(kmsg);  in ipc_kmsg_get_from_kernel()
5009 	if (ikm_is_linear(kmsg)) {  in ipc_kmsg_get_from_kernel()
5013 		memcpy(kmsg->ikm_udata, (char *)msg + kdata_sz, size - kdata_sz);  in ipc_kmsg_get_from_kernel()
5029 	*kmsgp = kmsg;  in ipc_kmsg_get_from_kernel()
5073 	ipc_kmsg_t      kmsg)  in ipc_kmsg_copyin_from_kernel()  argument
5075 	mach_msg_header_t   *hdr = ikm_header(kmsg);  in ipc_kmsg_copyin_from_kernel()
5082 	ipc_object_t         voucher = ip_to_object(ipc_kmsg_get_voucher_port(kmsg));  in ipc_kmsg_copyin_from_kernel()
5117 	ipc_kmsg_set_qos_kernel(kmsg);  in ipc_kmsg_copyin_from_kernel()
5120 	ipc_kmsg_init_trailer_and_sign(kmsg, TASK_NULL);  in ipc_kmsg_copyin_from_kernel()
5137 					ipc_kmsg_clean_header(kmsg);  in ipc_kmsg_copyin_from_kernel()
5226 	ipc_kmsg_t              kmsg,  in ipc_kmsg_send()  argument
5253 	hdr = ikm_header(kmsg);  in ipc_kmsg_send()
5256 		ipc_kmsg_destroy(kmsg, IPC_KMSG_DESTROY_ALL);  in ipc_kmsg_send()
5261 	ipc_voucher_send_preprocessing(kmsg);  in ipc_kmsg_send()
5295 		if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {  in ipc_kmsg_send()
5296 			flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);  in ipc_kmsg_send()
5308 			ipc_importance_clean(kmsg);  in ipc_kmsg_send()
5311 		ipc_kmsg_destroy(kmsg, IPC_KMSG_DESTROY_SKIP_REMOTE);  in ipc_kmsg_send()
5326 		kmsg = ipc_kobject_server(port, kmsg, options);  in ipc_kmsg_send()
5327 		if (kmsg == IKM_NULL) {  in ipc_kmsg_send()
5331 		hdr = ikm_header(kmsg);  in ipc_kmsg_send()
5333 		ipc_kmsg_init_trailer_and_sign(kmsg, TASK_NULL);  in ipc_kmsg_send()
5355 		if (ipc_importance_send(kmsg, options)) {  in ipc_kmsg_send()
5370 		error = ipc_mqueue_send_locked(&port->ip_messages, kmsg,  in ipc_kmsg_send()
5399 			ipc_importance_clean(kmsg);  in ipc_kmsg_send()
5419 		if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {  in ipc_kmsg_send()
5420 			flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);  in ipc_kmsg_send()
5424 		ipc_kmsg_destroy(kmsg, IPC_KMSG_DESTROY_SKIP_REMOTE);  in ipc_kmsg_send()
5436 		if (MACH_NODE_VALID(kmsg->ikm_node) && FPORT_VALID(port->ip_messages.imq_fport)) {  in ipc_kmsg_send()
5437 			flipc_msg_ack(kmsg->ikm_node, &port->ip_messages, FALSE);  in ipc_kmsg_send()
5441 		ipc_kmsg_destroy(kmsg, IPC_KMSG_DESTROY_SKIP_REMOTE);  in ipc_kmsg_send()
5461 	ipc_kmsg_t              kmsg,  in ipc_kmsg_put_to_kernel()  argument
5464 	mach_msg_header_t *hdr = ikm_header(kmsg);  in ipc_kmsg_put_to_kernel()
5468 	assert(kmsg->ikm_aux_size == 0);  in ipc_kmsg_put_to_kernel()
5480 	if (ikm_is_linear(kmsg)) {  in ipc_kmsg_put_to_kernel()
5485 		    kmsg->ikm_udata, rcv_size - kdata_sz);  in ipc_kmsg_put_to_kernel()
5500 	ipc_kmsg_free(kmsg);  in ipc_kmsg_put_to_kernel()
5543 ipc_kmsg_trace_send(ipc_kmsg_t kmsg, mach_msg_option64_t option)  in ipc_kmsg_trace_send()  argument
5570 	msg = ikm_header(kmsg);  in ipc_kmsg_trace_send()
5597 	if (unsafe_convert_port_to_voucher(ipc_kmsg_get_voucher_port(kmsg))) {  in ipc_kmsg_trace_send()
5766 	trailer = (mach_msg_trailer_t *)ipc_kmsg_get_trailer(kmsg);  in ipc_kmsg_trace_send()