1 /*
2 * Copyright (c) 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/cpu_data.h>
30 #include <kern/cpu_number.h>
31 #include <kern/host.h>
32
33 #include <mach/host_priv.h>
34 #include <mach/host_special_ports.h>
35 #include <mach/host_info.h>
36 #include <mach/iocompressionstats_notification_server.h>
37 #include <mach/mach_host.h>
38
39 #include <sys/mount_internal.h>
40 #include <sys/param.h>
41 #include <sys/sysctl.h>
42 #include <sys/vnode.h>
43 #include <sys/vnode_internal.h>
44
45 #include <vfs/vfs_io_compression_stats.h>
46
47 #include <vm/lz4.h>
48 #include <vm/vm_compressor_algorithms_xnu.h>
49 #include <vm/vm_protos.h>
50
51
52 int io_compression_stats_enable = 0;
53 int io_compression_stats_block_size = IO_COMPRESSION_STATS_DEFAULT_BLOCK_SIZE;
54
55 #define LZ4_SCRATCH_ALIGN (64)
56 typedef struct {
57 uint8_t lz4state[lz4_encode_scratch_size]__attribute((aligned(LZ4_SCRATCH_ALIGN)));
58 } lz4_encode_scratch_t;
59
60 static lz4_encode_scratch_t *PERCPU_DATA(per_cpu_scratch_buf);
61 static uint8_t *PERCPU_DATA(per_cpu_compression_buf);
62 static uint32_t per_cpu_buf_size;
63 static char *vnpath_scratch_buf;
64
65 LCK_GRP_DECLARE(io_compression_stats_lckgrp, "io_compression_stats");
66 LCK_RW_DECLARE(io_compression_stats_lock, &io_compression_stats_lckgrp);
67 LCK_MTX_DECLARE(iocs_store_buffer_lock, &io_compression_stats_lckgrp);
68
69 typedef enum io_compression_stats_allocate_type {
70 IO_COMPRESSION_STATS_NEW_ALLOC = 0,
71 IO_COMPRESSION_STATS_RESIZE = 1
72 } io_compression_stats_alloc_type_t;
73
74 static void io_compression_stats_deallocate_compression_buffers(void);
75
76 struct iocs_store_buffer iocs_store_buffer = {
77 .buffer = 0,
78 .current_position = 0,
79 .marked_point = 0
80 };
81
82 int iocs_sb_bytes_since_last_mark = 0;
83 int iocs_sb_bytes_since_last_notification = 0;
84
85 KALLOC_TYPE_DEFINE(io_compression_stats_zone, struct io_compression_stats, KT_DEFAULT);
86
87 static int
io_compression_stats_allocate_compression_buffers(uint32_t block_size)88 io_compression_stats_allocate_compression_buffers(uint32_t block_size)
89 {
90 int err = 0;
91 host_basic_info_data_t hinfo;
92 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
93 uint32_t old_block_size = per_cpu_buf_size;
94 #define BSD_HOST 1
95 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
96
97 per_cpu_buf_size = block_size;
98
99 if (old_block_size == 0) {
100 static_assert(sizeof(lz4_encode_scratch_t) <= KALLOC_SAFE_ALLOC_SIZE);
101 percpu_foreach(buf, per_cpu_scratch_buf) {
102 *buf = kalloc_type(lz4_encode_scratch_t, Z_WAITOK | Z_ZERO);
103 if (*buf == NULL) {
104 err = ENOMEM;
105 goto out;
106 }
107 }
108 } else {
109 percpu_foreach(buf, per_cpu_compression_buf) {
110 kfree_data(*buf, old_block_size);
111 *buf = NULL;
112 }
113 }
114
115 percpu_foreach(buf, per_cpu_compression_buf) {
116 *buf = kalloc_data(block_size, Z_WAITOK | Z_ZERO);
117 if (*buf == NULL) {
118 err = ENOMEM;
119 goto out;
120 }
121 }
122
123 bzero(&iocs_store_buffer, sizeof(struct iocs_store_buffer));
124 iocs_store_buffer.buffer = kalloc_data(IOCS_STORE_BUFFER_SIZE, Z_WAITOK | Z_ZERO);
125 if (iocs_store_buffer.buffer == NULL) {
126 err = ENOMEM;
127 goto out;
128 }
129 iocs_store_buffer.current_position = 0;
130 iocs_store_buffer.marked_point = 0;
131
132 assert(vnpath_scratch_buf == NULL);
133 vnpath_scratch_buf = kalloc_data(MAXPATHLEN, Z_ZERO);
134 if (vnpath_scratch_buf == NULL) {
135 err = ENOMEM;
136 goto out;
137 }
138
139 out:
140 if (err) {
141 /* In case of any error, irrespective of whether it is new alloc or resize,
142 * dellocate all buffers and fail */
143 io_compression_stats_deallocate_compression_buffers();
144 }
145 return err;
146 }
147
148 static void
io_compression_stats_deallocate_compression_buffers(void)149 io_compression_stats_deallocate_compression_buffers(void)
150 {
151 percpu_foreach(buf, per_cpu_scratch_buf) {
152 kfree_type(lz4_encode_scratch_t, *buf);
153 *buf = NULL;
154 }
155
156 percpu_foreach(buf, per_cpu_compression_buf) {
157 kfree_data(*buf, per_cpu_buf_size);
158 *buf = NULL;
159 }
160
161 if (iocs_store_buffer.buffer != NULL) {
162 kfree_data(iocs_store_buffer.buffer, IOCS_STORE_BUFFER_SIZE);
163 bzero(&iocs_store_buffer, sizeof(struct iocs_store_buffer));
164 }
165
166 iocs_sb_bytes_since_last_mark = 0;
167 iocs_sb_bytes_since_last_notification = 0;
168 per_cpu_buf_size = 0;
169
170 kfree_data(vnpath_scratch_buf, MAXPATHLEN);
171 }
172
173
174 static int
175 sysctl_io_compression_stats_enable SYSCTL_HANDLER_ARGS
176 {
177 #pragma unused (arg1, arg2, oidp)
178
179 int error = 0;
180 int enable = 0;
181
182 error = SYSCTL_OUT(req, &io_compression_stats_enable, sizeof(int));
183
184 if (error || !req->newptr) {
185 return error;
186 }
187
188 error = SYSCTL_IN(req, &enable, sizeof(int));
189 if (error) {
190 return error;
191 }
192
193 if (!((enable == 1) || (enable == 0))) {
194 return EINVAL;
195 }
196
197 lck_rw_lock_exclusive(&io_compression_stats_lock);
198 lck_mtx_lock(&iocs_store_buffer_lock);
199 if ((io_compression_stats_enable == 0) && (enable == 1)) {
200 /* Enabling collection of stats. Allocate appropriate buffers */
201 error = io_compression_stats_allocate_compression_buffers(io_compression_stats_block_size);
202 if (error == 0) {
203 io_compression_stats_enable = enable;
204 io_compression_stats_dbg("SUCCESS: setting io_compression_stats_enable to %d", io_compression_stats_enable);
205 } else {
206 io_compression_stats_dbg("FAILED: setting io_compression_stats_enable to %d", io_compression_stats_enable);
207 }
208 } else if ((io_compression_stats_enable == 1) && (enable == 0)) {
209 io_compression_stats_deallocate_compression_buffers();
210 io_compression_stats_enable = 0;
211 io_compression_stats_dbg("SUCCESS: setting io_compression_stats_enable to %d", io_compression_stats_enable);
212 }
213 lck_mtx_unlock(&iocs_store_buffer_lock);
214 lck_rw_unlock_exclusive(&io_compression_stats_lock);
215
216 return error;
217 }
218 SYSCTL_PROC(_vfs, OID_AUTO, io_compression_stats_enable, CTLTYPE_INT | CTLFLAG_RW, 0, 0, &sysctl_io_compression_stats_enable, "I", "");
219
220 static int
221 sysctl_io_compression_block_size SYSCTL_HANDLER_ARGS
222 {
223 #pragma unused (arg1, arg2, oidp)
224
225 int error = 0;
226 int block_size = io_compression_stats_block_size;
227
228 error = SYSCTL_OUT(req, &block_size, sizeof(int));
229
230 if (error || !req->newptr) {
231 return error;
232 }
233
234 error = SYSCTL_IN(req, &block_size, sizeof(int));
235 if (error) {
236 return error;
237 }
238
239 if (block_size < IO_COMPRESSION_STATS_MIN_BLOCK_SIZE || block_size > IO_COMPRESSION_STATS_MAX_BLOCK_SIZE) {
240 return EINVAL;
241 }
242
243 lck_rw_lock_exclusive(&io_compression_stats_lock);
244
245 if (io_compression_stats_block_size != block_size) {
246 if (io_compression_stats_enable == 1) {
247 /* IO compression stats is enabled, rellocate buffers. */
248 error = io_compression_stats_allocate_compression_buffers(block_size);
249 if (error == 0) {
250 io_compression_stats_block_size = block_size;
251 io_compression_stats_dbg("SUCCESS: setting io_compression_stats_block_size to %d", io_compression_stats_block_size);
252 } else {
253 /* Failed to allocate buffers, disable IO compression stats */
254 io_compression_stats_enable = 0;
255 io_compression_stats_dbg("FAILED: setting io_compression_stats_block_size to %d", io_compression_stats_block_size);
256 }
257 } else {
258 /* IO compression stats is disabled, only set the io_compression_stats_block_size */
259 io_compression_stats_block_size = block_size;
260 io_compression_stats_dbg("SUCCESS: setting io_compression_stats_block_size to %d", io_compression_stats_block_size);
261 }
262 }
263 lck_rw_unlock_exclusive(&io_compression_stats_lock);
264
265
266 return error;
267 }
268 SYSCTL_PROC(_vfs, OID_AUTO, io_compression_stats_block_size, CTLTYPE_INT | CTLFLAG_RW, 0, 0, &sysctl_io_compression_block_size, "I", "");
269
270
271 static int32_t
iocs_compress_block(uint8_t * block_ptr,uint32_t block_size)272 iocs_compress_block(uint8_t *block_ptr, uint32_t block_size)
273 {
274 disable_preemption();
275
276 lz4_encode_scratch_t *scratch_buf = *PERCPU_GET(per_cpu_scratch_buf);
277 uint8_t *dest_buf = *PERCPU_GET(per_cpu_compression_buf);
278
279 int compressed_block_size = (int) lz4raw_encode_buffer(dest_buf, block_size,
280 block_ptr, block_size, (lz4_hash_entry_t *) scratch_buf);
281
282 enable_preemption();
283
284 return compressed_block_size;
285 }
286 /*
287 * Compress buf in chunks of io_compression_stats_block_size
288 */
289 static uint32_t
iocs_compress_buffer(vnode_t vn,uint8_t * buf_ptr,uint32_t buf_size)290 iocs_compress_buffer(vnode_t vn, uint8_t *buf_ptr, uint32_t buf_size)
291 {
292 uint32_t offset;
293 uint32_t compressed_size = 0;
294 int block_size = io_compression_stats_block_size;
295 int block_stats_scaling_factor = block_size / IOCS_BLOCK_NUM_SIZE_BUCKETS;
296
297 for (offset = 0; offset < buf_size; offset += block_size) {
298 int current_block_size = min(block_size, buf_size - offset);
299 int current_compressed_block_size = iocs_compress_block(buf_ptr + offset, current_block_size);
300
301 if (current_compressed_block_size == 0) {
302 compressed_size += current_block_size;
303 vnode_updateiocompressionblockstats(vn, current_block_size / block_stats_scaling_factor);
304 } else if (current_compressed_block_size != -1) {
305 compressed_size += current_compressed_block_size;
306 vnode_updateiocompressionblockstats(vn, current_compressed_block_size / block_stats_scaling_factor);
307 }
308 }
309
310 return compressed_size;
311 }
312
313 static uint32_t
log2down(uint32_t x)314 log2down(uint32_t x)
315 {
316 return 31 - __builtin_clz(x);
317 }
318
319 /*
320 * Once we get the IO compression stats for the entire buffer, we update buffer_size_compressibility_dist,
321 * which helps us observe distribution across various io sizes and compression factors.
322 * The goal of next two functions is to get the index in this buffer_size_compressibility_dist table.
323 */
324
325 /*
326 * Maps IO size to a bucket between 0 - IO_COMPRESSION_STATS_MAX_SIZE_BUCKET
327 * for size < 4096 returns 0 and size > 1MB returns IO_COMPRESSION_STATS_MAX_SIZE_BUCKET (9).
328 * For IO sizes in-between we arrive at the index based on log2 function.
329 * sizes 4097 - 8192 => index = 1,
330 * sizes 8193 - 16384 => index = 2, and so on
331 */
332 #define SIZE_COMPRESSION_DIST_SIZE_BUCKET_MIN 4096
333 #define SIZE_COMPRESSION_DIST_SIZE_BUCKET_MAX (1024 * 1024)
334 static uint32_t
get_buffer_size_bucket(uint32_t size)335 get_buffer_size_bucket(uint32_t size)
336 {
337 if (size <= SIZE_COMPRESSION_DIST_SIZE_BUCKET_MIN) {
338 return 0;
339 }
340 if (size > SIZE_COMPRESSION_DIST_SIZE_BUCKET_MAX) {
341 return IOCS_BUFFER_MAX_BUCKET;
342 }
343 #define IOCS_INDEX_MAP_OFFSET 11
344 return log2down(size - 1) - IOCS_INDEX_MAP_OFFSET;
345 }
346
347 /*
348 * Maps compression factor to a bucket between 0 - IO_COMPRESSION_STATS_MAX_COMPRESSION_BUCKET
349 */
350 static uint32_t
get_buffer_compressibility_bucket(uint32_t uncompressed_size,uint32_t compressed_size)351 get_buffer_compressibility_bucket(uint32_t uncompressed_size, uint32_t compressed_size)
352 {
353 int saved_space_pc = (uncompressed_size - compressed_size) * 100 / uncompressed_size;
354
355 if (saved_space_pc < 0) {
356 saved_space_pc = 0;
357 }
358
359 /* saved_space_pc lies bw 0 - 100. log2(saved_space_pc) lies bw 0 - 6 */
360 return log2down(saved_space_pc);
361 }
362
363 void
io_compression_stats(buf_t bp)364 io_compression_stats(buf_t bp)
365 {
366 uint8_t *buf_ptr = NULL;
367 int bflags = bp->b_flags;
368 uint32_t compressed_size = 0;
369 uint32_t buf_cnt = buf_count(bp);
370 uint64_t duration = 0;
371 caddr_t vaddr = NULL;
372 vnode_t vn = buf_vnode(bp);
373 int err = 0;
374
375 if ((io_compression_stats_enable != 1) || (bflags & B_READ) || (buf_cnt <= 0)) {
376 return;
377 }
378
379 if (!lck_rw_try_lock_shared(&io_compression_stats_lock)) {
380 /* sysctl modifying IO compression stats parameters is in progress.
381 * Don't block, since malloc might be in progress. */
382 return;
383 }
384 /* re-check io_compression_stats_enable with lock */
385 if (io_compression_stats_enable != 1) {
386 goto out;
387 }
388
389 err = buf_map_range(bp, &vaddr);
390 if (!err) {
391 buf_ptr = (uint8_t *) vaddr;
392 }
393
394 if (buf_ptr != NULL) {
395 int64_t start = mach_absolute_time();
396 compressed_size = iocs_compress_buffer(vn, buf_ptr, buf_cnt);
397 absolutetime_to_nanoseconds(mach_absolute_time() - start, &duration);
398
399 if (compressed_size != 0) {
400 vnode_updateiocompressionbufferstats(vn, buf_cnt, compressed_size,
401 get_buffer_size_bucket(buf_cnt),
402 get_buffer_compressibility_bucket(buf_cnt, compressed_size));
403 }
404 }
405
406 KDBG_RELEASE(FSDBG_CODE(DBG_VFS, DBG_VFS_IO_COMPRESSION_STATS) | DBG_FUNC_NONE,
407 duration, io_compression_stats_block_size, compressed_size, buf_cnt, 0);
408
409 out:
410 lck_rw_unlock_shared(&io_compression_stats_lock);
411 if (buf_ptr != NULL) {
412 buf_unmap_range(bp);
413 }
414 }
415
416 static void
iocs_notify_user(void)417 iocs_notify_user(void)
418 {
419 mach_port_t user_port = MACH_PORT_NULL;
420 kern_return_t kr = host_get_iocompressionstats_port(host_priv_self(), &user_port);
421 if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
422 return;
423 }
424 iocompressionstats_notification(user_port, 0);
425 ipc_port_release_send(user_port);
426 }
427
428 static void
construct_iocs_sbe_from_vnode(struct vnode * vp,struct iocs_store_buffer_entry * iocs_sbe)429 construct_iocs_sbe_from_vnode(struct vnode *vp, struct iocs_store_buffer_entry *iocs_sbe)
430 {
431 int path_len = MAXPATHLEN;
432
433 if (vn_getpath(vp, vnpath_scratch_buf, &path_len) != 0) {
434 io_compression_stats_dbg("FAILED: Unable to get file path from vnode");
435 return;
436 }
437 /*
438 * Total path length is path_len, we can copy out IOCS_SBE_PATH_LEN bytes. We are interested
439 * in first segment of the path to try and figure out the process writing to the file, and we are
440 * interested in the last segment to figure out extention. So, in cases where
441 * IOCS_SBE_PATH_LEN < path_len, lets copy out first IOCS_PATH_START_BYTES_TO_COPY bytes and
442 * last IOCS_PATH_END_BYTES_TO_COPY (last segment includes the null character).
443 */
444 if (path_len > IOCS_SBE_PATH_LEN) {
445 strncpy(iocs_sbe->path_name, vnpath_scratch_buf, IOCS_PATH_START_BYTES_TO_COPY);
446 strncpy(iocs_sbe->path_name + IOCS_PATH_START_BYTES_TO_COPY,
447 vnpath_scratch_buf + path_len - IOCS_PATH_END_BYTES_TO_COPY,
448 IOCS_PATH_END_BYTES_TO_COPY);
449 } else {
450 strncpy(iocs_sbe->path_name, vnpath_scratch_buf, path_len);
451 }
452 memcpy(&iocs_sbe->iocs, vp->io_compression_stats, sizeof(struct io_compression_stats));
453 }
454
455 void
vnode_iocs_record_and_free(struct vnode * vp)456 vnode_iocs_record_and_free(struct vnode *vp)
457 {
458 int notify = 0;
459 struct iocs_store_buffer_entry *iocs_sbe = NULL;
460
461 if (!lck_mtx_try_lock(&iocs_store_buffer_lock)) {
462 goto out;
463 }
464
465 if (iocs_store_buffer.buffer == NULL) {
466 goto release;
467 }
468
469 assert(iocs_store_buffer.current_position + sizeof(struct iocs_store_buffer_entry) <= IOCS_STORE_BUFFER_SIZE);
470
471 iocs_sbe = (struct iocs_store_buffer_entry *)((uintptr_t)iocs_store_buffer.buffer + iocs_store_buffer.current_position);
472
473 construct_iocs_sbe_from_vnode(vp, iocs_sbe);
474
475 iocs_store_buffer.current_position += sizeof(struct iocs_store_buffer_entry);
476
477 if (iocs_store_buffer.current_position + sizeof(struct iocs_store_buffer_entry) > IOCS_STORE_BUFFER_SIZE) {
478 /* We've reached end of the buffer, move back to the top */
479 iocs_store_buffer.current_position = 0;
480 }
481
482 iocs_sb_bytes_since_last_mark += sizeof(struct iocs_store_buffer_entry);
483 iocs_sb_bytes_since_last_notification += sizeof(struct iocs_store_buffer_entry);
484
485 if ((iocs_sb_bytes_since_last_mark > IOCS_STORE_BUFFER_NOTIFY_AT) &&
486 (iocs_sb_bytes_since_last_notification > IOCS_STORE_BUFFER_NOTIFICATION_INTERVAL)) {
487 notify = 1;
488 iocs_sb_bytes_since_last_notification = 0;
489 }
490
491 release:
492 lck_mtx_unlock(&iocs_store_buffer_lock);
493 out:
494 /* We need to free io_compression_stats whether or not we were able to record it */
495 bzero(vp->io_compression_stats, sizeof(struct io_compression_stats));
496 zfree(io_compression_stats_zone, vp->io_compression_stats);
497 vp->io_compression_stats = NULL;
498 if (notify) {
499 iocs_notify_user();
500 }
501 }
502
503 struct vnode_iocs_context {
504 struct sysctl_req *addr;
505 unsigned long current_offset;
506 unsigned long length;
507 };
508
509 static int
vnode_iocs_callback(struct vnode * vp,void * vctx)510 vnode_iocs_callback(struct vnode *vp, void *vctx)
511 {
512 struct vnode_iocs_context *ctx = vctx;
513 struct sysctl_req *req = ctx->addr;
514 unsigned long current_offset = ctx->current_offset;
515 unsigned long length = ctx->length;
516 struct iocs_store_buffer_entry iocs_sbe_next;
517
518 if ((current_offset + sizeof(struct iocs_store_buffer_entry)) < length) {
519 if (vp->io_compression_stats != NULL) {
520 construct_iocs_sbe_from_vnode(vp, &iocs_sbe_next);
521
522 uint32_t error = copyout(&iocs_sbe_next, (user_addr_t)((unsigned char *)req->oldptr + current_offset), sizeof(struct iocs_store_buffer_entry));
523 if (error) {
524 return VNODE_RETURNED_DONE;
525 }
526
527 current_offset += sizeof(struct iocs_store_buffer_entry);
528 }
529 } else {
530 return VNODE_RETURNED_DONE;
531 }
532 ctx->current_offset = current_offset;
533
534 return VNODE_RETURNED;
535 }
536
537 static int
vfs_iocs_callback(mount_t mp,void * arg)538 vfs_iocs_callback(mount_t mp, void *arg)
539 {
540 if (mp->mnt_flag & MNT_LOCAL) {
541 vnode_iterate(mp, VNODE_ITERATE_ALL, vnode_iocs_callback, arg);
542 }
543
544 return VFS_RETURNED;
545 }
546
547 extern long numvnodes;
548
549 static int
550 sysctl_io_compression_dump_stats SYSCTL_HANDLER_ARGS
551 {
552 #pragma unused (arg1, arg2, oidp)
553
554 int32_t error = 0;
555 uint32_t inp_flag = 0;
556 uint32_t ret_len;
557
558 if (io_compression_stats_enable == 0) {
559 error = EINVAL;
560 goto out;
561 }
562
563 if ((req->newptr != USER_ADDR_NULL) && (req->newlen == sizeof(uint32_t))) {
564 error = SYSCTL_IN(req, &inp_flag, sizeof(uint32_t));
565 if (error) {
566 goto out;
567 }
568 switch (inp_flag) {
569 case IOCS_SYSCTL_LIVE:
570 case IOCS_SYSCTL_STORE_BUFFER_RD_ONLY:
571 case IOCS_SYSCTL_STORE_BUFFER_MARK:
572 break;
573 default:
574 error = EINVAL;
575 goto out;
576 }
577 } else {
578 error = EINVAL;
579 goto out;
580 }
581
582 if (req->oldptr == USER_ADDR_NULL) {
583 /* Query to figure out size of the buffer */
584 if (inp_flag & IOCS_SYSCTL_LIVE) {
585 req->oldidx = numvnodes * sizeof(struct iocs_store_buffer_entry);
586 } else {
587 /* Buffer size for archived case, let's keep it
588 * simple and return IOCS store buffer size */
589 req->oldidx = IOCS_STORE_BUFFER_SIZE;
590 }
591 goto out;
592 }
593
594 if (inp_flag & IOCS_SYSCTL_LIVE) {
595 struct vnode_iocs_context ctx;
596 bzero(&ctx, sizeof(struct vnode_iocs_context));
597 ctx.addr = req;
598 ctx.length = numvnodes * sizeof(struct iocs_store_buffer_entry);
599 vfs_iterate(0, vfs_iocs_callback, &ctx);
600 req->oldidx = ctx.current_offset;
601
602 goto out;
603 }
604
605 /* reading from store buffer */
606 lck_mtx_lock(&iocs_store_buffer_lock);
607
608 if (iocs_store_buffer.buffer == NULL) {
609 error = EINVAL;
610 goto release;
611 }
612 if (iocs_sb_bytes_since_last_mark == 0) {
613 req->oldidx = 0;
614 goto release;
615 }
616
617 int expected_size = 0;
618 /* Dry run to figure out amount of space required to copy out the
619 * iocs_store_buffer.buffer */
620 if (iocs_store_buffer.marked_point < iocs_store_buffer.current_position) {
621 expected_size = iocs_store_buffer.current_position - iocs_store_buffer.marked_point;
622 } else {
623 expected_size = IOCS_STORE_BUFFER_SIZE - iocs_store_buffer.marked_point;
624 expected_size += iocs_store_buffer.current_position;
625 }
626
627 if (req->oldlen < expected_size) {
628 error = ENOMEM;
629 req->oldidx = 0;
630 goto release;
631 }
632
633 if (iocs_store_buffer.marked_point < iocs_store_buffer.current_position) {
634 error = copyout((void *)((uintptr_t)iocs_store_buffer.buffer + iocs_store_buffer.marked_point),
635 req->oldptr,
636 iocs_store_buffer.current_position - iocs_store_buffer.marked_point);
637 if (error) {
638 req->oldidx = 0;
639 goto release;
640 }
641 ret_len = iocs_store_buffer.current_position - iocs_store_buffer.marked_point;
642 } else {
643 error = copyout((void *)((uintptr_t)iocs_store_buffer.buffer + iocs_store_buffer.marked_point),
644 req->oldptr,
645 IOCS_STORE_BUFFER_SIZE - iocs_store_buffer.marked_point);
646 if (error) {
647 req->oldidx = 0;
648 goto release;
649 }
650 ret_len = IOCS_STORE_BUFFER_SIZE - iocs_store_buffer.marked_point;
651
652 error = copyout(iocs_store_buffer.buffer,
653 req->oldptr + ret_len,
654 iocs_store_buffer.current_position);
655 if (error) {
656 req->oldidx = 0;
657 goto release;
658 }
659 ret_len += iocs_store_buffer.current_position;
660 }
661
662 req->oldidx = ret_len;
663 if ((ret_len != 0) && (inp_flag & IOCS_SYSCTL_STORE_BUFFER_MARK)) {
664 iocs_sb_bytes_since_last_mark = 0;
665 iocs_store_buffer.marked_point = iocs_store_buffer.current_position;
666 }
667 release:
668 lck_mtx_unlock(&iocs_store_buffer_lock);
669
670 out:
671 return error;
672 }
673 SYSCTL_PROC(_vfs, OID_AUTO, io_compression_dump_stats, CTLFLAG_WR | CTLTYPE_NODE, 0, 0, sysctl_io_compression_dump_stats, "-", "");
674
675 errno_t
vnode_updateiocompressionblockstats(vnode_t vp,uint32_t size_bucket)676 vnode_updateiocompressionblockstats(vnode_t vp, uint32_t size_bucket)
677 {
678 if (vp == NULL) {
679 return EINVAL;
680 }
681
682 if (size_bucket >= IOCS_BLOCK_NUM_SIZE_BUCKETS) {
683 return EINVAL;
684 }
685
686 if (vp->io_compression_stats == NULL) {
687 io_compression_stats_t iocs = zalloc_flags(io_compression_stats_zone,
688 Z_ZERO | Z_NOFAIL);
689 vnode_lock_spin(vp);
690 /* Re-check with lock */
691 if (vp->io_compression_stats == NULL) {
692 vp->io_compression_stats = iocs;
693 } else {
694 zfree(io_compression_stats_zone, iocs);
695 }
696 vnode_unlock(vp);
697 }
698 OSIncrementAtomic((SInt32 *)&vp->io_compression_stats->block_compressed_size_dist[size_bucket]);
699
700 return 0;
701 }
702 errno_t
vnode_updateiocompressionbufferstats(__unused vnode_t vp,__unused uint64_t uncompressed_size,__unused uint64_t compressed_size,__unused uint32_t size_bucket,__unused uint32_t compression_bucket)703 vnode_updateiocompressionbufferstats(__unused vnode_t vp, __unused uint64_t uncompressed_size, __unused uint64_t compressed_size, __unused uint32_t size_bucket, __unused uint32_t compression_bucket)
704 {
705 if (vp == NULL) {
706 return EINVAL;
707 }
708
709 /* vnode_updateiocompressionblockstats will always be called before vnode_updateiocompressionbufferstats.
710 * Hence vp->io_compression_stats should already be allocated */
711 if (vp->io_compression_stats == NULL) {
712 return EINVAL;
713 }
714
715 if ((size_bucket >= IOCS_BUFFER_NUM_SIZE_BUCKETS) || (compression_bucket >= IOCS_BUFFER_NUM_COMPRESSION_BUCKETS)) {
716 return EINVAL;
717 }
718
719 OSAddAtomic64(uncompressed_size, &vp->io_compression_stats->uncompressed_size);
720 OSAddAtomic64(compressed_size, &vp->io_compression_stats->compressed_size);
721
722 OSIncrementAtomic((SInt32 *)&vp->io_compression_stats->buffer_size_compression_dist[size_bucket][compression_bucket]);
723
724 return 0;
725 }
726