Lines Matching refs:buf
36 struct rchan_buf *buf = vmf->vma->vm_private_data; in relay_buf_fault() local
39 if (!buf) in relay_buf_fault()
42 page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT)); in relay_buf_fault()
83 static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma) in relay_mmap_buf() argument
87 if (!buf) in relay_mmap_buf()
90 if (length != (unsigned long)buf->chan->alloc_size) in relay_mmap_buf()
95 vma->vm_private_data = buf; in relay_mmap_buf()
108 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size) in relay_alloc_buf() argument
116 buf->page_array = relay_alloc_page_array(n_pages); in relay_alloc_buf()
117 if (!buf->page_array) in relay_alloc_buf()
121 buf->page_array[i] = alloc_page(GFP_KERNEL); in relay_alloc_buf()
122 if (unlikely(!buf->page_array[i])) in relay_alloc_buf()
124 set_page_private(buf->page_array[i], (unsigned long)buf); in relay_alloc_buf()
126 mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL); in relay_alloc_buf()
131 buf->page_count = n_pages; in relay_alloc_buf()
136 __free_page(buf->page_array[j]); in relay_alloc_buf()
137 relay_free_page_array(buf->page_array); in relay_alloc_buf()
149 struct rchan_buf *buf; in relay_create_buf() local
154 buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL); in relay_create_buf()
155 if (!buf) in relay_create_buf()
157 buf->padding = kmalloc_array(chan->n_subbufs, sizeof(size_t), in relay_create_buf()
159 if (!buf->padding) in relay_create_buf()
162 buf->start = relay_alloc_buf(buf, &chan->alloc_size); in relay_create_buf()
163 if (!buf->start) in relay_create_buf()
166 buf->chan = chan; in relay_create_buf()
167 kref_get(&buf->chan->kref); in relay_create_buf()
168 return buf; in relay_create_buf()
171 kfree(buf->padding); in relay_create_buf()
172 kfree(buf); in relay_create_buf()
185 free_percpu(chan->buf); in relay_destroy_channel()
193 static void relay_destroy_buf(struct rchan_buf *buf) in relay_destroy_buf() argument
195 struct rchan *chan = buf->chan; in relay_destroy_buf()
198 if (likely(buf->start)) { in relay_destroy_buf()
199 vunmap(buf->start); in relay_destroy_buf()
200 for (i = 0; i < buf->page_count; i++) in relay_destroy_buf()
201 __free_page(buf->page_array[i]); in relay_destroy_buf()
202 relay_free_page_array(buf->page_array); in relay_destroy_buf()
204 *per_cpu_ptr(chan->buf, buf->cpu) = NULL; in relay_destroy_buf()
205 kfree(buf->padding); in relay_destroy_buf()
206 kfree(buf); in relay_destroy_buf()
220 struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref); in relay_remove_buf() local
221 relay_destroy_buf(buf); in relay_remove_buf()
230 static int relay_buf_empty(struct rchan_buf *buf) in relay_buf_empty() argument
232 return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1; in relay_buf_empty()
241 int relay_buf_full(struct rchan_buf *buf) in relay_buf_full() argument
243 size_t ready = buf->subbufs_produced - buf->subbufs_consumed; in relay_buf_full()
244 return (ready >= buf->chan->n_subbufs) ? 1 : 0; in relay_buf_full()
252 static int relay_subbuf_start(struct rchan_buf *buf, void *subbuf, in relay_subbuf_start() argument
255 if (!buf->chan->cb->subbuf_start) in relay_subbuf_start()
256 return !relay_buf_full(buf); in relay_subbuf_start()
258 return buf->chan->cb->subbuf_start(buf, subbuf, in relay_subbuf_start()
270 struct rchan_buf *buf; in wakeup_readers() local
272 buf = container_of(work, struct rchan_buf, wakeup_work); in wakeup_readers()
273 wake_up_interruptible(&buf->read_wait); in wakeup_readers()
283 static void __relay_reset(struct rchan_buf *buf, unsigned int init) in __relay_reset() argument
288 init_waitqueue_head(&buf->read_wait); in __relay_reset()
289 kref_init(&buf->kref); in __relay_reset()
290 init_irq_work(&buf->wakeup_work, wakeup_readers); in __relay_reset()
292 irq_work_sync(&buf->wakeup_work); in __relay_reset()
295 buf->subbufs_produced = 0; in __relay_reset()
296 buf->subbufs_consumed = 0; in __relay_reset()
297 buf->bytes_consumed = 0; in __relay_reset()
298 buf->finalized = 0; in __relay_reset()
299 buf->data = buf->start; in __relay_reset()
300 buf->offset = 0; in __relay_reset()
302 for (i = 0; i < buf->chan->n_subbufs; i++) in __relay_reset()
303 buf->padding[i] = 0; in __relay_reset()
305 relay_subbuf_start(buf, buf->data, NULL, 0); in __relay_reset()
321 struct rchan_buf *buf; in relay_reset() local
327 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) { in relay_reset()
328 __relay_reset(buf, 0); in relay_reset()
334 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_reset()
335 __relay_reset(buf, 0); in relay_reset()
340 static inline void relay_set_buf_dentry(struct rchan_buf *buf, in relay_set_buf_dentry() argument
343 buf->dentry = dentry; in relay_set_buf_dentry()
344 d_inode(buf->dentry)->i_size = buf->early_bytes; in relay_set_buf_dentry()
348 struct rchan_buf *buf, in relay_create_buf_file() argument
360 S_IRUSR, buf, in relay_create_buf_file()
377 struct rchan_buf *buf; in relay_open_buf() local
381 return *per_cpu_ptr(chan->buf, 0); in relay_open_buf()
383 buf = relay_create_buf(chan); in relay_open_buf()
384 if (!buf) in relay_open_buf()
388 dentry = relay_create_buf_file(chan, buf, cpu); in relay_open_buf()
391 relay_set_buf_dentry(buf, dentry); in relay_open_buf()
395 S_IRUSR, buf, in relay_open_buf()
401 buf->cpu = cpu; in relay_open_buf()
402 __relay_reset(buf, 1); in relay_open_buf()
405 *per_cpu_ptr(chan->buf, 0) = buf; in relay_open_buf()
406 buf->cpu = 0; in relay_open_buf()
409 return buf; in relay_open_buf()
412 relay_destroy_buf(buf); in relay_open_buf()
424 static void relay_close_buf(struct rchan_buf *buf) in relay_close_buf() argument
426 buf->finalized = 1; in relay_close_buf()
427 irq_work_sync(&buf->wakeup_work); in relay_close_buf()
428 buf->chan->cb->remove_buf_file(buf->dentry); in relay_close_buf()
429 kref_put(&buf->kref, relay_remove_buf); in relay_close_buf()
435 struct rchan_buf *buf; in relay_prepare_cpu() local
439 if (*per_cpu_ptr(chan->buf, cpu)) in relay_prepare_cpu()
441 buf = relay_open_buf(chan, cpu); in relay_prepare_cpu()
442 if (!buf) { in relay_prepare_cpu()
447 *per_cpu_ptr(chan->buf, cpu) = buf; in relay_prepare_cpu()
482 struct rchan_buf *buf; in relay_open() local
495 chan->buf = alloc_percpu(struct rchan_buf *); in relay_open()
496 if (!chan->buf) { in relay_open()
516 buf = relay_open_buf(chan, i); in relay_open()
517 if (!buf) in relay_open()
519 *per_cpu_ptr(chan->buf, i) = buf; in relay_open()
528 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_open()
529 relay_close_buf(buf); in relay_open()
539 struct rchan_buf *buf; member
548 relay_set_buf_dentry(p->buf, p->dentry); in __relay_set_buf_dentry()
574 struct rchan_buf *buf; in relay_late_setup_files() local
593 buf = *per_cpu_ptr(chan->buf, 0); in relay_late_setup_files()
594 if (!WARN_ON_ONCE(!buf)) { in relay_late_setup_files()
595 dentry = relay_create_buf_file(chan, buf, 0); in relay_late_setup_files()
597 relay_set_buf_dentry(buf, dentry); in relay_late_setup_files()
612 buf = *per_cpu_ptr(chan->buf, i); in relay_late_setup_files()
613 if (unlikely(!buf)) { in relay_late_setup_files()
619 dentry = relay_create_buf_file(chan, buf, i); in relay_late_setup_files()
627 relay_set_buf_dentry(buf, dentry); in relay_late_setup_files()
630 disp.buf = buf; in relay_late_setup_files()
658 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length) in relay_switch_subbuf() argument
663 if (unlikely(length > buf->chan->subbuf_size)) in relay_switch_subbuf()
666 if (buf->offset != buf->chan->subbuf_size + 1) { in relay_switch_subbuf()
667 buf->prev_padding = buf->chan->subbuf_size - buf->offset; in relay_switch_subbuf()
668 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; in relay_switch_subbuf()
669 buf->padding[old_subbuf] = buf->prev_padding; in relay_switch_subbuf()
670 buf->subbufs_produced++; in relay_switch_subbuf()
671 if (buf->dentry) in relay_switch_subbuf()
672 d_inode(buf->dentry)->i_size += in relay_switch_subbuf()
673 buf->chan->subbuf_size - in relay_switch_subbuf()
674 buf->padding[old_subbuf]; in relay_switch_subbuf()
676 buf->early_bytes += buf->chan->subbuf_size - in relay_switch_subbuf()
677 buf->padding[old_subbuf]; in relay_switch_subbuf()
679 if (waitqueue_active(&buf->read_wait)) { in relay_switch_subbuf()
686 irq_work_queue(&buf->wakeup_work); in relay_switch_subbuf()
690 old = buf->data; in relay_switch_subbuf()
691 new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; in relay_switch_subbuf()
692 new = buf->start + new_subbuf * buf->chan->subbuf_size; in relay_switch_subbuf()
693 buf->offset = 0; in relay_switch_subbuf()
694 if (!relay_subbuf_start(buf, new, old, buf->prev_padding)) { in relay_switch_subbuf()
695 buf->offset = buf->chan->subbuf_size + 1; in relay_switch_subbuf()
698 buf->data = new; in relay_switch_subbuf()
699 buf->padding[new_subbuf] = 0; in relay_switch_subbuf()
701 if (unlikely(length + buf->offset > buf->chan->subbuf_size)) in relay_switch_subbuf()
707 buf->chan->last_toobig = length; in relay_switch_subbuf()
729 struct rchan_buf *buf; in relay_subbufs_consumed() local
734 buf = *per_cpu_ptr(chan->buf, cpu); in relay_subbufs_consumed()
735 if (!buf || subbufs_consumed > chan->n_subbufs) in relay_subbufs_consumed()
738 if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed) in relay_subbufs_consumed()
739 buf->subbufs_consumed = buf->subbufs_produced; in relay_subbufs_consumed()
741 buf->subbufs_consumed += subbufs_consumed; in relay_subbufs_consumed()
753 struct rchan_buf *buf; in relay_close() local
760 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) in relay_close()
761 relay_close_buf(buf); in relay_close()
764 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_close()
765 relay_close_buf(buf); in relay_close()
786 struct rchan_buf *buf; in relay_flush() local
792 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) { in relay_flush()
793 relay_switch_subbuf(buf, 0); in relay_flush()
799 if ((buf = *per_cpu_ptr(chan->buf, i))) in relay_flush()
800 relay_switch_subbuf(buf, 0); in relay_flush()
814 struct rchan_buf *buf = inode->i_private; in relay_file_open() local
815 kref_get(&buf->kref); in relay_file_open()
816 filp->private_data = buf; in relay_file_open()
830 struct rchan_buf *buf = filp->private_data; in relay_file_mmap() local
831 return relay_mmap_buf(buf, vma); in relay_file_mmap()
844 struct rchan_buf *buf = filp->private_data; in relay_file_poll() local
846 if (buf->finalized) in relay_file_poll()
850 poll_wait(filp, &buf->read_wait, wait); in relay_file_poll()
851 if (!relay_buf_empty(buf)) in relay_file_poll()
868 struct rchan_buf *buf = filp->private_data; in relay_file_release() local
869 kref_put(&buf->kref, relay_remove_buf); in relay_file_release()
877 static void relay_file_read_consume(struct rchan_buf *buf, in relay_file_read_consume() argument
881 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_consume()
882 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_consume()
885 if (buf->subbufs_produced == buf->subbufs_consumed && in relay_file_read_consume()
886 buf->offset == buf->bytes_consumed) in relay_file_read_consume()
889 if (buf->bytes_consumed + bytes_consumed > subbuf_size) { in relay_file_read_consume()
890 relay_subbufs_consumed(buf->chan, buf->cpu, 1); in relay_file_read_consume()
891 buf->bytes_consumed = 0; in relay_file_read_consume()
894 buf->bytes_consumed += bytes_consumed; in relay_file_read_consume()
896 read_subbuf = buf->subbufs_consumed % n_subbufs; in relay_file_read_consume()
898 read_subbuf = read_pos / buf->chan->subbuf_size; in relay_file_read_consume()
899 if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) { in relay_file_read_consume()
900 if ((read_subbuf == buf->subbufs_produced % n_subbufs) && in relay_file_read_consume()
901 (buf->offset == subbuf_size)) in relay_file_read_consume()
903 relay_subbufs_consumed(buf->chan, buf->cpu, 1); in relay_file_read_consume()
904 buf->bytes_consumed = 0; in relay_file_read_consume()
911 static int relay_file_read_avail(struct rchan_buf *buf) in relay_file_read_avail() argument
913 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_avail()
914 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_avail()
915 size_t produced = buf->subbufs_produced; in relay_file_read_avail()
918 relay_file_read_consume(buf, 0, 0); in relay_file_read_avail()
920 consumed = buf->subbufs_consumed; in relay_file_read_avail()
922 if (unlikely(buf->offset > subbuf_size)) { in relay_file_read_avail()
930 buf->subbufs_consumed = consumed; in relay_file_read_avail()
931 buf->bytes_consumed = 0; in relay_file_read_avail()
934 produced = (produced % n_subbufs) * subbuf_size + buf->offset; in relay_file_read_avail()
935 consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed; in relay_file_read_avail()
941 if (buf->offset == subbuf_size && in relay_file_read_avail()
942 buf->subbufs_produced > buf->subbufs_consumed) in relay_file_read_avail()
956 struct rchan_buf *buf) in relay_file_read_subbuf_avail() argument
960 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_subbuf_avail()
962 write_subbuf = (buf->data - buf->start) / subbuf_size; in relay_file_read_subbuf_avail()
963 write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset; in relay_file_read_subbuf_avail()
966 padding = buf->padding[read_subbuf]; in relay_file_read_subbuf_avail()
985 static size_t relay_file_read_start_pos(struct rchan_buf *buf) in relay_file_read_start_pos() argument
988 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_start_pos()
989 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_start_pos()
990 size_t consumed = buf->subbufs_consumed % n_subbufs; in relay_file_read_start_pos()
991 size_t read_pos = (consumed * subbuf_size + buf->bytes_consumed) in relay_file_read_start_pos()
995 padding = buf->padding[read_subbuf]; in relay_file_read_start_pos()
1012 static size_t relay_file_read_end_pos(struct rchan_buf *buf, in relay_file_read_end_pos() argument
1017 size_t subbuf_size = buf->chan->subbuf_size; in relay_file_read_end_pos()
1018 size_t n_subbufs = buf->chan->n_subbufs; in relay_file_read_end_pos()
1021 padding = buf->padding[read_subbuf]; in relay_file_read_end_pos()
1037 struct rchan_buf *buf = filp->private_data; in relay_file_read() local
1049 if (!relay_file_read_avail(buf)) in relay_file_read()
1052 read_start = relay_file_read_start_pos(buf); in relay_file_read()
1053 avail = relay_file_read_subbuf_avail(read_start, buf); in relay_file_read()
1058 from = buf->start + read_start; in relay_file_read()
1067 relay_file_read_consume(buf, read_start, ret); in relay_file_read()
1068 *ppos = relay_file_read_end_pos(buf, read_start, ret); in relay_file_read()