1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2009-2012,2016 Microsoft Corp.
3 * Copyright (c) 2012 NetApp Inc.
4 * Copyright (c) 2012 Citrix Inc.
5 * All rights reserved.
6 */
7
8 #include <unistd.h>
9 #include <stdint.h>
10 #include <stdbool.h>
11 #include <string.h>
12 #include <sys/uio.h>
13
14 #include <rte_eal.h>
15 #include <rte_tailq.h>
16 #include <rte_log.h>
17 #include <rte_malloc.h>
18 #include <rte_bus.h>
19 #include <rte_atomic.h>
20 #include <rte_memory.h>
21 #include <rte_pause.h>
22 #include <rte_bus_vmbus.h>
23
24 #include "private.h"
25
26 /* Increase bufring index by inc with wraparound */
vmbus_br_idxinc(uint32_t idx,uint32_t inc,uint32_t sz)27 static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz)
28 {
29 idx += inc;
30 if (idx >= sz)
31 idx -= sz;
32
33 return idx;
34 }
35
vmbus_br_setup(struct vmbus_br * br,void * buf,unsigned int blen)36 void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen)
37 {
38 br->vbr = buf;
39 br->windex = br->vbr->windex;
40 br->dsize = blen - sizeof(struct vmbus_bufring);
41 }
42
43 /*
44 * When we write to the ring buffer, check if the host needs to be
45 * signaled.
46 *
47 * The contract:
48 * - The host guarantees that while it is draining the TX bufring,
49 * it will set the br_imask to indicate it does not need to be
50 * interrupted when new data are added.
51 * - The host guarantees that it will completely drain the TX bufring
52 * before exiting the read loop. Further, once the TX bufring is
53 * empty, it will clear the br_imask and re-check to see if new
54 * data have arrived.
55 */
56 static inline bool
vmbus_txbr_need_signal(const struct vmbus_bufring * vbr,uint32_t old_windex)57 vmbus_txbr_need_signal(const struct vmbus_bufring *vbr, uint32_t old_windex)
58 {
59 rte_smp_mb();
60 if (vbr->imask)
61 return false;
62
63 rte_smp_rmb();
64
65 /*
66 * This is the only case we need to signal when the
67 * ring transitions from being empty to non-empty.
68 */
69 return old_windex == vbr->rindex;
70 }
71
72 static inline uint32_t
vmbus_txbr_copyto(const struct vmbus_br * tbr,uint32_t windex,const void * src0,uint32_t cplen)73 vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex,
74 const void *src0, uint32_t cplen)
75 {
76 uint8_t *br_data = tbr->vbr->data;
77 uint32_t br_dsize = tbr->dsize;
78 const uint8_t *src = src0;
79
80 /* XXX use double mapping like Linux kernel? */
81 if (cplen > br_dsize - windex) {
82 uint32_t fraglen = br_dsize - windex;
83
84 /* Wrap-around detected */
85 memcpy(br_data + windex, src, fraglen);
86 memcpy(br_data, src + fraglen, cplen - fraglen);
87 } else {
88 memcpy(br_data + windex, src, cplen);
89 }
90
91 return vmbus_br_idxinc(windex, cplen, br_dsize);
92 }
93
94 /*
95 * Write scattered channel packet to TX bufring.
96 *
97 * The offset of this channel packet is written as a 64bits value
98 * immediately after this channel packet.
99 *
100 * The write goes through three stages:
101 * 1. Reserve space in ring buffer for the new data.
102 * Writer atomically moves priv_write_index.
103 * 2. Copy the new data into the ring.
104 * 3. Update the tail of the ring (visible to host) that indicates
105 * next read location. Writer updates write_index
106 */
107 int
vmbus_txbr_write(struct vmbus_br * tbr,const struct iovec iov[],int iovlen,bool * need_sig)108 vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
109 bool *need_sig)
110 {
111 struct vmbus_bufring *vbr = tbr->vbr;
112 uint32_t ring_size = tbr->dsize;
113 uint32_t old_windex, next_windex, windex, total;
114 uint64_t save_windex;
115 int i;
116
117 total = 0;
118 for (i = 0; i < iovlen; i++)
119 total += iov[i].iov_len;
120 total += sizeof(save_windex);
121
122 /* Reserve space in ring */
123 do {
124 uint32_t avail;
125
126 /* Get current free location */
127 old_windex = tbr->windex;
128
129 /* Prevent compiler reordering this with calculation */
130 rte_compiler_barrier();
131
132 avail = vmbus_br_availwrite(tbr, old_windex);
133
134 /* If not enough space in ring, then tell caller. */
135 if (avail <= total)
136 return -EAGAIN;
137
138 next_windex = vmbus_br_idxinc(old_windex, total, ring_size);
139
140 /* Atomic update of next write_index for other threads */
141 } while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex));
142
143 /* Space from old..new is now reserved */
144 windex = old_windex;
145 for (i = 0; i < iovlen; i++) {
146 windex = vmbus_txbr_copyto(tbr, windex,
147 iov[i].iov_base, iov[i].iov_len);
148 }
149
150 /* Set the offset of the current channel packet. */
151 save_windex = ((uint64_t)old_windex) << 32;
152 windex = vmbus_txbr_copyto(tbr, windex, &save_windex,
153 sizeof(save_windex));
154
155 /* The region reserved should match region used */
156 RTE_ASSERT(windex == next_windex);
157
158 /* Ensure that data is available before updating host index */
159 rte_smp_wmb();
160
161 /* Checkin for our reservation. wait for our turn to update host */
162 while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex))
163 rte_pause();
164
165 /* If host had read all data before this, then need to signal */
166 *need_sig |= vmbus_txbr_need_signal(vbr, old_windex);
167 return 0;
168 }
169
170 static inline uint32_t
vmbus_rxbr_copyfrom(const struct vmbus_br * rbr,uint32_t rindex,void * dst0,size_t cplen)171 vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex,
172 void *dst0, size_t cplen)
173 {
174 const uint8_t *br_data = rbr->vbr->data;
175 uint32_t br_dsize = rbr->dsize;
176 uint8_t *dst = dst0;
177
178 if (cplen > br_dsize - rindex) {
179 uint32_t fraglen = br_dsize - rindex;
180
181 /* Wrap-around detected. */
182 memcpy(dst, br_data + rindex, fraglen);
183 memcpy(dst + fraglen, br_data, cplen - fraglen);
184 } else {
185 memcpy(dst, br_data + rindex, cplen);
186 }
187
188 return vmbus_br_idxinc(rindex, cplen, br_dsize);
189 }
190
191 /* Copy data from receive ring but don't change index */
192 int
vmbus_rxbr_peek(const struct vmbus_br * rbr,void * data,size_t dlen)193 vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen)
194 {
195 uint32_t avail;
196
197 /*
198 * The requested data and the 64bits channel packet
199 * offset should be there at least.
200 */
201 avail = vmbus_br_availread(rbr);
202 if (avail < dlen + sizeof(uint64_t))
203 return -EAGAIN;
204
205 vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen);
206 return 0;
207 }
208
209 /*
210 * Copy data from receive ring and change index
211 * NOTE:
212 * We assume (dlen + skip) == sizeof(channel packet).
213 */
214 int
vmbus_rxbr_read(struct vmbus_br * rbr,void * data,size_t dlen,size_t skip)215 vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip)
216 {
217 struct vmbus_bufring *vbr = rbr->vbr;
218 uint32_t br_dsize = rbr->dsize;
219 uint32_t rindex;
220
221 if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t))
222 return -EAGAIN;
223
224 /* Record where host was when we started read (for debug) */
225 rbr->windex = rbr->vbr->windex;
226
227 /*
228 * Copy channel packet from RX bufring.
229 */
230 rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize);
231 rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen);
232
233 /*
234 * Discard this channel packet's 64bits offset, which is useless to us.
235 */
236 rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize);
237
238 /* Update the read index _after_ the channel packet is fetched. */
239 rte_compiler_barrier();
240
241 vbr->rindex = rindex;
242
243 return 0;
244 }
245