1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2018 Microsoft Corp.
3 * Copyright (c) 2010-2012 Citrix Inc.
4 * Copyright (c) 2012 NetApp Inc.
5 * All rights reserved.
6 */
7
8 /*
9 * Network Virtualization Service.
10 */
11
12
13 #include <stdint.h>
14 #include <string.h>
15 #include <stdio.h>
16 #include <errno.h>
17 #include <unistd.h>
18
19 #include <rte_ethdev.h>
20 #include <rte_string_fns.h>
21 #include <rte_memzone.h>
22 #include <rte_malloc.h>
23 #include <rte_atomic.h>
24 #include <rte_branch_prediction.h>
25 #include <rte_ether.h>
26 #include <rte_common.h>
27 #include <rte_errno.h>
28 #include <rte_cycles.h>
29 #include <rte_memory.h>
30 #include <rte_eal.h>
31 #include <rte_dev.h>
32 #include <rte_bus_vmbus.h>
33
34 #include "hn_logs.h"
35 #include "hn_var.h"
36 #include "hn_nvs.h"
37
38 static const uint32_t hn_nvs_version[] = {
39 NVS_VERSION_61,
40 NVS_VERSION_6,
41 NVS_VERSION_5,
42 NVS_VERSION_4,
43 NVS_VERSION_2,
44 NVS_VERSION_1
45 };
46
hn_nvs_req_send(struct hn_data * hv,void * req,uint32_t reqlen)47 static int hn_nvs_req_send(struct hn_data *hv,
48 void *req, uint32_t reqlen)
49 {
50 return rte_vmbus_chan_send(hn_primary_chan(hv),
51 VMBUS_CHANPKT_TYPE_INBAND,
52 req, reqlen, 0,
53 VMBUS_CHANPKT_FLAG_NONE, NULL);
54 }
55
56 static int
__hn_nvs_execute(struct hn_data * hv,void * req,uint32_t reqlen,void * resp,uint32_t resplen,uint32_t type)57 __hn_nvs_execute(struct hn_data *hv,
58 void *req, uint32_t reqlen,
59 void *resp, uint32_t resplen,
60 uint32_t type)
61 {
62 struct vmbus_channel *chan = hn_primary_chan(hv);
63 char buffer[NVS_RESPSIZE_MAX];
64 const struct hn_nvs_hdr *hdr;
65 uint64_t xactid;
66 uint32_t len;
67 int ret;
68
69 /* Send request to ring buffer */
70 ret = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND,
71 req, reqlen, 0,
72 VMBUS_CHANPKT_FLAG_RC, NULL);
73
74 if (ret) {
75 PMD_DRV_LOG(ERR, "send request failed: %d", ret);
76 return ret;
77 }
78
79 retry:
80 len = sizeof(buffer);
81 ret = rte_vmbus_chan_recv(chan, buffer, &len, &xactid);
82 if (ret == -EAGAIN) {
83 rte_delay_us(HN_CHAN_INTERVAL_US);
84 goto retry;
85 }
86
87 if (ret < 0) {
88 PMD_DRV_LOG(ERR, "recv response failed: %d", ret);
89 return ret;
90 }
91
92 if (len < sizeof(*hdr)) {
93 PMD_DRV_LOG(ERR, "response missing NVS header");
94 return -EINVAL;
95 }
96
97 hdr = (struct hn_nvs_hdr *)buffer;
98
99 /* Silently drop received packets while waiting for response */
100 switch (hdr->type) {
101 case NVS_TYPE_RNDIS:
102 hn_nvs_ack_rxbuf(chan, xactid);
103 /* fallthrough */
104
105 case NVS_TYPE_TXTBL_NOTE:
106 PMD_DRV_LOG(DEBUG, "discard packet type 0x%x", hdr->type);
107 goto retry;
108 }
109
110 if (hdr->type != type) {
111 PMD_DRV_LOG(ERR, "unexpected NVS resp %#x, expect %#x",
112 hdr->type, type);
113 return -EINVAL;
114 }
115
116 if (len < resplen) {
117 PMD_DRV_LOG(ERR,
118 "invalid NVS resp len %u (expect %u)",
119 len, resplen);
120 return -EINVAL;
121 }
122
123 memcpy(resp, buffer, resplen);
124
125 /* All pass! */
126 return 0;
127 }
128
129
130 /*
131 * Execute one control command and get the response.
132 * Only one command can be active on a channel at once
133 * Unlike BSD, DPDK does not have an interrupt context
134 * so the polling is required to wait for response.
135 */
136 static int
hn_nvs_execute(struct hn_data * hv,void * req,uint32_t reqlen,void * resp,uint32_t resplen,uint32_t type)137 hn_nvs_execute(struct hn_data *hv,
138 void *req, uint32_t reqlen,
139 void *resp, uint32_t resplen,
140 uint32_t type)
141 {
142 struct hn_rx_queue *rxq = hv->primary;
143 int ret;
144
145 rte_spinlock_lock(&rxq->ring_lock);
146 ret = __hn_nvs_execute(hv, req, reqlen, resp, resplen, type);
147 rte_spinlock_unlock(&rxq->ring_lock);
148
149 return ret;
150 }
151
152 static int
hn_nvs_doinit(struct hn_data * hv,uint32_t nvs_ver)153 hn_nvs_doinit(struct hn_data *hv, uint32_t nvs_ver)
154 {
155 struct hn_nvs_init init;
156 struct hn_nvs_init_resp resp;
157 uint32_t status;
158 int error;
159
160 memset(&init, 0, sizeof(init));
161 init.type = NVS_TYPE_INIT;
162 init.ver_min = nvs_ver;
163 init.ver_max = nvs_ver;
164
165 error = hn_nvs_execute(hv, &init, sizeof(init),
166 &resp, sizeof(resp),
167 NVS_TYPE_INIT_RESP);
168 if (error)
169 return error;
170
171 status = resp.status;
172 if (status != NVS_STATUS_OK) {
173 /* Not fatal, try other versions */
174 PMD_INIT_LOG(DEBUG, "nvs init failed for ver 0x%x",
175 nvs_ver);
176 return -EINVAL;
177 }
178
179 return 0;
180 }
181
182 static int
hn_nvs_conn_rxbuf(struct hn_data * hv)183 hn_nvs_conn_rxbuf(struct hn_data *hv)
184 {
185 struct hn_nvs_rxbuf_conn conn;
186 struct hn_nvs_rxbuf_connresp resp;
187 uint32_t status;
188 int error;
189
190 /* Kernel has already setup RXBUF on primary channel. */
191
192 /*
193 * Connect RXBUF to NVS.
194 */
195 conn.type = NVS_TYPE_RXBUF_CONN;
196 conn.gpadl = hv->rxbuf_res->phys_addr;
197 conn.sig = NVS_RXBUF_SIG;
198 PMD_DRV_LOG(DEBUG, "connect rxbuff va=%p gpad=%#" PRIx64,
199 hv->rxbuf_res->addr,
200 hv->rxbuf_res->phys_addr);
201
202 error = hn_nvs_execute(hv, &conn, sizeof(conn),
203 &resp, sizeof(resp),
204 NVS_TYPE_RXBUF_CONNRESP);
205 if (error) {
206 PMD_DRV_LOG(ERR,
207 "exec nvs rxbuf conn failed: %d",
208 error);
209 return error;
210 }
211
212 status = resp.status;
213 if (status != NVS_STATUS_OK) {
214 PMD_DRV_LOG(ERR,
215 "nvs rxbuf conn failed: %x", status);
216 return -EIO;
217 }
218 if (resp.nsect != 1) {
219 PMD_DRV_LOG(ERR,
220 "nvs rxbuf response num sections %u != 1",
221 resp.nsect);
222 return -EIO;
223 }
224
225 PMD_DRV_LOG(INFO,
226 "receive buffer size %u count %u",
227 resp.nvs_sect[0].slotsz,
228 resp.nvs_sect[0].slotcnt);
229 hv->rxbuf_section_cnt = resp.nvs_sect[0].slotcnt;
230
231 /*
232 * Primary queue's rxbuf_info is not allocated at creation time.
233 * Now we can allocate it after we figure out the slotcnt.
234 */
235 hv->primary->rxbuf_info = rte_calloc("HN_RXBUF_INFO",
236 hv->rxbuf_section_cnt,
237 sizeof(*hv->primary->rxbuf_info),
238 RTE_CACHE_LINE_SIZE);
239 if (!hv->primary->rxbuf_info) {
240 PMD_DRV_LOG(ERR,
241 "could not allocate rxbuf info");
242 return -ENOMEM;
243 }
244
245 return 0;
246 }
247
248 static void
hn_nvs_disconn_rxbuf(struct hn_data * hv)249 hn_nvs_disconn_rxbuf(struct hn_data *hv)
250 {
251 struct hn_nvs_rxbuf_disconn disconn;
252 int error;
253
254 /*
255 * Disconnect RXBUF from NVS.
256 */
257 memset(&disconn, 0, sizeof(disconn));
258 disconn.type = NVS_TYPE_RXBUF_DISCONN;
259 disconn.sig = NVS_RXBUF_SIG;
260
261 /* NOTE: No response. */
262 error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
263 if (error) {
264 PMD_DRV_LOG(ERR,
265 "send nvs rxbuf disconn failed: %d",
266 error);
267 }
268
269 /*
270 * Linger long enough for NVS to disconnect RXBUF.
271 */
272 rte_delay_ms(200);
273 }
274
275 static void
hn_nvs_disconn_chim(struct hn_data * hv)276 hn_nvs_disconn_chim(struct hn_data *hv)
277 {
278 int error;
279
280 if (hv->chim_cnt != 0) {
281 struct hn_nvs_chim_disconn disconn;
282
283 /* Disconnect chimney sending buffer from NVS. */
284 memset(&disconn, 0, sizeof(disconn));
285 disconn.type = NVS_TYPE_CHIM_DISCONN;
286 disconn.sig = NVS_CHIM_SIG;
287
288 /* NOTE: No response. */
289 error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
290
291 if (error) {
292 PMD_DRV_LOG(ERR,
293 "send nvs chim disconn failed: %d", error);
294 }
295
296 hv->chim_cnt = 0;
297 /*
298 * Linger long enough for NVS to disconnect chimney
299 * sending buffer.
300 */
301 rte_delay_ms(200);
302 }
303 }
304
305 static int
hn_nvs_conn_chim(struct hn_data * hv)306 hn_nvs_conn_chim(struct hn_data *hv)
307 {
308 struct hn_nvs_chim_conn chim;
309 struct hn_nvs_chim_connresp resp;
310 uint32_t sectsz;
311 unsigned long len = hv->chim_res->len;
312 int error;
313
314 /* Connect chimney sending buffer to NVS */
315 memset(&chim, 0, sizeof(chim));
316 chim.type = NVS_TYPE_CHIM_CONN;
317 chim.gpadl = hv->chim_res->phys_addr;
318 chim.sig = NVS_CHIM_SIG;
319 PMD_DRV_LOG(DEBUG, "connect send buf va=%p gpad=%#" PRIx64,
320 hv->chim_res->addr,
321 hv->chim_res->phys_addr);
322
323 error = hn_nvs_execute(hv, &chim, sizeof(chim),
324 &resp, sizeof(resp),
325 NVS_TYPE_CHIM_CONNRESP);
326 if (error) {
327 PMD_DRV_LOG(ERR, "exec nvs chim conn failed");
328 return error;
329 }
330
331 if (resp.status != NVS_STATUS_OK) {
332 PMD_DRV_LOG(ERR, "nvs chim conn failed: %x",
333 resp.status);
334 return -EIO;
335 }
336
337 sectsz = resp.sectsz;
338 if (sectsz == 0 || sectsz & (sizeof(uint32_t) - 1)) {
339 /* Can't use chimney sending buffer; done! */
340 PMD_DRV_LOG(NOTICE,
341 "invalid chimney sending buffer section size: %u",
342 sectsz);
343 error = -EINVAL;
344 goto cleanup;
345 }
346
347 hv->chim_szmax = sectsz;
348 hv->chim_cnt = len / sectsz;
349
350 PMD_DRV_LOG(INFO, "send buffer %lu section size:%u, count:%u",
351 len, hv->chim_szmax, hv->chim_cnt);
352
353 /* Done! */
354 return 0;
355
356 cleanup:
357 hn_nvs_disconn_chim(hv);
358 return error;
359 }
360
361 /*
362 * Configure MTU and enable VLAN.
363 */
364 static int
hn_nvs_conf_ndis(struct hn_data * hv,unsigned int mtu)365 hn_nvs_conf_ndis(struct hn_data *hv, unsigned int mtu)
366 {
367 struct hn_nvs_ndis_conf conf;
368 int error;
369
370 memset(&conf, 0, sizeof(conf));
371 conf.type = NVS_TYPE_NDIS_CONF;
372 conf.mtu = mtu + RTE_ETHER_HDR_LEN;
373 conf.caps = NVS_NDIS_CONF_VLAN;
374
375 /* enable SRIOV */
376 if (hv->nvs_ver >= NVS_VERSION_5)
377 conf.caps |= NVS_NDIS_CONF_SRIOV;
378
379 /* NOTE: No response. */
380 error = hn_nvs_req_send(hv, &conf, sizeof(conf));
381 if (error) {
382 PMD_DRV_LOG(ERR,
383 "send nvs ndis conf failed: %d", error);
384 return error;
385 }
386
387 return 0;
388 }
389
390 static int
hn_nvs_init_ndis(struct hn_data * hv)391 hn_nvs_init_ndis(struct hn_data *hv)
392 {
393 struct hn_nvs_ndis_init ndis;
394 int error;
395
396 memset(&ndis, 0, sizeof(ndis));
397 ndis.type = NVS_TYPE_NDIS_INIT;
398 ndis.ndis_major = NDIS_VERSION_MAJOR(hv->ndis_ver);
399 ndis.ndis_minor = NDIS_VERSION_MINOR(hv->ndis_ver);
400
401 /* NOTE: No response. */
402 error = hn_nvs_req_send(hv, &ndis, sizeof(ndis));
403 if (error)
404 PMD_DRV_LOG(ERR,
405 "send nvs ndis init failed: %d", error);
406
407 return error;
408 }
409
410 static int
hn_nvs_init(struct hn_data * hv)411 hn_nvs_init(struct hn_data *hv)
412 {
413 unsigned int i;
414 int error;
415
416 /*
417 * Find the supported NVS version and set NDIS version accordingly.
418 */
419 for (i = 0; i < RTE_DIM(hn_nvs_version); ++i) {
420 error = hn_nvs_doinit(hv, hn_nvs_version[i]);
421 if (error) {
422 PMD_INIT_LOG(DEBUG, "version %#x error %d",
423 hn_nvs_version[i], error);
424 continue;
425 }
426
427 hv->nvs_ver = hn_nvs_version[i];
428
429 /* Set NDIS version according to NVS version. */
430 hv->ndis_ver = NDIS_VERSION_6_30;
431 if (hv->nvs_ver <= NVS_VERSION_4)
432 hv->ndis_ver = NDIS_VERSION_6_1;
433
434 PMD_INIT_LOG(DEBUG,
435 "NVS version %#x, NDIS version %u.%u",
436 hv->nvs_ver, NDIS_VERSION_MAJOR(hv->ndis_ver),
437 NDIS_VERSION_MINOR(hv->ndis_ver));
438 return 0;
439 }
440
441 PMD_DRV_LOG(ERR,
442 "no NVS compatible version available");
443 return -ENXIO;
444 }
445
446 int
hn_nvs_attach(struct hn_data * hv,unsigned int mtu)447 hn_nvs_attach(struct hn_data *hv, unsigned int mtu)
448 {
449 int error;
450
451 /*
452 * Initialize NVS.
453 */
454 error = hn_nvs_init(hv);
455 if (error)
456 return error;
457
458 /** Configure NDIS before initializing it. */
459 if (hv->nvs_ver >= NVS_VERSION_2) {
460 error = hn_nvs_conf_ndis(hv, mtu);
461 if (error)
462 return error;
463 }
464
465 /*
466 * Initialize NDIS.
467 */
468 error = hn_nvs_init_ndis(hv);
469 if (error)
470 return error;
471
472 /*
473 * Connect RXBUF.
474 */
475 error = hn_nvs_conn_rxbuf(hv);
476 if (error)
477 return error;
478
479 /*
480 * Connect chimney sending buffer.
481 */
482 error = hn_nvs_conn_chim(hv);
483 if (error) {
484 hn_nvs_disconn_rxbuf(hv);
485 return error;
486 }
487
488 return 0;
489 }
490
491 void
hn_nvs_detach(struct hn_data * hv __rte_unused)492 hn_nvs_detach(struct hn_data *hv __rte_unused)
493 {
494 PMD_INIT_FUNC_TRACE();
495
496 /* NOTE: there are no requests to stop the NVS. */
497 hn_nvs_disconn_rxbuf(hv);
498 hn_nvs_disconn_chim(hv);
499 }
500
501 /*
502 * Ack the consumed RXBUF associated w/ this channel packet,
503 * so that this RXBUF can be recycled by the hypervisor.
504 */
505 void
hn_nvs_ack_rxbuf(struct vmbus_channel * chan,uint64_t tid)506 hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid)
507 {
508 unsigned int retries = 0;
509 struct hn_nvs_rndis_ack ack = {
510 .type = NVS_TYPE_RNDIS_ACK,
511 .status = NVS_STATUS_OK,
512 };
513 int error;
514
515 PMD_RX_LOG(DEBUG, "ack RX id %" PRIu64, tid);
516
517 again:
518 error = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP,
519 &ack, sizeof(ack), tid,
520 VMBUS_CHANPKT_FLAG_NONE, NULL);
521
522 if (error == 0)
523 return;
524
525 if (error == -EAGAIN) {
526 /*
527 * NOTE:
528 * This should _not_ happen in real world, since the
529 * consumption of the TX bufring from the TX path is
530 * controlled.
531 */
532 PMD_RX_LOG(NOTICE, "RXBUF ack retry");
533 if (++retries < 10) {
534 rte_delay_ms(1);
535 goto again;
536 }
537 }
538 /* RXBUF leaks! */
539 PMD_DRV_LOG(ERR, "RXBUF ack failed");
540 }
541
542 int
hn_nvs_alloc_subchans(struct hn_data * hv,uint32_t * nsubch)543 hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch)
544 {
545 struct hn_nvs_subch_req req;
546 struct hn_nvs_subch_resp resp;
547 int error;
548
549 memset(&req, 0, sizeof(req));
550 req.type = NVS_TYPE_SUBCH_REQ;
551 req.op = NVS_SUBCH_OP_ALLOC;
552 req.nsubch = *nsubch;
553
554 error = hn_nvs_execute(hv, &req, sizeof(req),
555 &resp, sizeof(resp),
556 NVS_TYPE_SUBCH_RESP);
557 if (error)
558 return error;
559
560 if (resp.status != NVS_STATUS_OK) {
561 PMD_INIT_LOG(ERR,
562 "nvs subch alloc failed: %#x",
563 resp.status);
564 return -EIO;
565 }
566
567 if (resp.nsubch > *nsubch) {
568 PMD_INIT_LOG(NOTICE,
569 "%u subchans are allocated, requested %u",
570 resp.nsubch, *nsubch);
571 }
572 *nsubch = resp.nsubch;
573
574 return 0;
575 }
576
577 int
hn_nvs_set_datapath(struct hn_data * hv,uint32_t path)578 hn_nvs_set_datapath(struct hn_data *hv, uint32_t path)
579 {
580 struct hn_nvs_datapath dp;
581 int error;
582
583 PMD_DRV_LOG(DEBUG, "set datapath %s",
584 path ? "VF" : "Synthetic");
585
586 memset(&dp, 0, sizeof(dp));
587 dp.type = NVS_TYPE_SET_DATAPATH;
588 dp.active_path = path;
589
590 error = hn_nvs_req_send(hv, &dp, sizeof(dp));
591 if (error) {
592 PMD_DRV_LOG(ERR,
593 "send set datapath failed: %d",
594 error);
595 }
596
597 return error;
598 }
599