1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved.
3 */
4
5 #include <stdlib.h>
6 #include <fcntl.h>
7 #include <unistd.h>
8 #include <sys/types.h>
9 #include <sys/socket.h>
10 #include <sys/ioctl.h>
11 #include <errno.h>
12
13 #include <rte_version.h>
14 #include <rte_mbuf.h>
15 #include <rte_ether.h>
16 #include <rte_ethdev_driver.h>
17 #include <rte_ethdev_vdev.h>
18 #include <rte_malloc.h>
19 #include <rte_kvargs.h>
20 #include <rte_bus_vdev.h>
21 #include <rte_hash.h>
22 #include <rte_jhash.h>
23 #include <rte_string_fns.h>
24
25 #include "rte_eth_memif.h"
26 #include "memif_socket.h"
27
28 static void memif_intr_handler(void *arg);
29
30 static ssize_t
memif_msg_send(int fd,memif_msg_t * msg,int afd)31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
32 {
33 struct msghdr mh = { 0 };
34 struct iovec iov[1];
35 struct cmsghdr *cmsg;
36 char ctl[CMSG_SPACE(sizeof(int))];
37
38 iov[0].iov_base = msg;
39 iov[0].iov_len = sizeof(memif_msg_t);
40 mh.msg_iov = iov;
41 mh.msg_iovlen = 1;
42
43 if (afd > 0) {
44 memset(&ctl, 0, sizeof(ctl));
45 mh.msg_control = ctl;
46 mh.msg_controllen = sizeof(ctl);
47 cmsg = CMSG_FIRSTHDR(&mh);
48 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
49 cmsg->cmsg_level = SOL_SOCKET;
50 cmsg->cmsg_type = SCM_RIGHTS;
51 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
52 }
53
54 return sendmsg(fd, &mh, 0);
55 }
56
57 static int
memif_msg_send_from_queue(struct memif_control_channel * cc)58 memif_msg_send_from_queue(struct memif_control_channel *cc)
59 {
60 ssize_t size;
61 int ret = 0;
62 struct memif_msg_queue_elt *e;
63
64 e = TAILQ_FIRST(&cc->msg_queue);
65 if (e == NULL)
66 return 0;
67
68 size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
69 if (size != sizeof(memif_msg_t)) {
70 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
71 ret = -1;
72 } else {
73 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
74 }
75 TAILQ_REMOVE(&cc->msg_queue, e, next);
76 rte_free(e);
77
78 return ret;
79 }
80
81 static struct memif_msg_queue_elt *
memif_msg_enq(struct memif_control_channel * cc)82 memif_msg_enq(struct memif_control_channel *cc)
83 {
84 struct memif_msg_queue_elt *e;
85
86 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
87 if (e == NULL) {
88 MIF_LOG(ERR, "Failed to allocate control message.");
89 return NULL;
90 }
91
92 e->fd = -1;
93 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
94
95 return e;
96 }
97
98 void
memif_msg_enq_disconnect(struct memif_control_channel * cc,const char * reason,int err_code)99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
100 int err_code)
101 {
102 struct memif_msg_queue_elt *e;
103 struct pmd_internals *pmd;
104 memif_msg_disconnect_t *d;
105
106 if (cc == NULL) {
107 MIF_LOG(DEBUG, "Missing control channel.");
108 return;
109 }
110
111 e = memif_msg_enq(cc);
112 if (e == NULL) {
113 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
114 return;
115 }
116
117 d = &e->msg.disconnect;
118
119 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
120 d->code = err_code;
121
122 if (reason != NULL) {
123 strlcpy((char *)d->string, reason, sizeof(d->string));
124 if (cc->dev != NULL) {
125 pmd = cc->dev->data->dev_private;
126 strlcpy(pmd->local_disc_string, reason,
127 sizeof(pmd->local_disc_string));
128 }
129 }
130 }
131
132 static int
memif_msg_enq_hello(struct memif_control_channel * cc)133 memif_msg_enq_hello(struct memif_control_channel *cc)
134 {
135 struct memif_msg_queue_elt *e = memif_msg_enq(cc);
136 memif_msg_hello_t *h;
137
138 if (e == NULL)
139 return -1;
140
141 h = &e->msg.hello;
142
143 e->msg.type = MEMIF_MSG_TYPE_HELLO;
144 h->min_version = MEMIF_VERSION;
145 h->max_version = MEMIF_VERSION;
146 h->max_c2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
147 h->max_s2c_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
148 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
149 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
150
151 strlcpy((char *)h->name, rte_version(), sizeof(h->name));
152
153 return 0;
154 }
155
156 static int
memif_msg_receive_hello(struct rte_eth_dev * dev,memif_msg_t * msg)157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
158 {
159 struct pmd_internals *pmd = dev->data->dev_private;
160 memif_msg_hello_t *h = &msg->hello;
161
162 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
163 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
164 return -1;
165 }
166
167 /* Set parameters for active connection */
168 pmd->run.num_c2s_rings = RTE_MIN(h->max_c2s_ring + 1,
169 pmd->cfg.num_c2s_rings);
170 pmd->run.num_s2c_rings = RTE_MIN(h->max_s2c_ring + 1,
171 pmd->cfg.num_s2c_rings);
172 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
173 pmd->cfg.log2_ring_size);
174 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
175
176 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
177
178 MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name);
179
180 return 0;
181 }
182
183 static int
memif_msg_receive_init(struct memif_control_channel * cc,memif_msg_t * msg)184 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
185 {
186 memif_msg_init_t *i = &msg->init;
187 struct memif_socket_dev_list_elt *elt;
188 struct pmd_internals *pmd;
189 struct rte_eth_dev *dev;
190
191 if (i->version != MEMIF_VERSION) {
192 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
193 return -1;
194 }
195
196 if (cc->socket == NULL) {
197 memif_msg_enq_disconnect(cc, "Device error", 0);
198 return -1;
199 }
200
201 /* Find device with requested ID */
202 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
203 dev = elt->dev;
204 pmd = dev->data->dev_private;
205 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
206 (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_SERVER)) {
207 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
208 ETH_MEMIF_FLAG_CONNECTED)) {
209 memif_msg_enq_disconnect(cc,
210 "Already connected", 0);
211 return -1;
212 }
213
214 /* assign control channel to device */
215 cc->dev = dev;
216 pmd->cc = cc;
217
218 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
219 memif_msg_enq_disconnect(pmd->cc,
220 "Only ethernet mode supported",
221 0);
222 return -1;
223 }
224
225 strlcpy(pmd->remote_name, (char *)i->name,
226 sizeof(pmd->remote_name));
227
228 if (*pmd->secret != '\0') {
229 if (*i->secret == '\0') {
230 memif_msg_enq_disconnect(pmd->cc,
231 "Secret required", 0);
232 return -1;
233 }
234 if (strncmp(pmd->secret, (char *)i->secret,
235 ETH_MEMIF_SECRET_SIZE) != 0) {
236 memif_msg_enq_disconnect(pmd->cc,
237 "Incorrect secret", 0);
238 return -1;
239 }
240 }
241
242 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
243 return 0;
244 }
245 }
246
247 /* ID not found on this socket */
248 MIF_LOG(DEBUG, "ID %u not found.", i->id);
249 memif_msg_enq_disconnect(cc, "ID not found", 0);
250 return -1;
251 }
252
253 static int
memif_msg_receive_add_region(struct rte_eth_dev * dev,memif_msg_t * msg,int fd)254 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
255 int fd)
256 {
257 struct pmd_internals *pmd = dev->data->dev_private;
258 struct pmd_process_private *proc_private = dev->process_private;
259 memif_msg_add_region_t *ar = &msg->add_region;
260 struct memif_region *r;
261
262 if (fd < 0) {
263 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
264 return -1;
265 }
266
267 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
268 ar->index != proc_private->regions_num ||
269 proc_private->regions[ar->index] != NULL) {
270 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
271 return -1;
272 }
273
274 r = rte_zmalloc("region", sizeof(struct memif_region), 0);
275 if (r == NULL) {
276 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
277 return -ENOMEM;
278 }
279
280 r->fd = fd;
281 r->region_size = ar->size;
282 r->addr = NULL;
283
284 proc_private->regions[ar->index] = r;
285 proc_private->regions_num++;
286
287 return 0;
288 }
289
290 static int
memif_msg_receive_add_ring(struct rte_eth_dev * dev,memif_msg_t * msg,int fd)291 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
292 {
293 struct pmd_internals *pmd = dev->data->dev_private;
294 memif_msg_add_ring_t *ar = &msg->add_ring;
295 struct memif_queue *mq;
296
297 if (fd < 0) {
298 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
299 return -1;
300 }
301
302 /* check if we have enough queues */
303 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) {
304 if (ar->index >= pmd->cfg.num_c2s_rings) {
305 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
306 return -1;
307 }
308 pmd->run.num_c2s_rings++;
309 } else {
310 if (ar->index >= pmd->cfg.num_s2c_rings) {
311 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
312 return -1;
313 }
314 pmd->run.num_s2c_rings++;
315 }
316
317 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) ?
318 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
319
320 mq->intr_handle.fd = fd;
321 mq->log2_ring_size = ar->log2_ring_size;
322 mq->region = ar->region;
323 mq->ring_offset = ar->offset;
324
325 return 0;
326 }
327
328 static int
memif_msg_receive_connect(struct rte_eth_dev * dev,memif_msg_t * msg)329 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
330 {
331 struct pmd_internals *pmd = dev->data->dev_private;
332 memif_msg_connect_t *c = &msg->connect;
333 int ret;
334
335 ret = memif_connect(dev);
336 if (ret < 0)
337 return ret;
338
339 strlcpy(pmd->remote_if_name, (char *)c->if_name,
340 sizeof(pmd->remote_if_name));
341 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
342
343 return 0;
344 }
345
346 static int
memif_msg_receive_connected(struct rte_eth_dev * dev,memif_msg_t * msg)347 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
348 {
349 struct pmd_internals *pmd = dev->data->dev_private;
350 memif_msg_connected_t *c = &msg->connected;
351 int ret;
352
353 ret = memif_connect(dev);
354 if (ret < 0)
355 return ret;
356
357 strlcpy(pmd->remote_if_name, (char *)c->if_name,
358 sizeof(pmd->remote_if_name));
359 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
360
361 return 0;
362 }
363
364 static int
memif_msg_receive_disconnect(struct rte_eth_dev * dev,memif_msg_t * msg)365 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
366 {
367 struct pmd_internals *pmd = dev->data->dev_private;
368 memif_msg_disconnect_t *d = &msg->disconnect;
369
370 memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string));
371 strlcpy(pmd->remote_disc_string, (char *)d->string,
372 sizeof(pmd->remote_disc_string));
373
374 MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string);
375
376 memset(pmd->local_disc_string, 0, 96);
377 memif_disconnect(dev);
378 return 0;
379 }
380
381 static int
memif_msg_enq_ack(struct rte_eth_dev * dev)382 memif_msg_enq_ack(struct rte_eth_dev *dev)
383 {
384 struct pmd_internals *pmd = dev->data->dev_private;
385 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
386 if (e == NULL)
387 return -1;
388
389 e->msg.type = MEMIF_MSG_TYPE_ACK;
390
391 return 0;
392 }
393
394 static int
memif_msg_enq_init(struct rte_eth_dev * dev)395 memif_msg_enq_init(struct rte_eth_dev *dev)
396 {
397 struct pmd_internals *pmd = dev->data->dev_private;
398 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
399 memif_msg_init_t *i = &e->msg.init;
400
401 if (e == NULL)
402 return -1;
403
404 i = &e->msg.init;
405 e->msg.type = MEMIF_MSG_TYPE_INIT;
406 i->version = MEMIF_VERSION;
407 i->id = pmd->id;
408 i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
409
410 strlcpy((char *)i->name, rte_version(), sizeof(i->name));
411
412 if (*pmd->secret != '\0')
413 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
414
415 return 0;
416 }
417
418 static int
memif_msg_enq_add_region(struct rte_eth_dev * dev,uint8_t idx)419 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
420 {
421 struct pmd_internals *pmd = dev->data->dev_private;
422 struct pmd_process_private *proc_private = dev->process_private;
423 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
424 memif_msg_add_region_t *ar;
425 struct memif_region *mr = proc_private->regions[idx];
426
427 if (e == NULL)
428 return -1;
429
430 ar = &e->msg.add_region;
431 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
432 e->fd = mr->fd;
433 ar->index = idx;
434 ar->size = mr->region_size;
435
436 return 0;
437 }
438
439 static int
memif_msg_enq_add_ring(struct rte_eth_dev * dev,uint8_t idx,memif_ring_type_t type)440 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
441 memif_ring_type_t type)
442 {
443 struct pmd_internals *pmd = dev->data->dev_private;
444 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
445 struct memif_queue *mq;
446 memif_msg_add_ring_t *ar;
447
448 if (e == NULL)
449 return -1;
450
451 ar = &e->msg.add_ring;
452 mq = (type == MEMIF_RING_C2S) ? dev->data->tx_queues[idx] :
453 dev->data->rx_queues[idx];
454
455 e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
456 e->fd = mq->intr_handle.fd;
457 ar->index = idx;
458 ar->offset = mq->ring_offset;
459 ar->region = mq->region;
460 ar->log2_ring_size = mq->log2_ring_size;
461 ar->flags = (type == MEMIF_RING_C2S) ? MEMIF_MSG_ADD_RING_FLAG_C2S : 0;
462 ar->private_hdr_size = 0;
463
464 return 0;
465 }
466
467 static int
memif_msg_enq_connect(struct rte_eth_dev * dev)468 memif_msg_enq_connect(struct rte_eth_dev *dev)
469 {
470 struct pmd_internals *pmd = dev->data->dev_private;
471 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
472 memif_msg_connect_t *c;
473
474 if (e == NULL)
475 return -1;
476
477 c = &e->msg.connect;
478 e->msg.type = MEMIF_MSG_TYPE_CONNECT;
479 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
480
481 return 0;
482 }
483
484 static int
memif_msg_enq_connected(struct rte_eth_dev * dev)485 memif_msg_enq_connected(struct rte_eth_dev *dev)
486 {
487 struct pmd_internals *pmd = dev->data->dev_private;
488 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
489 memif_msg_connected_t *c;
490
491 if (e == NULL)
492 return -1;
493
494 c = &e->msg.connected;
495 e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
496 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
497
498 return 0;
499 }
500
501 static void
memif_intr_unregister_handler(struct rte_intr_handle * intr_handle,void * arg)502 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
503 {
504 struct memif_msg_queue_elt *elt;
505 struct memif_control_channel *cc = arg;
506
507 /* close control channel fd */
508 close(intr_handle->fd);
509 /* clear message queue */
510 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
511 TAILQ_REMOVE(&cc->msg_queue, elt, next);
512 rte_free(elt);
513 }
514 /* free control channel */
515 rte_free(cc);
516 }
517
518 void
memif_disconnect(struct rte_eth_dev * dev)519 memif_disconnect(struct rte_eth_dev *dev)
520 {
521 struct pmd_internals *pmd = dev->data->dev_private;
522 struct memif_msg_queue_elt *elt, *next;
523 struct memif_queue *mq;
524 struct rte_intr_handle *ih;
525 int i;
526 int ret;
527
528 dev->data->dev_link.link_status = ETH_LINK_DOWN;
529 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
530 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
531
532 rte_spinlock_lock(&pmd->cc_lock);
533 if (pmd->cc != NULL) {
534 /* Clear control message queue (except disconnect message if any). */
535 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
536 next = TAILQ_NEXT(elt, next);
537 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
538 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
539 rte_free(elt);
540 }
541 }
542 /* send disconnect message (if there is any in queue) */
543 memif_msg_send_from_queue(pmd->cc);
544
545 /* at this point, there should be no more messages in queue */
546 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
547 MIF_LOG(WARNING,
548 "Unexpected message(s) in message queue.");
549 }
550
551 ih = &pmd->cc->intr_handle;
552 if (ih->fd > 0) {
553 ret = rte_intr_callback_unregister(ih,
554 memif_intr_handler,
555 pmd->cc);
556 /*
557 * If callback is active (disconnecting based on
558 * received control message).
559 */
560 if (ret == -EAGAIN) {
561 ret = rte_intr_callback_unregister_pending(ih,
562 memif_intr_handler,
563 pmd->cc,
564 memif_intr_unregister_handler);
565 } else if (ret > 0) {
566 close(ih->fd);
567 rte_free(pmd->cc);
568 }
569 pmd->cc = NULL;
570 if (ret <= 0)
571 MIF_LOG(WARNING,
572 "Failed to unregister control channel callback.");
573 }
574 }
575 rte_spinlock_unlock(&pmd->cc_lock);
576
577 /* unconfig interrupts */
578 for (i = 0; i < pmd->cfg.num_c2s_rings; i++) {
579 if (pmd->role == MEMIF_ROLE_CLIENT) {
580 if (dev->data->tx_queues != NULL)
581 mq = dev->data->tx_queues[i];
582 else
583 continue;
584 } else {
585 if (dev->data->rx_queues != NULL)
586 mq = dev->data->rx_queues[i];
587 else
588 continue;
589 }
590 if (mq->intr_handle.fd > 0) {
591 close(mq->intr_handle.fd);
592 mq->intr_handle.fd = -1;
593 }
594 }
595 for (i = 0; i < pmd->cfg.num_s2c_rings; i++) {
596 if (pmd->role == MEMIF_ROLE_SERVER) {
597 if (dev->data->tx_queues != NULL)
598 mq = dev->data->tx_queues[i];
599 else
600 continue;
601 } else {
602 if (dev->data->rx_queues != NULL)
603 mq = dev->data->rx_queues[i];
604 else
605 continue;
606 }
607 if (mq->intr_handle.fd > 0) {
608 close(mq->intr_handle.fd);
609 mq->intr_handle.fd = -1;
610 }
611 }
612
613 memif_free_regions(dev);
614
615 /* reset connection configuration */
616 memset(&pmd->run, 0, sizeof(pmd->run));
617
618 MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id,
619 (pmd->role == MEMIF_ROLE_SERVER) ? "server" : "client");
620 }
621
622 static int
memif_msg_receive(struct memif_control_channel * cc)623 memif_msg_receive(struct memif_control_channel *cc)
624 {
625 char ctl[CMSG_SPACE(sizeof(int)) +
626 CMSG_SPACE(sizeof(struct ucred))] = { 0 };
627 struct msghdr mh = { 0 };
628 struct iovec iov[1];
629 memif_msg_t msg = { 0 };
630 ssize_t size;
631 int ret = 0;
632 struct ucred *cr __rte_unused;
633 cr = 0;
634 struct cmsghdr *cmsg;
635 int afd = -1;
636 int i;
637 struct pmd_internals *pmd;
638 struct pmd_process_private *proc_private;
639
640 iov[0].iov_base = (void *)&msg;
641 iov[0].iov_len = sizeof(memif_msg_t);
642 mh.msg_iov = iov;
643 mh.msg_iovlen = 1;
644 mh.msg_control = ctl;
645 mh.msg_controllen = sizeof(ctl);
646
647 size = recvmsg(cc->intr_handle.fd, &mh, 0);
648 if (size != sizeof(memif_msg_t)) {
649 MIF_LOG(DEBUG, "Invalid message size = %zd", size);
650 if (size > 0)
651 /* 0 means end-of-file, negative size means error,
652 * don't send further disconnect message in such cases.
653 */
654 memif_msg_enq_disconnect(cc, "Invalid message size", 0);
655 return -1;
656 }
657 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
658
659 cmsg = CMSG_FIRSTHDR(&mh);
660 while (cmsg) {
661 if (cmsg->cmsg_level == SOL_SOCKET) {
662 if (cmsg->cmsg_type == SCM_CREDENTIALS)
663 cr = (struct ucred *)CMSG_DATA(cmsg);
664 else if (cmsg->cmsg_type == SCM_RIGHTS)
665 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
666 }
667 cmsg = CMSG_NXTHDR(&mh, cmsg);
668 }
669
670 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
671 MIF_LOG(DEBUG, "Unexpected message.");
672 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
673 return -1;
674 }
675
676 /* get device from hash data */
677 switch (msg.type) {
678 case MEMIF_MSG_TYPE_ACK:
679 break;
680 case MEMIF_MSG_TYPE_HELLO:
681 ret = memif_msg_receive_hello(cc->dev, &msg);
682 if (ret < 0)
683 goto exit;
684 ret = memif_init_regions_and_queues(cc->dev);
685 if (ret < 0)
686 goto exit;
687 ret = memif_msg_enq_init(cc->dev);
688 if (ret < 0)
689 goto exit;
690 pmd = cc->dev->data->dev_private;
691 proc_private = cc->dev->process_private;
692 for (i = 0; i < proc_private->regions_num; i++) {
693 ret = memif_msg_enq_add_region(cc->dev, i);
694 if (ret < 0)
695 goto exit;
696 }
697 for (i = 0; i < pmd->run.num_c2s_rings; i++) {
698 ret = memif_msg_enq_add_ring(cc->dev, i,
699 MEMIF_RING_C2S);
700 if (ret < 0)
701 goto exit;
702 }
703 for (i = 0; i < pmd->run.num_s2c_rings; i++) {
704 ret = memif_msg_enq_add_ring(cc->dev, i,
705 MEMIF_RING_S2C);
706 if (ret < 0)
707 goto exit;
708 }
709 ret = memif_msg_enq_connect(cc->dev);
710 if (ret < 0)
711 goto exit;
712 break;
713 case MEMIF_MSG_TYPE_INIT:
714 /*
715 * This cc does not have an interface asociated with it.
716 * If suitable interface is found it will be assigned here.
717 */
718 ret = memif_msg_receive_init(cc, &msg);
719 if (ret < 0)
720 goto exit;
721 ret = memif_msg_enq_ack(cc->dev);
722 if (ret < 0)
723 goto exit;
724 break;
725 case MEMIF_MSG_TYPE_ADD_REGION:
726 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
727 if (ret < 0)
728 goto exit;
729 ret = memif_msg_enq_ack(cc->dev);
730 if (ret < 0)
731 goto exit;
732 break;
733 case MEMIF_MSG_TYPE_ADD_RING:
734 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
735 if (ret < 0)
736 goto exit;
737 ret = memif_msg_enq_ack(cc->dev);
738 if (ret < 0)
739 goto exit;
740 break;
741 case MEMIF_MSG_TYPE_CONNECT:
742 ret = memif_msg_receive_connect(cc->dev, &msg);
743 if (ret < 0)
744 goto exit;
745 ret = memif_msg_enq_connected(cc->dev);
746 if (ret < 0)
747 goto exit;
748 break;
749 case MEMIF_MSG_TYPE_CONNECTED:
750 ret = memif_msg_receive_connected(cc->dev, &msg);
751 break;
752 case MEMIF_MSG_TYPE_DISCONNECT:
753 ret = memif_msg_receive_disconnect(cc->dev, &msg);
754 if (ret < 0)
755 goto exit;
756 break;
757 default:
758 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
759 ret = -1;
760 goto exit;
761 }
762
763 exit:
764 return ret;
765 }
766
767 static void
memif_intr_handler(void * arg)768 memif_intr_handler(void *arg)
769 {
770 struct memif_control_channel *cc = arg;
771 int ret;
772
773 ret = memif_msg_receive(cc);
774 /* if driver failed to assign device */
775 if (cc->dev == NULL) {
776 memif_msg_send_from_queue(cc);
777 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
778 memif_intr_handler,
779 cc,
780 memif_intr_unregister_handler);
781 if (ret < 0)
782 MIF_LOG(WARNING,
783 "Failed to unregister control channel callback.");
784 return;
785 }
786 /* if memif_msg_receive failed */
787 if (ret < 0)
788 goto disconnect;
789
790 ret = memif_msg_send_from_queue(cc);
791 if (ret < 0)
792 goto disconnect;
793
794 return;
795
796 disconnect:
797 if (cc->dev == NULL) {
798 MIF_LOG(WARNING, "eth dev not allocated");
799 return;
800 }
801 memif_disconnect(cc->dev);
802 }
803
804 static void
memif_listener_handler(void * arg)805 memif_listener_handler(void *arg)
806 {
807 struct memif_socket *socket = arg;
808 int sockfd;
809 int addr_len;
810 struct sockaddr_un client;
811 struct memif_control_channel *cc;
812 int ret;
813
814 addr_len = sizeof(client);
815 sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
816 (socklen_t *)&addr_len);
817 if (sockfd < 0) {
818 MIF_LOG(ERR,
819 "Failed to accept connection request on socket fd %d",
820 socket->intr_handle.fd);
821 return;
822 }
823
824 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
825
826 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
827 if (cc == NULL) {
828 MIF_LOG(ERR, "Failed to allocate control channel.");
829 goto error;
830 }
831
832 cc->intr_handle.fd = sockfd;
833 cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
834 cc->socket = socket;
835 cc->dev = NULL;
836 TAILQ_INIT(&cc->msg_queue);
837
838 ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
839 if (ret < 0) {
840 MIF_LOG(ERR, "Failed to register control channel callback.");
841 goto error;
842 }
843
844 ret = memif_msg_enq_hello(cc);
845 if (ret < 0) {
846 MIF_LOG(ERR, "Failed to enqueue hello message.");
847 goto error;
848 }
849 ret = memif_msg_send_from_queue(cc);
850 if (ret < 0)
851 goto error;
852
853 return;
854
855 error:
856 if (sockfd >= 0) {
857 close(sockfd);
858 sockfd = -1;
859 }
860 if (cc != NULL)
861 rte_free(cc);
862 }
863
864 static struct memif_socket *
memif_socket_create(char * key,uint8_t listener,bool is_abstract)865 memif_socket_create(char *key, uint8_t listener, bool is_abstract)
866 {
867 struct memif_socket *sock;
868 struct sockaddr_un un = { 0 };
869 int sockfd;
870 int ret;
871 int on = 1;
872
873 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
874 if (sock == NULL) {
875 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
876 return NULL;
877 }
878
879 sock->listener = listener;
880 strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
881 TAILQ_INIT(&sock->dev_queue);
882
883 if (listener != 0) {
884 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
885 if (sockfd < 0)
886 goto error;
887
888 un.sun_family = AF_UNIX;
889 if (is_abstract) {
890 /* abstract address */
891 un.sun_path[0] = '\0';
892 strlcpy(un.sun_path + 1, sock->filename, MEMIF_SOCKET_UN_SIZE - 1);
893 } else {
894 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
895 }
896
897 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
898 sizeof(on));
899 if (ret < 0)
900 goto error;
901
902 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un));
903 if (ret < 0)
904 goto error;
905
906 ret = listen(sockfd, 1);
907 if (ret < 0)
908 goto error;
909
910 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename);
911
912 sock->intr_handle.fd = sockfd;
913 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
914 ret = rte_intr_callback_register(&sock->intr_handle,
915 memif_listener_handler, sock);
916 if (ret < 0) {
917 MIF_LOG(ERR, "Failed to register interrupt "
918 "callback for listener socket");
919 return NULL;
920 }
921 }
922
923 return sock;
924
925 error:
926 MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno));
927 if (sock != NULL)
928 rte_free(sock);
929 if (sockfd >= 0)
930 close(sockfd);
931 return NULL;
932 }
933
934 static struct rte_hash *
memif_create_socket_hash(void)935 memif_create_socket_hash(void)
936 {
937 struct rte_hash_parameters params = { 0 };
938
939 params.name = MEMIF_SOCKET_HASH_NAME;
940 params.entries = 256;
941 params.key_len = MEMIF_SOCKET_UN_SIZE;
942 params.hash_func = rte_jhash;
943 params.hash_func_init_val = 0;
944 return rte_hash_create(¶ms);
945 }
946
947 int
memif_socket_init(struct rte_eth_dev * dev,const char * socket_filename)948 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
949 {
950 struct pmd_internals *pmd = dev->data->dev_private;
951 struct memif_socket *socket = NULL;
952 struct memif_socket_dev_list_elt *elt;
953 struct pmd_internals *tmp_pmd;
954 struct rte_hash *hash;
955 int ret;
956 char key[MEMIF_SOCKET_UN_SIZE];
957
958 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
959 if (hash == NULL) {
960 hash = memif_create_socket_hash();
961 if (hash == NULL) {
962 MIF_LOG(ERR, "Failed to create memif socket hash.");
963 return -1;
964 }
965 }
966
967 memset(key, 0, MEMIF_SOCKET_UN_SIZE);
968 strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
969 ret = rte_hash_lookup_data(hash, key, (void **)&socket);
970 if (ret < 0) {
971 socket = memif_socket_create(key,
972 (pmd->role == MEMIF_ROLE_CLIENT) ? 0 : 1,
973 pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT);
974 if (socket == NULL)
975 return -1;
976 ret = rte_hash_add_key_data(hash, key, socket);
977 if (ret < 0) {
978 MIF_LOG(ERR, "Failed to add socket to socket hash.");
979 return ret;
980 }
981 }
982 pmd->socket_filename = socket->filename;
983
984 TAILQ_FOREACH(elt, &socket->dev_queue, next) {
985 tmp_pmd = elt->dev->data->dev_private;
986 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) {
987 MIF_LOG(ERR, "Two interfaces with the same id (%d) can "
988 "not have the same role.", pmd->id);
989 return -1;
990 }
991 }
992
993 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
994 if (elt == NULL) {
995 MIF_LOG(ERR, "Failed to add device to socket device list.");
996 return -1;
997 }
998 elt->dev = dev;
999 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1000
1001 return 0;
1002 }
1003
1004 void
memif_socket_remove_device(struct rte_eth_dev * dev)1005 memif_socket_remove_device(struct rte_eth_dev *dev)
1006 {
1007 struct pmd_internals *pmd = dev->data->dev_private;
1008 struct memif_socket *socket = NULL;
1009 struct memif_socket_dev_list_elt *elt, *next;
1010 struct rte_hash *hash;
1011 int ret;
1012
1013 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1014 if (hash == NULL)
1015 return;
1016
1017 if (pmd->socket_filename == NULL)
1018 return;
1019
1020 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1021 return;
1022
1023 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1024 next = TAILQ_NEXT(elt, next);
1025 if (elt->dev == dev) {
1026 TAILQ_REMOVE(&socket->dev_queue, elt, next);
1027 rte_free(elt);
1028 pmd->socket_filename = NULL;
1029 }
1030 }
1031
1032 /* remove socket, if this was the last device using it */
1033 if (TAILQ_EMPTY(&socket->dev_queue)) {
1034 rte_hash_del_key(hash, socket->filename);
1035 if (socket->listener && !(pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT)) {
1036 /* remove listener socket file,
1037 * so we can create new one later.
1038 */
1039 ret = remove(socket->filename);
1040 if (ret < 0)
1041 MIF_LOG(ERR, "Failed to remove socket file: %s",
1042 socket->filename);
1043 }
1044 rte_free(socket);
1045 }
1046 }
1047
1048 int
memif_connect_server(struct rte_eth_dev * dev)1049 memif_connect_server(struct rte_eth_dev *dev)
1050 {
1051 struct pmd_internals *pmd = dev->data->dev_private;
1052
1053 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1054 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1055 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1056 return 0;
1057 }
1058
1059 int
memif_connect_client(struct rte_eth_dev * dev)1060 memif_connect_client(struct rte_eth_dev *dev)
1061 {
1062 int sockfd;
1063 int ret;
1064 struct sockaddr_un sun = { 0 };
1065 struct pmd_internals *pmd = dev->data->dev_private;
1066
1067 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1068 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1069 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1070
1071 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1072 if (sockfd < 0) {
1073 MIF_LOG(ERR, "Failed to open socket.");
1074 return -1;
1075 }
1076
1077 sun.sun_family = AF_UNIX;
1078 if (pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT) {
1079 /* abstract address */
1080 sun.sun_path[0] = '\0';
1081 strlcpy(sun.sun_path + 1, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE - 1);
1082 } else {
1083 strlcpy(sun.sun_path, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE);
1084 }
1085
1086 ret = connect(sockfd, (struct sockaddr *)&sun,
1087 sizeof(struct sockaddr_un));
1088 if (ret < 0) {
1089 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename);
1090 goto error;
1091 }
1092
1093 MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename);
1094
1095 pmd->cc = rte_zmalloc("memif-cc",
1096 sizeof(struct memif_control_channel), 0);
1097 if (pmd->cc == NULL) {
1098 MIF_LOG(ERR, "Failed to allocate control channel.");
1099 goto error;
1100 }
1101
1102 pmd->cc->intr_handle.fd = sockfd;
1103 pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1104 pmd->cc->socket = NULL;
1105 pmd->cc->dev = dev;
1106 TAILQ_INIT(&pmd->cc->msg_queue);
1107
1108 ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1109 memif_intr_handler, pmd->cc);
1110 if (ret < 0) {
1111 MIF_LOG(ERR, "Failed to register interrupt callback for control fd");
1112 goto error;
1113 }
1114
1115 return 0;
1116
1117 error:
1118 if (sockfd >= 0) {
1119 close(sockfd);
1120 sockfd = -1;
1121 }
1122 if (pmd->cc != NULL) {
1123 rte_free(pmd->cc);
1124 pmd->cc = NULL;
1125 }
1126 return -1;
1127 }
1128