1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Ericsson AB 3 */ 4 5 #include <stdbool.h> 6 7 #include <rte_cycles.h> 8 #include <eventdev_pmd.h> 9 #include <eventdev_pmd_vdev.h> 10 #include <rte_random.h> 11 #include <rte_ring_elem.h> 12 13 #include "dsw_evdev.h" 14 15 #define EVENTDEV_NAME_DSW_PMD event_dsw 16 17 static int 18 dsw_port_setup(struct rte_eventdev *dev, uint8_t port_id, 19 const struct rte_event_port_conf *conf) 20 { 21 struct dsw_evdev *dsw = dsw_pmd_priv(dev); 22 struct dsw_port *port; 23 struct rte_event_ring *in_ring; 24 struct rte_ring *ctl_in_ring; 25 char ring_name[RTE_RING_NAMESIZE]; 26 27 port = &dsw->ports[port_id]; 28 29 *port = (struct dsw_port) { 30 .id = port_id, 31 .dsw = dsw, 32 .dequeue_depth = conf->dequeue_depth, 33 .enqueue_depth = conf->enqueue_depth, 34 .new_event_threshold = conf->new_event_threshold 35 }; 36 37 snprintf(ring_name, sizeof(ring_name), "dsw%d_p%u", dev->data->dev_id, 38 port_id); 39 40 in_ring = rte_event_ring_create(ring_name, DSW_IN_RING_SIZE, 41 dev->data->socket_id, 42 RING_F_SC_DEQ|RING_F_EXACT_SZ); 43 44 if (in_ring == NULL) 45 return -ENOMEM; 46 47 snprintf(ring_name, sizeof(ring_name), "dswctl%d_p%u", 48 dev->data->dev_id, port_id); 49 50 ctl_in_ring = rte_ring_create_elem(ring_name, 51 sizeof(struct dsw_ctl_msg), 52 DSW_CTL_IN_RING_SIZE, 53 dev->data->socket_id, 54 RING_F_SC_DEQ|RING_F_EXACT_SZ); 55 56 if (ctl_in_ring == NULL) { 57 rte_event_ring_free(in_ring); 58 return -ENOMEM; 59 } 60 61 port->in_ring = in_ring; 62 port->ctl_in_ring = ctl_in_ring; 63 64 port->load_update_interval = 65 (DSW_LOAD_UPDATE_INTERVAL * rte_get_timer_hz()) / US_PER_S; 66 67 port->migration_interval = 68 (DSW_MIGRATION_INTERVAL * rte_get_timer_hz()) / US_PER_S; 69 70 dev->data->ports[port_id] = port; 71 72 return 0; 73 } 74 75 static void 76 dsw_port_def_conf(struct rte_eventdev *dev __rte_unused, 77 uint8_t port_id __rte_unused, 78 struct rte_event_port_conf *port_conf) 79 { 80 *port_conf = (struct rte_event_port_conf) { 81 .new_event_threshold = 1024, 82 .dequeue_depth = DSW_MAX_PORT_DEQUEUE_DEPTH / 4, 83 .enqueue_depth = DSW_MAX_PORT_ENQUEUE_DEPTH / 4 84 }; 85 } 86 87 static void 88 dsw_port_release(void *p) 89 { 90 struct dsw_port *port = p; 91 92 rte_event_ring_free(port->in_ring); 93 rte_ring_free(port->ctl_in_ring); 94 } 95 96 static int 97 dsw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, 98 const struct rte_event_queue_conf *conf) 99 { 100 struct dsw_evdev *dsw = dsw_pmd_priv(dev); 101 struct dsw_queue *queue = &dsw->queues[queue_id]; 102 103 if (RTE_EVENT_QUEUE_CFG_ALL_TYPES & conf->event_queue_cfg) 104 return -ENOTSUP; 105 106 /* SINGLE_LINK is better off treated as TYPE_ATOMIC, since it 107 * avoid the "fake" TYPE_PARALLEL flow_id assignment. Since 108 * the queue will only have a single serving port, no 109 * migration will ever happen, so the extra TYPE_ATOMIC 110 * migration overhead is avoided. 111 */ 112 if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) 113 queue->schedule_type = RTE_SCHED_TYPE_ATOMIC; 114 else { 115 if (conf->schedule_type == RTE_SCHED_TYPE_ORDERED) 116 return -ENOTSUP; 117 /* atomic or parallel */ 118 queue->schedule_type = conf->schedule_type; 119 } 120 121 queue->num_serving_ports = 0; 122 123 return 0; 124 } 125 126 static void 127 dsw_queue_def_conf(struct rte_eventdev *dev __rte_unused, 128 uint8_t queue_id __rte_unused, 129 struct rte_event_queue_conf *queue_conf) 130 { 131 *queue_conf = (struct rte_event_queue_conf) { 132 .nb_atomic_flows = 4096, 133 .schedule_type = RTE_SCHED_TYPE_ATOMIC, 134 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL 135 }; 136 } 137 138 static void 139 dsw_queue_release(struct rte_eventdev *dev __rte_unused, 140 uint8_t queue_id __rte_unused) 141 { 142 } 143 144 static void 145 queue_add_port(struct dsw_queue *queue, uint16_t port_id) 146 { 147 queue->serving_ports[queue->num_serving_ports] = port_id; 148 queue->num_serving_ports++; 149 } 150 151 static bool 152 queue_remove_port(struct dsw_queue *queue, uint16_t port_id) 153 { 154 uint16_t i; 155 156 for (i = 0; i < queue->num_serving_ports; i++) 157 if (queue->serving_ports[i] == port_id) { 158 uint16_t last_idx = queue->num_serving_ports - 1; 159 if (i != last_idx) 160 queue->serving_ports[i] = 161 queue->serving_ports[last_idx]; 162 queue->num_serving_ports--; 163 return true; 164 } 165 return false; 166 } 167 168 static int 169 dsw_port_link_unlink(struct rte_eventdev *dev, void *port, 170 const uint8_t queues[], uint16_t num, bool link) 171 { 172 struct dsw_evdev *dsw = dsw_pmd_priv(dev); 173 struct dsw_port *p = port; 174 uint16_t i; 175 uint16_t count = 0; 176 177 for (i = 0; i < num; i++) { 178 uint8_t qid = queues[i]; 179 struct dsw_queue *q = &dsw->queues[qid]; 180 if (link) { 181 queue_add_port(q, p->id); 182 count++; 183 } else { 184 bool removed = queue_remove_port(q, p->id); 185 if (removed) 186 count++; 187 } 188 } 189 190 return count; 191 } 192 193 static int 194 dsw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[], 195 const uint8_t priorities[] __rte_unused, uint16_t num) 196 { 197 return dsw_port_link_unlink(dev, port, queues, num, true); 198 } 199 200 static int 201 dsw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[], 202 uint16_t num) 203 { 204 return dsw_port_link_unlink(dev, port, queues, num, false); 205 } 206 207 static void 208 dsw_info_get(struct rte_eventdev *dev __rte_unused, 209 struct rte_event_dev_info *info) 210 { 211 *info = (struct rte_event_dev_info) { 212 .driver_name = DSW_PMD_NAME, 213 .max_event_queues = DSW_MAX_QUEUES, 214 .max_event_queue_flows = DSW_MAX_FLOWS, 215 .max_event_queue_priority_levels = 1, 216 .max_event_priority_levels = 1, 217 .max_event_ports = DSW_MAX_PORTS, 218 .max_event_port_dequeue_depth = DSW_MAX_PORT_DEQUEUE_DEPTH, 219 .max_event_port_enqueue_depth = DSW_MAX_PORT_ENQUEUE_DEPTH, 220 .max_num_events = DSW_MAX_EVENTS, 221 .event_dev_cap = RTE_EVENT_DEV_CAP_BURST_MODE| 222 RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED| 223 RTE_EVENT_DEV_CAP_NONSEQ_MODE| 224 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT| 225 RTE_EVENT_DEV_CAP_CARRY_FLOW_ID 226 }; 227 } 228 229 static int 230 dsw_configure(const struct rte_eventdev *dev) 231 { 232 struct dsw_evdev *dsw = dsw_pmd_priv(dev); 233 const struct rte_event_dev_config *conf = &dev->data->dev_conf; 234 int32_t min_max_in_flight; 235 236 dsw->num_ports = conf->nb_event_ports; 237 dsw->num_queues = conf->nb_event_queues; 238 239 /* Avoid a situation where consumer ports are holding all the 240 * credits, without making use of them. 241 */ 242 min_max_in_flight = conf->nb_event_ports * DSW_PORT_MAX_CREDITS; 243 244 dsw->max_inflight = RTE_MAX(conf->nb_events_limit, min_max_in_flight); 245 246 return 0; 247 } 248 249 250 static void 251 initial_flow_to_port_assignment(struct dsw_evdev *dsw) 252 { 253 uint8_t queue_id; 254 for (queue_id = 0; queue_id < dsw->num_queues; queue_id++) { 255 struct dsw_queue *queue = &dsw->queues[queue_id]; 256 uint16_t flow_hash; 257 for (flow_hash = 0; flow_hash < DSW_MAX_FLOWS; flow_hash++) { 258 uint8_t port_idx = 259 rte_rand() % queue->num_serving_ports; 260 uint8_t port_id = 261 queue->serving_ports[port_idx]; 262 dsw->queues[queue_id].flow_to_port_map[flow_hash] = 263 port_id; 264 } 265 } 266 } 267 268 static int 269 dsw_start(struct rte_eventdev *dev) 270 { 271 struct dsw_evdev *dsw = dsw_pmd_priv(dev); 272 uint16_t i; 273 uint64_t now; 274 275 dsw->credits_on_loan = 0; 276 277 initial_flow_to_port_assignment(dsw); 278 279 now = rte_get_timer_cycles(); 280 for (i = 0; i < dsw->num_ports; i++) { 281 dsw->ports[i].measurement_start = now; 282 dsw->ports[i].busy_start = now; 283 } 284 285 return 0; 286 } 287 288 static void 289 dsw_port_drain_buf(uint8_t dev_id, struct rte_event *buf, uint16_t buf_len, 290 eventdev_stop_flush_t flush, void *flush_arg) 291 { 292 uint16_t i; 293 294 for (i = 0; i < buf_len; i++) 295 flush(dev_id, buf[i], flush_arg); 296 } 297 298 static void 299 dsw_port_drain_paused(uint8_t dev_id, struct dsw_port *port, 300 eventdev_stop_flush_t flush, void *flush_arg) 301 { 302 dsw_port_drain_buf(dev_id, port->paused_events, port->paused_events_len, 303 flush, flush_arg); 304 } 305 306 static void 307 dsw_port_drain_out(uint8_t dev_id, struct dsw_evdev *dsw, struct dsw_port *port, 308 eventdev_stop_flush_t flush, void *flush_arg) 309 { 310 uint16_t dport_id; 311 312 for (dport_id = 0; dport_id < dsw->num_ports; dport_id++) 313 if (dport_id != port->id) 314 dsw_port_drain_buf(dev_id, port->out_buffer[dport_id], 315 port->out_buffer_len[dport_id], 316 flush, flush_arg); 317 } 318 319 static void 320 dsw_port_drain_in_ring(uint8_t dev_id, struct dsw_port *port, 321 eventdev_stop_flush_t flush, void *flush_arg) 322 { 323 struct rte_event ev; 324 325 while (rte_event_ring_dequeue_burst(port->in_ring, &ev, 1, NULL)) 326 flush(dev_id, ev, flush_arg); 327 } 328 329 static void 330 dsw_drain(uint8_t dev_id, struct dsw_evdev *dsw, 331 eventdev_stop_flush_t flush, void *flush_arg) 332 { 333 uint16_t port_id; 334 335 if (flush == NULL) 336 return; 337 338 for (port_id = 0; port_id < dsw->num_ports; port_id++) { 339 struct dsw_port *port = &dsw->ports[port_id]; 340 341 dsw_port_drain_out(dev_id, dsw, port, flush, flush_arg); 342 dsw_port_drain_paused(dev_id, port, flush, flush_arg); 343 dsw_port_drain_in_ring(dev_id, port, flush, flush_arg); 344 } 345 } 346 347 static void 348 dsw_stop(struct rte_eventdev *dev) 349 { 350 struct dsw_evdev *dsw = dsw_pmd_priv(dev); 351 uint8_t dev_id; 352 eventdev_stop_flush_t flush; 353 void *flush_arg; 354 355 dev_id = dev->data->dev_id; 356 flush = dev->dev_ops->dev_stop_flush; 357 flush_arg = dev->data->dev_stop_flush_arg; 358 359 dsw_drain(dev_id, dsw, flush, flush_arg); 360 } 361 362 static int 363 dsw_close(struct rte_eventdev *dev) 364 { 365 struct dsw_evdev *dsw = dsw_pmd_priv(dev); 366 367 dsw->num_ports = 0; 368 dsw->num_queues = 0; 369 370 return 0; 371 } 372 373 static int 374 dsw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev __rte_unused, 375 const struct rte_eth_dev *eth_dev __rte_unused, 376 uint32_t *caps) 377 { 378 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; 379 return 0; 380 } 381 382 static int 383 dsw_timer_adapter_caps_get(const struct rte_eventdev *dev __rte_unused, 384 uint64_t flags __rte_unused, uint32_t *caps, 385 const struct event_timer_adapter_ops **ops) 386 { 387 *caps = 0; 388 *ops = NULL; 389 return 0; 390 } 391 392 static int 393 dsw_crypto_adapter_caps_get(const struct rte_eventdev *dev __rte_unused, 394 const struct rte_cryptodev *cdev __rte_unused, 395 uint32_t *caps) 396 { 397 *caps = RTE_EVENT_CRYPTO_ADAPTER_SW_CAP; 398 return 0; 399 } 400 401 static struct eventdev_ops dsw_evdev_ops = { 402 .port_setup = dsw_port_setup, 403 .port_def_conf = dsw_port_def_conf, 404 .port_release = dsw_port_release, 405 .queue_setup = dsw_queue_setup, 406 .queue_def_conf = dsw_queue_def_conf, 407 .queue_release = dsw_queue_release, 408 .port_link = dsw_port_link, 409 .port_unlink = dsw_port_unlink, 410 .dev_infos_get = dsw_info_get, 411 .dev_configure = dsw_configure, 412 .dev_start = dsw_start, 413 .dev_stop = dsw_stop, 414 .dev_close = dsw_close, 415 .eth_rx_adapter_caps_get = dsw_eth_rx_adapter_caps_get, 416 .timer_adapter_caps_get = dsw_timer_adapter_caps_get, 417 .crypto_adapter_caps_get = dsw_crypto_adapter_caps_get, 418 .xstats_get = dsw_xstats_get, 419 .xstats_get_names = dsw_xstats_get_names, 420 .xstats_get_by_name = dsw_xstats_get_by_name 421 }; 422 423 static int 424 dsw_probe(struct rte_vdev_device *vdev) 425 { 426 const char *name; 427 struct rte_eventdev *dev; 428 struct dsw_evdev *dsw; 429 430 name = rte_vdev_device_name(vdev); 431 432 dev = rte_event_pmd_vdev_init(name, sizeof(struct dsw_evdev), 433 rte_socket_id()); 434 if (dev == NULL) 435 return -EFAULT; 436 437 dev->dev_ops = &dsw_evdev_ops; 438 dev->enqueue = dsw_event_enqueue; 439 dev->enqueue_burst = dsw_event_enqueue_burst; 440 dev->enqueue_new_burst = dsw_event_enqueue_new_burst; 441 dev->enqueue_forward_burst = dsw_event_enqueue_forward_burst; 442 dev->dequeue = dsw_event_dequeue; 443 dev->dequeue_burst = dsw_event_dequeue_burst; 444 dev->maintain = dsw_event_maintain; 445 446 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 447 return 0; 448 449 dsw = dev->data->dev_private; 450 dsw->data = dev->data; 451 452 event_dev_probing_finish(dev); 453 return 0; 454 } 455 456 static int 457 dsw_remove(struct rte_vdev_device *vdev) 458 { 459 const char *name; 460 461 name = rte_vdev_device_name(vdev); 462 if (name == NULL) 463 return -EINVAL; 464 465 return rte_event_pmd_vdev_uninit(name); 466 } 467 468 static struct rte_vdev_driver evdev_dsw_pmd_drv = { 469 .probe = dsw_probe, 470 .remove = dsw_remove 471 }; 472 473 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DSW_PMD, evdev_dsw_pmd_drv); 474