1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev_driver.h> 36 #include <rte_ethdev_pci.h> 37 #include <rte_tcp.h> 38 #include <rte_atomic.h> 39 #include <rte_dev.h> 40 #include <rte_errno.h> 41 #include <rte_version.h> 42 #include <rte_net.h> 43 44 #include "ena_ethdev.h" 45 #include "ena_logs.h" 46 #include "ena_platform.h" 47 #include "ena_com.h" 48 #include "ena_eth_com.h" 49 50 #include <ena_common_defs.h> 51 #include <ena_regs_defs.h> 52 #include <ena_admin_defs.h> 53 #include <ena_eth_io_defs.h> 54 55 #define DRV_MODULE_VER_MAJOR 1 56 #define DRV_MODULE_VER_MINOR 1 57 #define DRV_MODULE_VER_SUBMINOR 1 58 59 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 60 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 61 /*reverse version of ENA_IO_RXQ_IDX*/ 62 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 63 64 /* While processing submitted and completed descriptors (rx and tx path 65 * respectively) in a loop it is desired to: 66 * - perform batch submissions while populating sumbissmion queue 67 * - avoid blocking transmission of other packets during cleanup phase 68 * Hence the utilization ratio of 1/8 of a queue size. 69 */ 70 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 71 72 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 73 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 74 75 #define GET_L4_HDR_LEN(mbuf) \ 76 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 77 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 78 79 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 80 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 81 #define ENA_HASH_KEY_SIZE 40 82 #define ENA_ETH_SS_STATS 0xFF 83 #define ETH_GSTRING_LEN 32 84 85 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 86 87 #define ENA_MAX_RING_DESC ENA_DEFAULT_RING_SIZE 88 #define ENA_MIN_RING_DESC 128 89 90 enum ethtool_stringset { 91 ETH_SS_TEST = 0, 92 ETH_SS_STATS, 93 }; 94 95 struct ena_stats { 96 char name[ETH_GSTRING_LEN]; 97 int stat_offset; 98 }; 99 100 #define ENA_STAT_ENA_COM_ENTRY(stat) { \ 101 .name = #stat, \ 102 .stat_offset = offsetof(struct ena_com_stats_admin, stat) \ 103 } 104 105 #define ENA_STAT_ENTRY(stat, stat_type) { \ 106 .name = #stat, \ 107 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 108 } 109 110 #define ENA_STAT_RX_ENTRY(stat) \ 111 ENA_STAT_ENTRY(stat, rx) 112 113 #define ENA_STAT_TX_ENTRY(stat) \ 114 ENA_STAT_ENTRY(stat, tx) 115 116 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 117 ENA_STAT_ENTRY(stat, dev) 118 119 /* 120 * Each rte_memzone should have unique name. 121 * To satisfy it, count number of allocation and add it to name. 122 */ 123 uint32_t ena_alloc_cnt; 124 125 static const struct ena_stats ena_stats_global_strings[] = { 126 ENA_STAT_GLOBAL_ENTRY(tx_timeout), 127 ENA_STAT_GLOBAL_ENTRY(io_suspend), 128 ENA_STAT_GLOBAL_ENTRY(io_resume), 129 ENA_STAT_GLOBAL_ENTRY(wd_expired), 130 ENA_STAT_GLOBAL_ENTRY(interface_up), 131 ENA_STAT_GLOBAL_ENTRY(interface_down), 132 ENA_STAT_GLOBAL_ENTRY(admin_q_pause), 133 }; 134 135 static const struct ena_stats ena_stats_tx_strings[] = { 136 ENA_STAT_TX_ENTRY(cnt), 137 ENA_STAT_TX_ENTRY(bytes), 138 ENA_STAT_TX_ENTRY(queue_stop), 139 ENA_STAT_TX_ENTRY(queue_wakeup), 140 ENA_STAT_TX_ENTRY(dma_mapping_err), 141 ENA_STAT_TX_ENTRY(linearize), 142 ENA_STAT_TX_ENTRY(linearize_failed), 143 ENA_STAT_TX_ENTRY(tx_poll), 144 ENA_STAT_TX_ENTRY(doorbells), 145 ENA_STAT_TX_ENTRY(prepare_ctx_err), 146 ENA_STAT_TX_ENTRY(missing_tx_comp), 147 ENA_STAT_TX_ENTRY(bad_req_id), 148 }; 149 150 static const struct ena_stats ena_stats_rx_strings[] = { 151 ENA_STAT_RX_ENTRY(cnt), 152 ENA_STAT_RX_ENTRY(bytes), 153 ENA_STAT_RX_ENTRY(refil_partial), 154 ENA_STAT_RX_ENTRY(bad_csum), 155 ENA_STAT_RX_ENTRY(page_alloc_fail), 156 ENA_STAT_RX_ENTRY(skb_alloc_fail), 157 ENA_STAT_RX_ENTRY(dma_mapping_err), 158 ENA_STAT_RX_ENTRY(bad_desc_num), 159 ENA_STAT_RX_ENTRY(small_copy_len_pkt), 160 }; 161 162 static const struct ena_stats ena_stats_ena_com_strings[] = { 163 ENA_STAT_ENA_COM_ENTRY(aborted_cmd), 164 ENA_STAT_ENA_COM_ENTRY(submitted_cmd), 165 ENA_STAT_ENA_COM_ENTRY(completed_cmd), 166 ENA_STAT_ENA_COM_ENTRY(out_of_space), 167 ENA_STAT_ENA_COM_ENTRY(no_completion), 168 }; 169 170 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 171 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 172 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 173 #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) 174 175 #define QUEUE_OFFLOADS (DEV_TX_OFFLOAD_TCP_CKSUM |\ 176 DEV_TX_OFFLOAD_UDP_CKSUM |\ 177 DEV_TX_OFFLOAD_IPV4_CKSUM |\ 178 DEV_TX_OFFLOAD_TCP_TSO) 179 #define MBUF_OFFLOADS (PKT_TX_L4_MASK |\ 180 PKT_TX_IP_CKSUM |\ 181 PKT_TX_TCP_SEG) 182 183 /** Vendor ID used by Amazon devices */ 184 #define PCI_VENDOR_ID_AMAZON 0x1D0F 185 /** Amazon devices */ 186 #define PCI_DEVICE_ID_ENA_VF 0xEC20 187 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 188 189 #define ENA_TX_OFFLOAD_MASK (\ 190 PKT_TX_L4_MASK | \ 191 PKT_TX_IPV6 | \ 192 PKT_TX_IPV4 | \ 193 PKT_TX_IP_CKSUM | \ 194 PKT_TX_TCP_SEG) 195 196 #define ENA_TX_OFFLOAD_NOTSUP_MASK \ 197 (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) 198 199 int ena_logtype_init; 200 int ena_logtype_driver; 201 202 static const struct rte_pci_id pci_id_ena_map[] = { 203 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 204 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, 205 { .device_id = 0 }, 206 }; 207 208 static struct ena_aenq_handlers aenq_handlers; 209 210 static int ena_device_init(struct ena_com_dev *ena_dev, 211 struct ena_com_dev_get_features_ctx *get_feat_ctx, 212 bool *wd_state); 213 static int ena_dev_configure(struct rte_eth_dev *dev); 214 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 215 uint16_t nb_pkts); 216 static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 217 uint16_t nb_pkts); 218 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 219 uint16_t nb_desc, unsigned int socket_id, 220 const struct rte_eth_txconf *tx_conf); 221 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 222 uint16_t nb_desc, unsigned int socket_id, 223 const struct rte_eth_rxconf *rx_conf, 224 struct rte_mempool *mp); 225 static uint16_t eth_ena_recv_pkts(void *rx_queue, 226 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 227 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 228 static void ena_init_rings(struct ena_adapter *adapter); 229 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 230 static int ena_start(struct rte_eth_dev *dev); 231 static void ena_stop(struct rte_eth_dev *dev); 232 static void ena_close(struct rte_eth_dev *dev); 233 static int ena_dev_reset(struct rte_eth_dev *dev); 234 static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 235 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 236 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 237 static void ena_rx_queue_release(void *queue); 238 static void ena_tx_queue_release(void *queue); 239 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 240 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 241 static int ena_link_update(struct rte_eth_dev *dev, 242 int wait_to_complete); 243 static int ena_create_io_queue(struct ena_ring *ring); 244 static void ena_queue_stop(struct ena_ring *ring); 245 static void ena_queue_stop_all(struct rte_eth_dev *dev, 246 enum ena_ring_type ring_type); 247 static int ena_queue_start(struct ena_ring *ring); 248 static int ena_queue_start_all(struct rte_eth_dev *dev, 249 enum ena_ring_type ring_type); 250 static void ena_stats_restart(struct rte_eth_dev *dev); 251 static void ena_infos_get(struct rte_eth_dev *dev, 252 struct rte_eth_dev_info *dev_info); 253 static int ena_rss_reta_update(struct rte_eth_dev *dev, 254 struct rte_eth_rss_reta_entry64 *reta_conf, 255 uint16_t reta_size); 256 static int ena_rss_reta_query(struct rte_eth_dev *dev, 257 struct rte_eth_rss_reta_entry64 *reta_conf, 258 uint16_t reta_size); 259 static int ena_get_sset_count(struct rte_eth_dev *dev, int sset); 260 static void ena_interrupt_handler_rte(void *cb_arg); 261 static void ena_timer_wd_callback(struct rte_timer *timer, void *arg); 262 static void ena_destroy_device(struct rte_eth_dev *eth_dev); 263 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev); 264 265 static const struct eth_dev_ops ena_dev_ops = { 266 .dev_configure = ena_dev_configure, 267 .dev_infos_get = ena_infos_get, 268 .rx_queue_setup = ena_rx_queue_setup, 269 .tx_queue_setup = ena_tx_queue_setup, 270 .dev_start = ena_start, 271 .dev_stop = ena_stop, 272 .link_update = ena_link_update, 273 .stats_get = ena_stats_get, 274 .mtu_set = ena_mtu_set, 275 .rx_queue_release = ena_rx_queue_release, 276 .tx_queue_release = ena_tx_queue_release, 277 .dev_close = ena_close, 278 .dev_reset = ena_dev_reset, 279 .reta_update = ena_rss_reta_update, 280 .reta_query = ena_rss_reta_query, 281 }; 282 283 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 284 struct ena_com_rx_ctx *ena_rx_ctx) 285 { 286 uint64_t ol_flags = 0; 287 uint32_t packet_type = 0; 288 289 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 290 packet_type |= RTE_PTYPE_L4_TCP; 291 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 292 packet_type |= RTE_PTYPE_L4_UDP; 293 294 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 295 packet_type |= RTE_PTYPE_L3_IPV4; 296 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 297 packet_type |= RTE_PTYPE_L3_IPV6; 298 299 if (unlikely(ena_rx_ctx->l4_csum_err)) 300 ol_flags |= PKT_RX_L4_CKSUM_BAD; 301 if (unlikely(ena_rx_ctx->l3_csum_err)) 302 ol_flags |= PKT_RX_IP_CKSUM_BAD; 303 304 mbuf->ol_flags = ol_flags; 305 mbuf->packet_type = packet_type; 306 } 307 308 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 309 struct ena_com_tx_ctx *ena_tx_ctx, 310 uint64_t queue_offloads) 311 { 312 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 313 314 if ((mbuf->ol_flags & MBUF_OFFLOADS) && 315 (queue_offloads & QUEUE_OFFLOADS)) { 316 /* check if TSO is required */ 317 if ((mbuf->ol_flags & PKT_TX_TCP_SEG) && 318 (queue_offloads & DEV_TX_OFFLOAD_TCP_TSO)) { 319 ena_tx_ctx->tso_enable = true; 320 321 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 322 } 323 324 /* check if L3 checksum is needed */ 325 if ((mbuf->ol_flags & PKT_TX_IP_CKSUM) && 326 (queue_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)) 327 ena_tx_ctx->l3_csum_enable = true; 328 329 if (mbuf->ol_flags & PKT_TX_IPV6) { 330 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 331 } else { 332 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 333 334 /* set don't fragment (DF) flag */ 335 if (mbuf->packet_type & 336 (RTE_PTYPE_L4_NONFRAG 337 | RTE_PTYPE_INNER_L4_NONFRAG)) 338 ena_tx_ctx->df = true; 339 } 340 341 /* check if L4 checksum is needed */ 342 if (((mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) && 343 (queue_offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) { 344 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 345 ena_tx_ctx->l4_csum_enable = true; 346 } else if (((mbuf->ol_flags & PKT_TX_L4_MASK) == 347 PKT_TX_UDP_CKSUM) && 348 (queue_offloads & DEV_TX_OFFLOAD_UDP_CKSUM)) { 349 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 350 ena_tx_ctx->l4_csum_enable = true; 351 } else { 352 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 353 ena_tx_ctx->l4_csum_enable = false; 354 } 355 356 ena_meta->mss = mbuf->tso_segsz; 357 ena_meta->l3_hdr_len = mbuf->l3_len; 358 ena_meta->l3_hdr_offset = mbuf->l2_len; 359 360 ena_tx_ctx->meta_valid = true; 361 } else { 362 ena_tx_ctx->meta_valid = false; 363 } 364 } 365 366 static inline int validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id) 367 { 368 if (likely(req_id < rx_ring->ring_size)) 369 return 0; 370 371 RTE_LOG(ERR, PMD, "Invalid rx req_id: %hu\n", req_id); 372 373 rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; 374 rx_ring->adapter->trigger_reset = true; 375 376 return -EFAULT; 377 } 378 379 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id) 380 { 381 struct ena_tx_buffer *tx_info = NULL; 382 383 if (likely(req_id < tx_ring->ring_size)) { 384 tx_info = &tx_ring->tx_buffer_info[req_id]; 385 if (likely(tx_info->mbuf)) 386 return 0; 387 } 388 389 if (tx_info) 390 RTE_LOG(ERR, PMD, "tx_info doesn't have valid mbuf\n"); 391 else 392 RTE_LOG(ERR, PMD, "Invalid req_id: %hu\n", req_id); 393 394 /* Trigger device reset */ 395 tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID; 396 tx_ring->adapter->trigger_reset = true; 397 return -EFAULT; 398 } 399 400 static void ena_config_host_info(struct ena_com_dev *ena_dev) 401 { 402 struct ena_admin_host_info *host_info; 403 int rc; 404 405 /* Allocate only the host info */ 406 rc = ena_com_allocate_host_info(ena_dev); 407 if (rc) { 408 RTE_LOG(ERR, PMD, "Cannot allocate host info\n"); 409 return; 410 } 411 412 host_info = ena_dev->host_attr.host_info; 413 414 host_info->os_type = ENA_ADMIN_OS_DPDK; 415 host_info->kernel_ver = RTE_VERSION; 416 snprintf((char *)host_info->kernel_ver_str, 417 sizeof(host_info->kernel_ver_str), 418 "%s", rte_version()); 419 host_info->os_dist = RTE_VERSION; 420 snprintf((char *)host_info->os_dist_str, 421 sizeof(host_info->os_dist_str), 422 "%s", rte_version()); 423 host_info->driver_version = 424 (DRV_MODULE_VER_MAJOR) | 425 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 426 (DRV_MODULE_VER_SUBMINOR << 427 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 428 429 rc = ena_com_set_host_attributes(ena_dev); 430 if (rc) { 431 if (rc == -ENA_COM_UNSUPPORTED) 432 RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); 433 else 434 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 435 436 goto err; 437 } 438 439 return; 440 441 err: 442 ena_com_delete_host_info(ena_dev); 443 } 444 445 static int 446 ena_get_sset_count(struct rte_eth_dev *dev, int sset) 447 { 448 if (sset != ETH_SS_STATS) 449 return -EOPNOTSUPP; 450 451 /* Workaround for clang: 452 * touch internal structures to prevent 453 * compiler error 454 */ 455 ENA_TOUCH(ena_stats_global_strings); 456 ENA_TOUCH(ena_stats_tx_strings); 457 ENA_TOUCH(ena_stats_rx_strings); 458 ENA_TOUCH(ena_stats_ena_com_strings); 459 460 return dev->data->nb_tx_queues * 461 (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) + 462 ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; 463 } 464 465 static void ena_config_debug_area(struct ena_adapter *adapter) 466 { 467 u32 debug_area_size; 468 int rc, ss_count; 469 470 ss_count = ena_get_sset_count(adapter->rte_dev, ETH_SS_STATS); 471 if (ss_count <= 0) { 472 RTE_LOG(ERR, PMD, "SS count is negative\n"); 473 return; 474 } 475 476 /* allocate 32 bytes for each string and 64bit for the value */ 477 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 478 479 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 480 if (rc) { 481 RTE_LOG(ERR, PMD, "Cannot allocate debug area\n"); 482 return; 483 } 484 485 rc = ena_com_set_host_attributes(&adapter->ena_dev); 486 if (rc) { 487 if (rc == -ENA_COM_UNSUPPORTED) 488 RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); 489 else 490 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 491 492 goto err; 493 } 494 495 return; 496 err: 497 ena_com_delete_debug_area(&adapter->ena_dev); 498 } 499 500 static void ena_close(struct rte_eth_dev *dev) 501 { 502 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 503 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 504 struct ena_adapter *adapter = dev->data->dev_private; 505 506 if (adapter->state == ENA_ADAPTER_STATE_RUNNING) 507 ena_stop(dev); 508 adapter->state = ENA_ADAPTER_STATE_CLOSED; 509 510 ena_rx_queue_release_all(dev); 511 ena_tx_queue_release_all(dev); 512 513 rte_free(adapter->drv_stats); 514 adapter->drv_stats = NULL; 515 516 rte_intr_disable(intr_handle); 517 rte_intr_callback_unregister(intr_handle, 518 ena_interrupt_handler_rte, 519 adapter); 520 521 /* 522 * MAC is not allocated dynamically. Setting NULL should prevent from 523 * release of the resource in the rte_eth_dev_release_port(). 524 */ 525 dev->data->mac_addrs = NULL; 526 } 527 528 static int 529 ena_dev_reset(struct rte_eth_dev *dev) 530 { 531 int rc = 0; 532 533 ena_destroy_device(dev); 534 rc = eth_ena_dev_init(dev); 535 if (rc) 536 PMD_INIT_LOG(CRIT, "Cannot initialize device\n"); 537 538 return rc; 539 } 540 541 static int ena_rss_reta_update(struct rte_eth_dev *dev, 542 struct rte_eth_rss_reta_entry64 *reta_conf, 543 uint16_t reta_size) 544 { 545 struct ena_adapter *adapter = dev->data->dev_private; 546 struct ena_com_dev *ena_dev = &adapter->ena_dev; 547 int rc, i; 548 u16 entry_value; 549 int conf_idx; 550 int idx; 551 552 if ((reta_size == 0) || (reta_conf == NULL)) 553 return -EINVAL; 554 555 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 556 RTE_LOG(WARNING, PMD, 557 "indirection table %d is bigger than supported (%d)\n", 558 reta_size, ENA_RX_RSS_TABLE_SIZE); 559 return -EINVAL; 560 } 561 562 for (i = 0 ; i < reta_size ; i++) { 563 /* each reta_conf is for 64 entries. 564 * to support 128 we use 2 conf of 64 565 */ 566 conf_idx = i / RTE_RETA_GROUP_SIZE; 567 idx = i % RTE_RETA_GROUP_SIZE; 568 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 569 entry_value = 570 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 571 572 rc = ena_com_indirect_table_fill_entry(ena_dev, 573 i, 574 entry_value); 575 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 576 RTE_LOG(ERR, PMD, 577 "Cannot fill indirect table\n"); 578 return rc; 579 } 580 } 581 } 582 583 rc = ena_com_indirect_table_set(ena_dev); 584 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 585 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 586 return rc; 587 } 588 589 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 590 __func__, reta_size, adapter->rte_dev->data->port_id); 591 592 return 0; 593 } 594 595 /* Query redirection table. */ 596 static int ena_rss_reta_query(struct rte_eth_dev *dev, 597 struct rte_eth_rss_reta_entry64 *reta_conf, 598 uint16_t reta_size) 599 { 600 struct ena_adapter *adapter = dev->data->dev_private; 601 struct ena_com_dev *ena_dev = &adapter->ena_dev; 602 int rc; 603 int i; 604 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 605 int reta_conf_idx; 606 int reta_idx; 607 608 if (reta_size == 0 || reta_conf == NULL || 609 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 610 return -EINVAL; 611 612 rc = ena_com_indirect_table_get(ena_dev, indirect_table); 613 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 614 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 615 return -ENOTSUP; 616 } 617 618 for (i = 0 ; i < reta_size ; i++) { 619 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 620 reta_idx = i % RTE_RETA_GROUP_SIZE; 621 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 622 reta_conf[reta_conf_idx].reta[reta_idx] = 623 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 624 } 625 626 return 0; 627 } 628 629 static int ena_rss_init_default(struct ena_adapter *adapter) 630 { 631 struct ena_com_dev *ena_dev = &adapter->ena_dev; 632 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 633 int rc, i; 634 u32 val; 635 636 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 637 if (unlikely(rc)) { 638 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 639 goto err_rss_init; 640 } 641 642 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 643 val = i % nb_rx_queues; 644 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 645 ENA_IO_RXQ_IDX(val)); 646 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 647 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 648 goto err_fill_indir; 649 } 650 } 651 652 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 653 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 654 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 655 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 656 goto err_fill_indir; 657 } 658 659 rc = ena_com_set_default_hash_ctrl(ena_dev); 660 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 661 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 662 goto err_fill_indir; 663 } 664 665 rc = ena_com_indirect_table_set(ena_dev); 666 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 667 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 668 goto err_fill_indir; 669 } 670 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 671 adapter->rte_dev->data->port_id); 672 673 return 0; 674 675 err_fill_indir: 676 ena_com_rss_destroy(ena_dev); 677 err_rss_init: 678 679 return rc; 680 } 681 682 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 683 { 684 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 685 int nb_queues = dev->data->nb_rx_queues; 686 int i; 687 688 for (i = 0; i < nb_queues; i++) 689 ena_rx_queue_release(queues[i]); 690 } 691 692 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 693 { 694 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 695 int nb_queues = dev->data->nb_tx_queues; 696 int i; 697 698 for (i = 0; i < nb_queues; i++) 699 ena_tx_queue_release(queues[i]); 700 } 701 702 static void ena_rx_queue_release(void *queue) 703 { 704 struct ena_ring *ring = (struct ena_ring *)queue; 705 706 /* Free ring resources */ 707 if (ring->rx_buffer_info) 708 rte_free(ring->rx_buffer_info); 709 ring->rx_buffer_info = NULL; 710 711 if (ring->rx_refill_buffer) 712 rte_free(ring->rx_refill_buffer); 713 ring->rx_refill_buffer = NULL; 714 715 if (ring->empty_rx_reqs) 716 rte_free(ring->empty_rx_reqs); 717 ring->empty_rx_reqs = NULL; 718 719 ring->configured = 0; 720 721 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 722 ring->port_id, ring->id); 723 } 724 725 static void ena_tx_queue_release(void *queue) 726 { 727 struct ena_ring *ring = (struct ena_ring *)queue; 728 729 /* Free ring resources */ 730 if (ring->tx_buffer_info) 731 rte_free(ring->tx_buffer_info); 732 733 if (ring->empty_tx_reqs) 734 rte_free(ring->empty_tx_reqs); 735 736 ring->empty_tx_reqs = NULL; 737 ring->tx_buffer_info = NULL; 738 739 ring->configured = 0; 740 741 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 742 ring->port_id, ring->id); 743 } 744 745 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 746 { 747 unsigned int i; 748 749 for (i = 0; i < ring->ring_size; ++i) 750 if (ring->rx_buffer_info[i]) { 751 rte_mbuf_raw_free(ring->rx_buffer_info[i]); 752 ring->rx_buffer_info[i] = NULL; 753 } 754 } 755 756 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 757 { 758 unsigned int i; 759 760 for (i = 0; i < ring->ring_size; ++i) { 761 struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i]; 762 763 if (tx_buf->mbuf) 764 rte_pktmbuf_free(tx_buf->mbuf); 765 } 766 } 767 768 static int ena_link_update(struct rte_eth_dev *dev, 769 __rte_unused int wait_to_complete) 770 { 771 struct rte_eth_link *link = &dev->data->dev_link; 772 struct ena_adapter *adapter = dev->data->dev_private; 773 774 link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN; 775 link->link_speed = ETH_SPEED_NUM_NONE; 776 link->link_duplex = ETH_LINK_FULL_DUPLEX; 777 778 return 0; 779 } 780 781 static int ena_queue_start_all(struct rte_eth_dev *dev, 782 enum ena_ring_type ring_type) 783 { 784 struct ena_adapter *adapter = dev->data->dev_private; 785 struct ena_ring *queues = NULL; 786 int nb_queues; 787 int i = 0; 788 int rc = 0; 789 790 if (ring_type == ENA_RING_TYPE_RX) { 791 queues = adapter->rx_ring; 792 nb_queues = dev->data->nb_rx_queues; 793 } else { 794 queues = adapter->tx_ring; 795 nb_queues = dev->data->nb_tx_queues; 796 } 797 for (i = 0; i < nb_queues; i++) { 798 if (queues[i].configured) { 799 if (ring_type == ENA_RING_TYPE_RX) { 800 ena_assert_msg( 801 dev->data->rx_queues[i] == &queues[i], 802 "Inconsistent state of rx queues\n"); 803 } else { 804 ena_assert_msg( 805 dev->data->tx_queues[i] == &queues[i], 806 "Inconsistent state of tx queues\n"); 807 } 808 809 rc = ena_queue_start(&queues[i]); 810 811 if (rc) { 812 PMD_INIT_LOG(ERR, 813 "failed to start queue %d type(%d)", 814 i, ring_type); 815 goto err; 816 } 817 } 818 } 819 820 return 0; 821 822 err: 823 while (i--) 824 if (queues[i].configured) 825 ena_queue_stop(&queues[i]); 826 827 return rc; 828 } 829 830 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 831 { 832 uint32_t max_frame_len = adapter->max_mtu; 833 834 if (adapter->rte_eth_dev_data->dev_conf.rxmode.offloads & 835 DEV_RX_OFFLOAD_JUMBO_FRAME) 836 max_frame_len = 837 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 838 839 return max_frame_len; 840 } 841 842 static int ena_check_valid_conf(struct ena_adapter *adapter) 843 { 844 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 845 846 if (max_frame_len > adapter->max_mtu || max_frame_len < ENA_MIN_MTU) { 847 PMD_INIT_LOG(ERR, "Unsupported MTU of %d. " 848 "max mtu: %d, min mtu: %d\n", 849 max_frame_len, adapter->max_mtu, ENA_MIN_MTU); 850 return ENA_COM_UNSUPPORTED; 851 } 852 853 return 0; 854 } 855 856 static int 857 ena_calc_queue_size(struct ena_com_dev *ena_dev, 858 u16 *max_tx_sgl_size, 859 struct ena_com_dev_get_features_ctx *get_feat_ctx) 860 { 861 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 862 863 queue_size = RTE_MIN(queue_size, 864 get_feat_ctx->max_queues.max_cq_depth); 865 queue_size = RTE_MIN(queue_size, 866 get_feat_ctx->max_queues.max_sq_depth); 867 868 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 869 queue_size = RTE_MIN(queue_size, 870 get_feat_ctx->max_queues.max_llq_depth); 871 872 /* Round down to power of 2 */ 873 if (!rte_is_power_of_2(queue_size)) 874 queue_size = rte_align32pow2(queue_size >> 1); 875 876 if (unlikely(queue_size == 0)) { 877 PMD_INIT_LOG(ERR, "Invalid queue size"); 878 return -EFAULT; 879 } 880 881 *max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 882 get_feat_ctx->max_queues.max_packet_tx_descs); 883 884 return queue_size; 885 } 886 887 static void ena_stats_restart(struct rte_eth_dev *dev) 888 { 889 struct ena_adapter *adapter = dev->data->dev_private; 890 891 rte_atomic64_init(&adapter->drv_stats->ierrors); 892 rte_atomic64_init(&adapter->drv_stats->oerrors); 893 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 894 } 895 896 static int ena_stats_get(struct rte_eth_dev *dev, 897 struct rte_eth_stats *stats) 898 { 899 struct ena_admin_basic_stats ena_stats; 900 struct ena_adapter *adapter = dev->data->dev_private; 901 struct ena_com_dev *ena_dev = &adapter->ena_dev; 902 int rc; 903 904 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 905 return -ENOTSUP; 906 907 memset(&ena_stats, 0, sizeof(ena_stats)); 908 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 909 if (unlikely(rc)) { 910 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 911 return rc; 912 } 913 914 /* Set of basic statistics from ENA */ 915 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 916 ena_stats.rx_pkts_low); 917 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 918 ena_stats.tx_pkts_low); 919 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 920 ena_stats.rx_bytes_low); 921 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 922 ena_stats.tx_bytes_low); 923 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 924 ena_stats.rx_drops_low); 925 926 /* Driver related stats */ 927 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 928 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 929 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 930 return 0; 931 } 932 933 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 934 { 935 struct ena_adapter *adapter; 936 struct ena_com_dev *ena_dev; 937 int rc = 0; 938 939 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 940 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 941 adapter = dev->data->dev_private; 942 943 ena_dev = &adapter->ena_dev; 944 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 945 946 if (mtu > ena_get_mtu_conf(adapter) || mtu < ENA_MIN_MTU) { 947 RTE_LOG(ERR, PMD, 948 "Invalid MTU setting. new_mtu: %d " 949 "max mtu: %d min mtu: %d\n", 950 mtu, ena_get_mtu_conf(adapter), ENA_MIN_MTU); 951 return -EINVAL; 952 } 953 954 rc = ena_com_set_dev_mtu(ena_dev, mtu); 955 if (rc) 956 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 957 else 958 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 959 960 return rc; 961 } 962 963 static int ena_start(struct rte_eth_dev *dev) 964 { 965 struct ena_adapter *adapter = dev->data->dev_private; 966 uint64_t ticks; 967 int rc = 0; 968 969 rc = ena_check_valid_conf(adapter); 970 if (rc) 971 return rc; 972 973 rc = ena_queue_start_all(dev, ENA_RING_TYPE_RX); 974 if (rc) 975 return rc; 976 977 rc = ena_queue_start_all(dev, ENA_RING_TYPE_TX); 978 if (rc) 979 goto err_start_tx; 980 981 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 982 ETH_MQ_RX_RSS_FLAG && adapter->rte_dev->data->nb_rx_queues > 0) { 983 rc = ena_rss_init_default(adapter); 984 if (rc) 985 goto err_rss_init; 986 } 987 988 ena_stats_restart(dev); 989 990 adapter->timestamp_wd = rte_get_timer_cycles(); 991 adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT; 992 993 ticks = rte_get_timer_hz(); 994 rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(), 995 ena_timer_wd_callback, adapter); 996 997 adapter->state = ENA_ADAPTER_STATE_RUNNING; 998 999 return 0; 1000 1001 err_rss_init: 1002 ena_queue_stop_all(dev, ENA_RING_TYPE_TX); 1003 err_start_tx: 1004 ena_queue_stop_all(dev, ENA_RING_TYPE_RX); 1005 return rc; 1006 } 1007 1008 static void ena_stop(struct rte_eth_dev *dev) 1009 { 1010 struct ena_adapter *adapter = dev->data->dev_private; 1011 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1012 int rc; 1013 1014 rte_timer_stop_sync(&adapter->timer_wd); 1015 ena_queue_stop_all(dev, ENA_RING_TYPE_TX); 1016 ena_queue_stop_all(dev, ENA_RING_TYPE_RX); 1017 1018 if (adapter->trigger_reset) { 1019 rc = ena_com_dev_reset(ena_dev, adapter->reset_reason); 1020 if (rc) 1021 RTE_LOG(ERR, PMD, "Device reset failed rc=%d\n", rc); 1022 } 1023 1024 adapter->state = ENA_ADAPTER_STATE_STOPPED; 1025 } 1026 1027 static int ena_create_io_queue(struct ena_ring *ring) 1028 { 1029 struct ena_adapter *adapter; 1030 struct ena_com_dev *ena_dev; 1031 struct ena_com_create_io_ctx ctx = 1032 /* policy set to _HOST just to satisfy icc compiler */ 1033 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1034 0, 0, 0, 0, 0 }; 1035 uint16_t ena_qid; 1036 unsigned int i; 1037 int rc; 1038 1039 adapter = ring->adapter; 1040 ena_dev = &adapter->ena_dev; 1041 1042 if (ring->type == ENA_RING_TYPE_TX) { 1043 ena_qid = ENA_IO_TXQ_IDX(ring->id); 1044 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1045 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1046 ctx.queue_size = adapter->tx_ring_size; 1047 for (i = 0; i < ring->ring_size; i++) 1048 ring->empty_tx_reqs[i] = i; 1049 } else { 1050 ena_qid = ENA_IO_RXQ_IDX(ring->id); 1051 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1052 ctx.queue_size = adapter->rx_ring_size; 1053 for (i = 0; i < ring->ring_size; i++) 1054 ring->empty_rx_reqs[i] = i; 1055 } 1056 ctx.qid = ena_qid; 1057 ctx.msix_vector = -1; /* interrupts not used */ 1058 ctx.numa_node = ring->numa_socket_id; 1059 1060 rc = ena_com_create_io_queue(ena_dev, &ctx); 1061 if (rc) { 1062 RTE_LOG(ERR, PMD, 1063 "failed to create io queue #%d (qid:%d) rc: %d\n", 1064 ring->id, ena_qid, rc); 1065 return rc; 1066 } 1067 1068 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1069 &ring->ena_com_io_sq, 1070 &ring->ena_com_io_cq); 1071 if (rc) { 1072 RTE_LOG(ERR, PMD, 1073 "Failed to get io queue handlers. queue num %d rc: %d\n", 1074 ring->id, rc); 1075 ena_com_destroy_io_queue(ena_dev, ena_qid); 1076 return rc; 1077 } 1078 1079 if (ring->type == ENA_RING_TYPE_TX) 1080 ena_com_update_numa_node(ring->ena_com_io_cq, ctx.numa_node); 1081 1082 return 0; 1083 } 1084 1085 static void ena_queue_stop(struct ena_ring *ring) 1086 { 1087 struct ena_com_dev *ena_dev = &ring->adapter->ena_dev; 1088 1089 if (ring->type == ENA_RING_TYPE_RX) { 1090 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(ring->id)); 1091 ena_rx_queue_release_bufs(ring); 1092 } else { 1093 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(ring->id)); 1094 ena_tx_queue_release_bufs(ring); 1095 } 1096 } 1097 1098 static void ena_queue_stop_all(struct rte_eth_dev *dev, 1099 enum ena_ring_type ring_type) 1100 { 1101 struct ena_adapter *adapter = dev->data->dev_private; 1102 struct ena_ring *queues = NULL; 1103 uint16_t nb_queues, i; 1104 1105 if (ring_type == ENA_RING_TYPE_RX) { 1106 queues = adapter->rx_ring; 1107 nb_queues = dev->data->nb_rx_queues; 1108 } else { 1109 queues = adapter->tx_ring; 1110 nb_queues = dev->data->nb_tx_queues; 1111 } 1112 1113 for (i = 0; i < nb_queues; ++i) 1114 if (queues[i].configured) 1115 ena_queue_stop(&queues[i]); 1116 } 1117 1118 static int ena_queue_start(struct ena_ring *ring) 1119 { 1120 int rc, bufs_num; 1121 1122 ena_assert_msg(ring->configured == 1, 1123 "Trying to start unconfigured queue\n"); 1124 1125 rc = ena_create_io_queue(ring); 1126 if (rc) { 1127 PMD_INIT_LOG(ERR, "Failed to create IO queue!\n"); 1128 return rc; 1129 } 1130 1131 ring->next_to_clean = 0; 1132 ring->next_to_use = 0; 1133 1134 if (ring->type == ENA_RING_TYPE_TX) 1135 return 0; 1136 1137 bufs_num = ring->ring_size - 1; 1138 rc = ena_populate_rx_queue(ring, bufs_num); 1139 if (rc != bufs_num) { 1140 ena_com_destroy_io_queue(&ring->adapter->ena_dev, 1141 ENA_IO_RXQ_IDX(ring->id)); 1142 PMD_INIT_LOG(ERR, "Failed to populate rx ring !"); 1143 return ENA_COM_FAULT; 1144 } 1145 1146 return 0; 1147 } 1148 1149 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 1150 uint16_t queue_idx, 1151 uint16_t nb_desc, 1152 unsigned int socket_id, 1153 const struct rte_eth_txconf *tx_conf) 1154 { 1155 struct ena_ring *txq = NULL; 1156 struct ena_adapter *adapter = dev->data->dev_private; 1157 unsigned int i; 1158 1159 txq = &adapter->tx_ring[queue_idx]; 1160 1161 if (txq->configured) { 1162 RTE_LOG(CRIT, PMD, 1163 "API violation. Queue %d is already configured\n", 1164 queue_idx); 1165 return ENA_COM_FAULT; 1166 } 1167 1168 if (!rte_is_power_of_2(nb_desc)) { 1169 RTE_LOG(ERR, PMD, 1170 "Unsupported size of TX queue: %d is not a power of 2.", 1171 nb_desc); 1172 return -EINVAL; 1173 } 1174 1175 if (nb_desc > adapter->tx_ring_size) { 1176 RTE_LOG(ERR, PMD, 1177 "Unsupported size of TX queue (max size: %d)\n", 1178 adapter->tx_ring_size); 1179 return -EINVAL; 1180 } 1181 1182 txq->port_id = dev->data->port_id; 1183 txq->next_to_clean = 0; 1184 txq->next_to_use = 0; 1185 txq->ring_size = nb_desc; 1186 txq->numa_socket_id = socket_id; 1187 1188 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1189 sizeof(struct ena_tx_buffer) * 1190 txq->ring_size, 1191 RTE_CACHE_LINE_SIZE); 1192 if (!txq->tx_buffer_info) { 1193 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 1194 return -ENOMEM; 1195 } 1196 1197 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1198 sizeof(u16) * txq->ring_size, 1199 RTE_CACHE_LINE_SIZE); 1200 if (!txq->empty_tx_reqs) { 1201 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 1202 rte_free(txq->tx_buffer_info); 1203 return -ENOMEM; 1204 } 1205 1206 for (i = 0; i < txq->ring_size; i++) 1207 txq->empty_tx_reqs[i] = i; 1208 1209 if (tx_conf != NULL) { 1210 txq->offloads = 1211 tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1212 } 1213 1214 /* Store pointer to this queue in upper layer */ 1215 txq->configured = 1; 1216 dev->data->tx_queues[queue_idx] = txq; 1217 1218 return 0; 1219 } 1220 1221 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1222 uint16_t queue_idx, 1223 uint16_t nb_desc, 1224 unsigned int socket_id, 1225 __rte_unused const struct rte_eth_rxconf *rx_conf, 1226 struct rte_mempool *mp) 1227 { 1228 struct ena_adapter *adapter = dev->data->dev_private; 1229 struct ena_ring *rxq = NULL; 1230 int i; 1231 1232 rxq = &adapter->rx_ring[queue_idx]; 1233 if (rxq->configured) { 1234 RTE_LOG(CRIT, PMD, 1235 "API violation. Queue %d is already configured\n", 1236 queue_idx); 1237 return ENA_COM_FAULT; 1238 } 1239 1240 if (!rte_is_power_of_2(nb_desc)) { 1241 RTE_LOG(ERR, PMD, 1242 "Unsupported size of RX queue: %d is not a power of 2.", 1243 nb_desc); 1244 return -EINVAL; 1245 } 1246 1247 if (nb_desc > adapter->rx_ring_size) { 1248 RTE_LOG(ERR, PMD, 1249 "Unsupported size of RX queue (max size: %d)\n", 1250 adapter->rx_ring_size); 1251 return -EINVAL; 1252 } 1253 1254 rxq->port_id = dev->data->port_id; 1255 rxq->next_to_clean = 0; 1256 rxq->next_to_use = 0; 1257 rxq->ring_size = nb_desc; 1258 rxq->numa_socket_id = socket_id; 1259 rxq->mb_pool = mp; 1260 1261 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1262 sizeof(struct rte_mbuf *) * nb_desc, 1263 RTE_CACHE_LINE_SIZE); 1264 if (!rxq->rx_buffer_info) { 1265 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 1266 return -ENOMEM; 1267 } 1268 1269 rxq->rx_refill_buffer = rte_zmalloc("rxq->rx_refill_buffer", 1270 sizeof(struct rte_mbuf *) * nb_desc, 1271 RTE_CACHE_LINE_SIZE); 1272 1273 if (!rxq->rx_refill_buffer) { 1274 RTE_LOG(ERR, PMD, "failed to alloc mem for rx refill buffer\n"); 1275 rte_free(rxq->rx_buffer_info); 1276 rxq->rx_buffer_info = NULL; 1277 return -ENOMEM; 1278 } 1279 1280 rxq->empty_rx_reqs = rte_zmalloc("rxq->empty_rx_reqs", 1281 sizeof(uint16_t) * nb_desc, 1282 RTE_CACHE_LINE_SIZE); 1283 if (!rxq->empty_rx_reqs) { 1284 RTE_LOG(ERR, PMD, "failed to alloc mem for empty rx reqs\n"); 1285 rte_free(rxq->rx_buffer_info); 1286 rxq->rx_buffer_info = NULL; 1287 rte_free(rxq->rx_refill_buffer); 1288 rxq->rx_refill_buffer = NULL; 1289 return -ENOMEM; 1290 } 1291 1292 for (i = 0; i < nb_desc; i++) 1293 rxq->empty_rx_reqs[i] = i; 1294 1295 /* Store pointer to this queue in upper layer */ 1296 rxq->configured = 1; 1297 dev->data->rx_queues[queue_idx] = rxq; 1298 1299 return 0; 1300 } 1301 1302 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1303 { 1304 unsigned int i; 1305 int rc; 1306 uint16_t ring_size = rxq->ring_size; 1307 uint16_t ring_mask = ring_size - 1; 1308 uint16_t next_to_use = rxq->next_to_use; 1309 uint16_t in_use, req_id; 1310 struct rte_mbuf **mbufs = rxq->rx_refill_buffer; 1311 1312 if (unlikely(!count)) 1313 return 0; 1314 1315 in_use = rxq->next_to_use - rxq->next_to_clean; 1316 ena_assert_msg(((in_use + count) < ring_size), "bad ring state"); 1317 1318 /* get resources for incoming packets */ 1319 rc = rte_mempool_get_bulk(rxq->mb_pool, (void **)mbufs, count); 1320 if (unlikely(rc < 0)) { 1321 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1322 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 1323 return 0; 1324 } 1325 1326 for (i = 0; i < count; i++) { 1327 uint16_t next_to_use_masked = next_to_use & ring_mask; 1328 struct rte_mbuf *mbuf = mbufs[i]; 1329 struct ena_com_buf ebuf; 1330 1331 if (likely((i + 4) < count)) 1332 rte_prefetch0(mbufs[i + 4]); 1333 1334 req_id = rxq->empty_rx_reqs[next_to_use_masked]; 1335 rc = validate_rx_req_id(rxq, req_id); 1336 if (unlikely(rc < 0)) 1337 break; 1338 rxq->rx_buffer_info[req_id] = mbuf; 1339 1340 /* prepare physical address for DMA transaction */ 1341 ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM; 1342 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1343 /* pass resource to device */ 1344 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 1345 &ebuf, req_id); 1346 if (unlikely(rc)) { 1347 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 1348 rxq->rx_buffer_info[req_id] = NULL; 1349 break; 1350 } 1351 next_to_use++; 1352 } 1353 1354 if (unlikely(i < count)) { 1355 RTE_LOG(WARNING, PMD, "refilled rx qid %d with only %d " 1356 "buffers (from %d)\n", rxq->id, i, count); 1357 rte_mempool_put_bulk(rxq->mb_pool, (void **)(&mbufs[i]), 1358 count - i); 1359 } 1360 1361 /* When we submitted free recources to device... */ 1362 if (likely(i > 0)) { 1363 /* ...let HW know that it can fill buffers with data 1364 * 1365 * Add memory barrier to make sure the desc were written before 1366 * issue a doorbell 1367 */ 1368 rte_wmb(); 1369 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1370 1371 rxq->next_to_use = next_to_use; 1372 } 1373 1374 return i; 1375 } 1376 1377 static int ena_device_init(struct ena_com_dev *ena_dev, 1378 struct ena_com_dev_get_features_ctx *get_feat_ctx, 1379 bool *wd_state) 1380 { 1381 uint32_t aenq_groups; 1382 int rc; 1383 bool readless_supported; 1384 1385 /* Initialize mmio registers */ 1386 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1387 if (rc) { 1388 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 1389 return rc; 1390 } 1391 1392 /* The PCIe configuration space revision id indicate if mmio reg 1393 * read is disabled. 1394 */ 1395 readless_supported = 1396 !(((struct rte_pci_device *)ena_dev->dmadev)->id.class_id 1397 & ENA_MMIO_DISABLE_REG_READ); 1398 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1399 1400 /* reset device */ 1401 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 1402 if (rc) { 1403 RTE_LOG(ERR, PMD, "cannot reset device\n"); 1404 goto err_mmio_read_less; 1405 } 1406 1407 /* check FW version */ 1408 rc = ena_com_validate_version(ena_dev); 1409 if (rc) { 1410 RTE_LOG(ERR, PMD, "device version is too low\n"); 1411 goto err_mmio_read_less; 1412 } 1413 1414 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1415 1416 /* ENA device administration layer init */ 1417 rc = ena_com_admin_init(ena_dev, &aenq_handlers, true); 1418 if (rc) { 1419 RTE_LOG(ERR, PMD, 1420 "cannot initialize ena admin queue with device\n"); 1421 goto err_mmio_read_less; 1422 } 1423 1424 /* To enable the msix interrupts the driver needs to know the number 1425 * of queues. So the driver uses polling mode to retrieve this 1426 * information. 1427 */ 1428 ena_com_set_admin_polling_mode(ena_dev, true); 1429 1430 ena_config_host_info(ena_dev); 1431 1432 /* Get Device Attributes and features */ 1433 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1434 if (rc) { 1435 RTE_LOG(ERR, PMD, 1436 "cannot get attribute for ena device rc= %d\n", rc); 1437 goto err_admin_init; 1438 } 1439 1440 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 1441 BIT(ENA_ADMIN_NOTIFICATION) | 1442 BIT(ENA_ADMIN_KEEP_ALIVE) | 1443 BIT(ENA_ADMIN_FATAL_ERROR) | 1444 BIT(ENA_ADMIN_WARNING); 1445 1446 aenq_groups &= get_feat_ctx->aenq.supported_groups; 1447 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 1448 if (rc) { 1449 RTE_LOG(ERR, PMD, "Cannot configure aenq groups rc: %d\n", rc); 1450 goto err_admin_init; 1451 } 1452 1453 *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 1454 1455 return 0; 1456 1457 err_admin_init: 1458 ena_com_admin_destroy(ena_dev); 1459 1460 err_mmio_read_less: 1461 ena_com_mmio_reg_read_request_destroy(ena_dev); 1462 1463 return rc; 1464 } 1465 1466 static void ena_interrupt_handler_rte(void *cb_arg) 1467 { 1468 struct ena_adapter *adapter = cb_arg; 1469 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1470 1471 ena_com_admin_q_comp_intr_handler(ena_dev); 1472 if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED)) 1473 ena_com_aenq_intr_handler(ena_dev, adapter); 1474 } 1475 1476 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 1477 { 1478 if (!adapter->wd_state) 1479 return; 1480 1481 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 1482 return; 1483 1484 if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >= 1485 adapter->keep_alive_timeout)) { 1486 RTE_LOG(ERR, PMD, "Keep alive timeout\n"); 1487 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO; 1488 adapter->trigger_reset = true; 1489 } 1490 } 1491 1492 /* Check if admin queue is enabled */ 1493 static void check_for_admin_com_state(struct ena_adapter *adapter) 1494 { 1495 if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) { 1496 RTE_LOG(ERR, PMD, "ENA admin queue is not in running state!\n"); 1497 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO; 1498 adapter->trigger_reset = true; 1499 } 1500 } 1501 1502 static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer, 1503 void *arg) 1504 { 1505 struct ena_adapter *adapter = arg; 1506 struct rte_eth_dev *dev = adapter->rte_dev; 1507 1508 check_for_missing_keep_alive(adapter); 1509 check_for_admin_com_state(adapter); 1510 1511 if (unlikely(adapter->trigger_reset)) { 1512 RTE_LOG(ERR, PMD, "Trigger reset is on\n"); 1513 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 1514 NULL); 1515 } 1516 } 1517 1518 static int ena_calc_io_queue_num(__rte_unused struct ena_com_dev *ena_dev, 1519 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1520 { 1521 int io_sq_num, io_cq_num, io_queue_num; 1522 1523 io_sq_num = get_feat_ctx->max_queues.max_sq_num; 1524 io_cq_num = get_feat_ctx->max_queues.max_cq_num; 1525 1526 io_queue_num = RTE_MIN(io_sq_num, io_cq_num); 1527 1528 if (unlikely(io_queue_num == 0)) { 1529 RTE_LOG(ERR, PMD, "Number of IO queues should not be 0\n"); 1530 return -EFAULT; 1531 } 1532 1533 return io_queue_num; 1534 } 1535 1536 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1537 { 1538 struct rte_pci_device *pci_dev; 1539 struct rte_intr_handle *intr_handle; 1540 struct ena_adapter *adapter = eth_dev->data->dev_private; 1541 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1542 struct ena_com_dev_get_features_ctx get_feat_ctx; 1543 int queue_size, rc; 1544 u16 tx_sgl_size = 0; 1545 1546 static int adapters_found; 1547 bool wd_state; 1548 1549 eth_dev->dev_ops = &ena_dev_ops; 1550 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1551 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1552 eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; 1553 1554 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1555 return 0; 1556 1557 memset(adapter, 0, sizeof(struct ena_adapter)); 1558 ena_dev = &adapter->ena_dev; 1559 1560 adapter->rte_eth_dev_data = eth_dev->data; 1561 adapter->rte_dev = eth_dev; 1562 1563 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1564 adapter->pdev = pci_dev; 1565 1566 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d", 1567 pci_dev->addr.domain, 1568 pci_dev->addr.bus, 1569 pci_dev->addr.devid, 1570 pci_dev->addr.function); 1571 1572 intr_handle = &pci_dev->intr_handle; 1573 1574 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1575 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1576 1577 if (!adapter->regs) { 1578 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)", 1579 ENA_REGS_BAR); 1580 return -ENXIO; 1581 } 1582 1583 ena_dev->reg_bar = adapter->regs; 1584 ena_dev->dmadev = adapter->pdev; 1585 1586 adapter->id_number = adapters_found; 1587 1588 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1589 adapter->id_number); 1590 1591 /* device specific initialization routine */ 1592 rc = ena_device_init(ena_dev, &get_feat_ctx, &wd_state); 1593 if (rc) { 1594 PMD_INIT_LOG(CRIT, "Failed to init ENA device"); 1595 goto err; 1596 } 1597 adapter->wd_state = wd_state; 1598 1599 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1600 adapter->num_queues = ena_calc_io_queue_num(ena_dev, 1601 &get_feat_ctx); 1602 1603 queue_size = ena_calc_queue_size(ena_dev, &tx_sgl_size, &get_feat_ctx); 1604 if (queue_size <= 0 || adapter->num_queues <= 0) { 1605 rc = -EFAULT; 1606 goto err_device_destroy; 1607 } 1608 1609 adapter->tx_ring_size = queue_size; 1610 adapter->rx_ring_size = queue_size; 1611 1612 adapter->max_tx_sgl_size = tx_sgl_size; 1613 1614 /* prepare ring structures */ 1615 ena_init_rings(adapter); 1616 1617 ena_config_debug_area(adapter); 1618 1619 /* Set max MTU for this device */ 1620 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1621 1622 /* set device support for TSO */ 1623 adapter->tso4_supported = get_feat_ctx.offload.tx & 1624 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK; 1625 1626 /* Copy MAC address and point DPDK to it */ 1627 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1628 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1629 (struct ether_addr *)adapter->mac_addr); 1630 1631 /* 1632 * Pass the information to the rte_eth_dev_close() that it should also 1633 * release the private port resources. 1634 */ 1635 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 1636 1637 adapter->drv_stats = rte_zmalloc("adapter stats", 1638 sizeof(*adapter->drv_stats), 1639 RTE_CACHE_LINE_SIZE); 1640 if (!adapter->drv_stats) { 1641 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1642 rc = -ENOMEM; 1643 goto err_delete_debug_area; 1644 } 1645 1646 rte_intr_callback_register(intr_handle, 1647 ena_interrupt_handler_rte, 1648 adapter); 1649 rte_intr_enable(intr_handle); 1650 ena_com_set_admin_polling_mode(ena_dev, false); 1651 ena_com_admin_aenq_enable(ena_dev); 1652 1653 if (adapters_found == 0) 1654 rte_timer_subsystem_init(); 1655 rte_timer_init(&adapter->timer_wd); 1656 1657 adapters_found++; 1658 adapter->state = ENA_ADAPTER_STATE_INIT; 1659 1660 return 0; 1661 1662 err_delete_debug_area: 1663 ena_com_delete_debug_area(ena_dev); 1664 1665 err_device_destroy: 1666 ena_com_delete_host_info(ena_dev); 1667 ena_com_admin_destroy(ena_dev); 1668 1669 err: 1670 return rc; 1671 } 1672 1673 static void ena_destroy_device(struct rte_eth_dev *eth_dev) 1674 { 1675 struct ena_adapter *adapter = eth_dev->data->dev_private; 1676 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1677 1678 if (adapter->state == ENA_ADAPTER_STATE_FREE) 1679 return; 1680 1681 ena_com_set_admin_running_state(ena_dev, false); 1682 1683 if (adapter->state != ENA_ADAPTER_STATE_CLOSED) 1684 ena_close(eth_dev); 1685 1686 ena_com_delete_debug_area(ena_dev); 1687 ena_com_delete_host_info(ena_dev); 1688 1689 ena_com_abort_admin_commands(ena_dev); 1690 ena_com_wait_for_abort_completion(ena_dev); 1691 ena_com_admin_destroy(ena_dev); 1692 ena_com_mmio_reg_read_request_destroy(ena_dev); 1693 1694 adapter->state = ENA_ADAPTER_STATE_FREE; 1695 } 1696 1697 static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev) 1698 { 1699 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1700 return 0; 1701 1702 ena_destroy_device(eth_dev); 1703 1704 eth_dev->dev_ops = NULL; 1705 eth_dev->rx_pkt_burst = NULL; 1706 eth_dev->tx_pkt_burst = NULL; 1707 eth_dev->tx_pkt_prepare = NULL; 1708 1709 return 0; 1710 } 1711 1712 static int ena_dev_configure(struct rte_eth_dev *dev) 1713 { 1714 struct ena_adapter *adapter = dev->data->dev_private; 1715 1716 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1717 1718 adapter->tx_selected_offloads = dev->data->dev_conf.txmode.offloads; 1719 adapter->rx_selected_offloads = dev->data->dev_conf.rxmode.offloads; 1720 return 0; 1721 } 1722 1723 static void ena_init_rings(struct ena_adapter *adapter) 1724 { 1725 int i; 1726 1727 for (i = 0; i < adapter->num_queues; i++) { 1728 struct ena_ring *ring = &adapter->tx_ring[i]; 1729 1730 ring->configured = 0; 1731 ring->type = ENA_RING_TYPE_TX; 1732 ring->adapter = adapter; 1733 ring->id = i; 1734 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1735 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1736 ring->sgl_size = adapter->max_tx_sgl_size; 1737 } 1738 1739 for (i = 0; i < adapter->num_queues; i++) { 1740 struct ena_ring *ring = &adapter->rx_ring[i]; 1741 1742 ring->configured = 0; 1743 ring->type = ENA_RING_TYPE_RX; 1744 ring->adapter = adapter; 1745 ring->id = i; 1746 } 1747 } 1748 1749 static void ena_infos_get(struct rte_eth_dev *dev, 1750 struct rte_eth_dev_info *dev_info) 1751 { 1752 struct ena_adapter *adapter; 1753 struct ena_com_dev *ena_dev; 1754 struct ena_com_dev_get_features_ctx feat; 1755 uint64_t rx_feat = 0, tx_feat = 0; 1756 int rc = 0; 1757 1758 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1759 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1760 adapter = dev->data->dev_private; 1761 1762 ena_dev = &adapter->ena_dev; 1763 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1764 1765 dev_info->speed_capa = 1766 ETH_LINK_SPEED_1G | 1767 ETH_LINK_SPEED_2_5G | 1768 ETH_LINK_SPEED_5G | 1769 ETH_LINK_SPEED_10G | 1770 ETH_LINK_SPEED_25G | 1771 ETH_LINK_SPEED_40G | 1772 ETH_LINK_SPEED_50G | 1773 ETH_LINK_SPEED_100G; 1774 1775 /* Get supported features from HW */ 1776 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1777 if (unlikely(rc)) { 1778 RTE_LOG(ERR, PMD, 1779 "Cannot get attribute for ena device rc= %d\n", rc); 1780 return; 1781 } 1782 1783 /* Set Tx & Rx features available for device */ 1784 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1785 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1786 1787 if (feat.offload.tx & 1788 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1789 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1790 DEV_TX_OFFLOAD_UDP_CKSUM | 1791 DEV_TX_OFFLOAD_TCP_CKSUM; 1792 1793 if (feat.offload.rx_supported & 1794 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1795 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1796 DEV_RX_OFFLOAD_UDP_CKSUM | 1797 DEV_RX_OFFLOAD_TCP_CKSUM; 1798 1799 rx_feat |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1800 1801 /* Inform framework about available features */ 1802 dev_info->rx_offload_capa = rx_feat; 1803 dev_info->rx_queue_offload_capa = rx_feat; 1804 dev_info->tx_offload_capa = tx_feat; 1805 dev_info->tx_queue_offload_capa = tx_feat; 1806 1807 dev_info->flow_type_rss_offloads = ETH_RSS_IP | ETH_RSS_TCP | 1808 ETH_RSS_UDP; 1809 1810 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1811 dev_info->max_rx_pktlen = adapter->max_mtu; 1812 dev_info->max_mac_addrs = 1; 1813 1814 dev_info->max_rx_queues = adapter->num_queues; 1815 dev_info->max_tx_queues = adapter->num_queues; 1816 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1817 1818 adapter->tx_supported_offloads = tx_feat; 1819 adapter->rx_supported_offloads = rx_feat; 1820 1821 dev_info->rx_desc_lim.nb_max = ENA_MAX_RING_DESC; 1822 dev_info->rx_desc_lim.nb_min = ENA_MIN_RING_DESC; 1823 1824 dev_info->tx_desc_lim.nb_max = ENA_MAX_RING_DESC; 1825 dev_info->tx_desc_lim.nb_min = ENA_MIN_RING_DESC; 1826 dev_info->tx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 1827 feat.max_queues.max_packet_tx_descs); 1828 dev_info->tx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 1829 feat.max_queues.max_packet_tx_descs); 1830 } 1831 1832 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1833 uint16_t nb_pkts) 1834 { 1835 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1836 unsigned int ring_size = rx_ring->ring_size; 1837 unsigned int ring_mask = ring_size - 1; 1838 uint16_t next_to_clean = rx_ring->next_to_clean; 1839 uint16_t desc_in_use = 0; 1840 uint16_t req_id; 1841 unsigned int recv_idx = 0; 1842 struct rte_mbuf *mbuf = NULL; 1843 struct rte_mbuf *mbuf_head = NULL; 1844 struct rte_mbuf *mbuf_prev = NULL; 1845 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1846 unsigned int completed; 1847 1848 struct ena_com_rx_ctx ena_rx_ctx; 1849 int rc = 0; 1850 1851 /* Check adapter state */ 1852 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1853 RTE_LOG(ALERT, PMD, 1854 "Trying to receive pkts while device is NOT running\n"); 1855 return 0; 1856 } 1857 1858 desc_in_use = rx_ring->next_to_use - next_to_clean; 1859 if (unlikely(nb_pkts > desc_in_use)) 1860 nb_pkts = desc_in_use; 1861 1862 for (completed = 0; completed < nb_pkts; completed++) { 1863 int segments = 0; 1864 1865 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1866 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1867 ena_rx_ctx.descs = 0; 1868 /* receive packet context */ 1869 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1870 rx_ring->ena_com_io_sq, 1871 &ena_rx_ctx); 1872 if (unlikely(rc)) { 1873 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1874 rx_ring->adapter->reset_reason = 1875 ENA_REGS_RESET_TOO_MANY_RX_DESCS; 1876 rx_ring->adapter->trigger_reset = true; 1877 return 0; 1878 } 1879 1880 if (unlikely(ena_rx_ctx.descs == 0)) 1881 break; 1882 1883 while (segments < ena_rx_ctx.descs) { 1884 req_id = ena_rx_ctx.ena_bufs[segments].req_id; 1885 rc = validate_rx_req_id(rx_ring, req_id); 1886 if (unlikely(rc)) { 1887 if (segments != 0) 1888 rte_mbuf_raw_free(mbuf_head); 1889 break; 1890 } 1891 1892 mbuf = rx_buff_info[req_id]; 1893 rx_buff_info[req_id] = NULL; 1894 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1895 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1896 mbuf->refcnt = 1; 1897 mbuf->next = NULL; 1898 if (unlikely(segments == 0)) { 1899 mbuf->nb_segs = ena_rx_ctx.descs; 1900 mbuf->port = rx_ring->port_id; 1901 mbuf->pkt_len = 0; 1902 mbuf_head = mbuf; 1903 } else { 1904 /* for multi-segment pkts create mbuf chain */ 1905 mbuf_prev->next = mbuf; 1906 } 1907 mbuf_head->pkt_len += mbuf->data_len; 1908 1909 mbuf_prev = mbuf; 1910 rx_ring->empty_rx_reqs[next_to_clean & ring_mask] = 1911 req_id; 1912 segments++; 1913 next_to_clean++; 1914 } 1915 if (unlikely(rc)) 1916 break; 1917 1918 /* fill mbuf attributes if any */ 1919 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1920 1921 if (unlikely(mbuf_head->ol_flags & 1922 (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD))) 1923 rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors); 1924 1925 1926 mbuf_head->hash.rss = ena_rx_ctx.hash; 1927 1928 /* pass to DPDK application head mbuf */ 1929 rx_pkts[recv_idx] = mbuf_head; 1930 recv_idx++; 1931 } 1932 1933 rx_ring->next_to_clean = next_to_clean; 1934 1935 desc_in_use = desc_in_use - completed + 1; 1936 /* Burst refill to save doorbells, memory barriers, const interval */ 1937 if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size)) { 1938 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); 1939 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use); 1940 } 1941 1942 return recv_idx; 1943 } 1944 1945 static uint16_t 1946 eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1947 uint16_t nb_pkts) 1948 { 1949 int32_t ret; 1950 uint32_t i; 1951 struct rte_mbuf *m; 1952 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1953 struct ipv4_hdr *ip_hdr; 1954 uint64_t ol_flags; 1955 uint16_t frag_field; 1956 1957 for (i = 0; i != nb_pkts; i++) { 1958 m = tx_pkts[i]; 1959 ol_flags = m->ol_flags; 1960 1961 if (!(ol_flags & PKT_TX_IPV4)) 1962 continue; 1963 1964 /* If there was not L2 header length specified, assume it is 1965 * length of the ethernet header. 1966 */ 1967 if (unlikely(m->l2_len == 0)) 1968 m->l2_len = sizeof(struct ether_hdr); 1969 1970 ip_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, 1971 m->l2_len); 1972 frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset); 1973 1974 if ((frag_field & IPV4_HDR_DF_FLAG) != 0) { 1975 m->packet_type |= RTE_PTYPE_L4_NONFRAG; 1976 1977 /* If IPv4 header has DF flag enabled and TSO support is 1978 * disabled, partial chcecksum should not be calculated. 1979 */ 1980 if (!tx_ring->adapter->tso4_supported) 1981 continue; 1982 } 1983 1984 if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || 1985 (ol_flags & PKT_TX_L4_MASK) == 1986 PKT_TX_SCTP_CKSUM) { 1987 rte_errno = ENOTSUP; 1988 return i; 1989 } 1990 1991 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1992 ret = rte_validate_tx_offload(m); 1993 if (ret != 0) { 1994 rte_errno = -ret; 1995 return i; 1996 } 1997 #endif 1998 1999 /* In case we are supposed to TSO and have DF not set (DF=0) 2000 * hardware must be provided with partial checksum, otherwise 2001 * it will take care of necessary calculations. 2002 */ 2003 2004 ret = rte_net_intel_cksum_flags_prepare(m, 2005 ol_flags & ~PKT_TX_TCP_SEG); 2006 if (ret != 0) { 2007 rte_errno = -ret; 2008 return i; 2009 } 2010 } 2011 2012 return i; 2013 } 2014 2015 static void ena_update_hints(struct ena_adapter *adapter, 2016 struct ena_admin_ena_hw_hints *hints) 2017 { 2018 if (hints->admin_completion_tx_timeout) 2019 adapter->ena_dev.admin_queue.completion_timeout = 2020 hints->admin_completion_tx_timeout * 1000; 2021 2022 if (hints->mmio_read_timeout) 2023 /* convert to usec */ 2024 adapter->ena_dev.mmio_read.reg_read_to = 2025 hints->mmio_read_timeout * 1000; 2026 2027 if (hints->driver_watchdog_timeout) { 2028 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2029 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 2030 else 2031 // Convert msecs to ticks 2032 adapter->keep_alive_timeout = 2033 (hints->driver_watchdog_timeout * 2034 rte_get_timer_hz()) / 1000; 2035 } 2036 } 2037 2038 static int ena_check_and_linearize_mbuf(struct ena_ring *tx_ring, 2039 struct rte_mbuf *mbuf) 2040 { 2041 int num_segments, rc; 2042 2043 num_segments = mbuf->nb_segs; 2044 2045 if (likely(num_segments < tx_ring->sgl_size)) 2046 return 0; 2047 2048 rc = rte_pktmbuf_linearize(mbuf); 2049 if (unlikely(rc)) 2050 RTE_LOG(WARNING, PMD, "Mbuf linearize failed\n"); 2051 2052 return rc; 2053 } 2054 2055 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 2056 uint16_t nb_pkts) 2057 { 2058 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 2059 uint16_t next_to_use = tx_ring->next_to_use; 2060 uint16_t next_to_clean = tx_ring->next_to_clean; 2061 struct rte_mbuf *mbuf; 2062 unsigned int ring_size = tx_ring->ring_size; 2063 unsigned int ring_mask = ring_size - 1; 2064 struct ena_com_tx_ctx ena_tx_ctx; 2065 struct ena_tx_buffer *tx_info; 2066 struct ena_com_buf *ebuf; 2067 uint16_t rc, req_id, total_tx_descs = 0; 2068 uint16_t sent_idx = 0, empty_tx_reqs; 2069 int nb_hw_desc; 2070 2071 /* Check adapter state */ 2072 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 2073 RTE_LOG(ALERT, PMD, 2074 "Trying to xmit pkts while device is NOT running\n"); 2075 return 0; 2076 } 2077 2078 empty_tx_reqs = ring_size - (next_to_use - next_to_clean); 2079 if (nb_pkts > empty_tx_reqs) 2080 nb_pkts = empty_tx_reqs; 2081 2082 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 2083 mbuf = tx_pkts[sent_idx]; 2084 2085 rc = ena_check_and_linearize_mbuf(tx_ring, mbuf); 2086 if (unlikely(rc)) 2087 break; 2088 2089 req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask]; 2090 tx_info = &tx_ring->tx_buffer_info[req_id]; 2091 tx_info->mbuf = mbuf; 2092 tx_info->num_of_bufs = 0; 2093 ebuf = tx_info->bufs; 2094 2095 /* Prepare TX context */ 2096 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 2097 memset(&ena_tx_ctx.ena_meta, 0x0, 2098 sizeof(struct ena_com_tx_meta)); 2099 ena_tx_ctx.ena_bufs = ebuf; 2100 ena_tx_ctx.req_id = req_id; 2101 if (tx_ring->tx_mem_queue_type == 2102 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 2103 /* prepare the push buffer with 2104 * virtual address of the data 2105 */ 2106 ena_tx_ctx.header_len = 2107 RTE_MIN(mbuf->data_len, 2108 tx_ring->tx_max_header_size); 2109 ena_tx_ctx.push_header = 2110 (void *)((char *)mbuf->buf_addr + 2111 mbuf->data_off); 2112 } /* there's no else as we take advantage of memset zeroing */ 2113 2114 /* Set TX offloads flags, if applicable */ 2115 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads); 2116 2117 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 2118 2119 /* Process first segment taking into 2120 * consideration pushed header 2121 */ 2122 if (mbuf->data_len > ena_tx_ctx.header_len) { 2123 ebuf->paddr = mbuf->buf_iova + 2124 mbuf->data_off + 2125 ena_tx_ctx.header_len; 2126 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 2127 ebuf++; 2128 tx_info->num_of_bufs++; 2129 } 2130 2131 while ((mbuf = mbuf->next) != NULL) { 2132 ebuf->paddr = mbuf->buf_iova + mbuf->data_off; 2133 ebuf->len = mbuf->data_len; 2134 ebuf++; 2135 tx_info->num_of_bufs++; 2136 } 2137 2138 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 2139 2140 /* Write data to device */ 2141 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 2142 &ena_tx_ctx, &nb_hw_desc); 2143 if (unlikely(rc)) 2144 break; 2145 2146 tx_info->tx_descs = nb_hw_desc; 2147 2148 next_to_use++; 2149 } 2150 2151 /* If there are ready packets to be xmitted... */ 2152 if (sent_idx > 0) { 2153 /* ...let HW do its best :-) */ 2154 rte_wmb(); 2155 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 2156 2157 tx_ring->next_to_use = next_to_use; 2158 } 2159 2160 /* Clear complete packets */ 2161 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 2162 rc = validate_tx_req_id(tx_ring, req_id); 2163 if (rc) 2164 break; 2165 2166 /* Get Tx info & store how many descs were processed */ 2167 tx_info = &tx_ring->tx_buffer_info[req_id]; 2168 total_tx_descs += tx_info->tx_descs; 2169 2170 /* Free whole mbuf chain */ 2171 mbuf = tx_info->mbuf; 2172 rte_pktmbuf_free(mbuf); 2173 tx_info->mbuf = NULL; 2174 2175 /* Put back descriptor to the ring for reuse */ 2176 tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; 2177 next_to_clean++; 2178 2179 /* If too many descs to clean, leave it for another run */ 2180 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 2181 break; 2182 } 2183 2184 if (total_tx_descs > 0) { 2185 /* acknowledge completion of sent packets */ 2186 tx_ring->next_to_clean = next_to_clean; 2187 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 2188 ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); 2189 } 2190 2191 return sent_idx; 2192 } 2193 2194 /********************************************************************* 2195 * PMD configuration 2196 *********************************************************************/ 2197 static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2198 struct rte_pci_device *pci_dev) 2199 { 2200 return rte_eth_dev_pci_generic_probe(pci_dev, 2201 sizeof(struct ena_adapter), eth_ena_dev_init); 2202 } 2203 2204 static int eth_ena_pci_remove(struct rte_pci_device *pci_dev) 2205 { 2206 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ena_dev_uninit); 2207 } 2208 2209 static struct rte_pci_driver rte_ena_pmd = { 2210 .id_table = pci_id_ena_map, 2211 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 2212 RTE_PCI_DRV_WC_ACTIVATE, 2213 .probe = eth_ena_pci_probe, 2214 .remove = eth_ena_pci_remove, 2215 }; 2216 2217 RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd); 2218 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 2219 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci"); 2220 2221 RTE_INIT(ena_init_log) 2222 { 2223 ena_logtype_init = rte_log_register("pmd.net.ena.init"); 2224 if (ena_logtype_init >= 0) 2225 rte_log_set_level(ena_logtype_init, RTE_LOG_NOTICE); 2226 ena_logtype_driver = rte_log_register("pmd.net.ena.driver"); 2227 if (ena_logtype_driver >= 0) 2228 rte_log_set_level(ena_logtype_driver, RTE_LOG_NOTICE); 2229 } 2230 2231 /****************************************************************************** 2232 ******************************** AENQ Handlers ******************************* 2233 *****************************************************************************/ 2234 static void ena_update_on_link_change(void *adapter_data, 2235 struct ena_admin_aenq_entry *aenq_e) 2236 { 2237 struct rte_eth_dev *eth_dev; 2238 struct ena_adapter *adapter; 2239 struct ena_admin_aenq_link_change_desc *aenq_link_desc; 2240 uint32_t status; 2241 2242 adapter = adapter_data; 2243 aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 2244 eth_dev = adapter->rte_dev; 2245 2246 status = get_ena_admin_aenq_link_change_desc_link_status(aenq_link_desc); 2247 adapter->link_status = status; 2248 2249 ena_link_update(eth_dev, 0); 2250 _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2251 } 2252 2253 static void ena_notification(void *data, 2254 struct ena_admin_aenq_entry *aenq_e) 2255 { 2256 struct ena_adapter *adapter = data; 2257 struct ena_admin_ena_hw_hints *hints; 2258 2259 if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION) 2260 RTE_LOG(WARNING, PMD, "Invalid group(%x) expected %x\n", 2261 aenq_e->aenq_common_desc.group, 2262 ENA_ADMIN_NOTIFICATION); 2263 2264 switch (aenq_e->aenq_common_desc.syndrom) { 2265 case ENA_ADMIN_UPDATE_HINTS: 2266 hints = (struct ena_admin_ena_hw_hints *) 2267 (&aenq_e->inline_data_w4); 2268 ena_update_hints(adapter, hints); 2269 break; 2270 default: 2271 RTE_LOG(ERR, PMD, "Invalid aenq notification link state %d\n", 2272 aenq_e->aenq_common_desc.syndrom); 2273 } 2274 } 2275 2276 static void ena_keep_alive(void *adapter_data, 2277 __rte_unused struct ena_admin_aenq_entry *aenq_e) 2278 { 2279 struct ena_adapter *adapter = adapter_data; 2280 2281 adapter->timestamp_wd = rte_get_timer_cycles(); 2282 } 2283 2284 /** 2285 * This handler will called for unknown event group or unimplemented handlers 2286 **/ 2287 static void unimplemented_aenq_handler(__rte_unused void *data, 2288 __rte_unused struct ena_admin_aenq_entry *aenq_e) 2289 { 2290 RTE_LOG(ERR, PMD, "Unknown event was received or event with " 2291 "unimplemented handler\n"); 2292 } 2293 2294 static struct ena_aenq_handlers aenq_handlers = { 2295 .handlers = { 2296 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 2297 [ENA_ADMIN_NOTIFICATION] = ena_notification, 2298 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive 2299 }, 2300 .unimplemented_handler = unimplemented_aenq_handler 2301 }; 2302