1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 NXP. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of NXP nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 /* System headers */ 33 #include <stdio.h> 34 #include <inttypes.h> 35 #include <unistd.h> 36 #include <limits.h> 37 #include <sched.h> 38 #include <signal.h> 39 #include <pthread.h> 40 #include <sys/types.h> 41 #include <sys/syscall.h> 42 43 #include <rte_byteorder.h> 44 #include <rte_common.h> 45 #include <rte_interrupts.h> 46 #include <rte_log.h> 47 #include <rte_debug.h> 48 #include <rte_pci.h> 49 #include <rte_atomic.h> 50 #include <rte_branch_prediction.h> 51 #include <rte_memory.h> 52 #include <rte_tailq.h> 53 #include <rte_eal.h> 54 #include <rte_alarm.h> 55 #include <rte_ether.h> 56 #include <rte_ethdev.h> 57 #include <rte_malloc.h> 58 #include <rte_ring.h> 59 #include <rte_bus.h> 60 61 #include <rte_dpaa_bus.h> 62 #include <rte_dpaa_logs.h> 63 64 #include <fsl_usd.h> 65 #include <fsl_qman.h> 66 #include <fsl_bman.h> 67 #include <of.h> 68 #include <netcfg.h> 69 70 int dpaa_logtype_bus; 71 int dpaa_logtype_mempool; 72 int dpaa_logtype_pmd; 73 74 struct rte_dpaa_bus rte_dpaa_bus; 75 struct netcfg_info *dpaa_netcfg; 76 77 /* define a variable to hold the portal_key, once created.*/ 78 pthread_key_t dpaa_portal_key; 79 80 RTE_DEFINE_PER_LCORE(bool, _dpaa_io); 81 82 static inline void 83 dpaa_add_to_device_list(struct rte_dpaa_device *dev) 84 { 85 TAILQ_INSERT_TAIL(&rte_dpaa_bus.device_list, dev, next); 86 } 87 88 static inline void 89 dpaa_remove_from_device_list(struct rte_dpaa_device *dev) 90 { 91 TAILQ_INSERT_TAIL(&rte_dpaa_bus.device_list, dev, next); 92 } 93 94 /* 95 * Reads the SEC device from DTS 96 * Returns -1 if SEC devices not available, 0 otherwise 97 */ 98 static inline int 99 dpaa_sec_available(void) 100 { 101 const struct device_node *caam_node; 102 103 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { 104 return 0; 105 } 106 107 return -1; 108 } 109 110 static void dpaa_clean_device_list(void); 111 112 static int 113 dpaa_create_device_list(void) 114 { 115 int i; 116 int ret; 117 struct rte_dpaa_device *dev; 118 struct fm_eth_port_cfg *cfg; 119 struct fman_if *fman_intf; 120 121 /* Creating Ethernet Devices */ 122 for (i = 0; i < dpaa_netcfg->num_ethports; i++) { 123 dev = calloc(1, sizeof(struct rte_dpaa_device)); 124 if (!dev) { 125 DPAA_BUS_LOG(ERR, "Failed to allocate ETH devices"); 126 ret = -ENOMEM; 127 goto cleanup; 128 } 129 130 cfg = &dpaa_netcfg->port_cfg[i]; 131 fman_intf = cfg->fman_if; 132 133 /* Device identifiers */ 134 dev->id.fman_id = fman_intf->fman_idx + 1; 135 dev->id.mac_id = fman_intf->mac_idx; 136 dev->device_type = FSL_DPAA_ETH; 137 dev->id.dev_id = i; 138 139 /* Create device name */ 140 memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN); 141 sprintf(dev->name, "fm%d-mac%d", (fman_intf->fman_idx + 1), 142 fman_intf->mac_idx); 143 DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name); 144 dev->device.name = dev->name; 145 146 dpaa_add_to_device_list(dev); 147 } 148 149 rte_dpaa_bus.device_count = i; 150 151 /* Unlike case of ETH, RTE_LIBRTE_DPAA_MAX_CRYPTODEV SEC devices are 152 * constantly created only if "sec" property is found in the device 153 * tree. Logically there is no limit for number of devices (QI 154 * interfaces) that can be created. 155 */ 156 157 if (dpaa_sec_available()) { 158 DPAA_BUS_LOG(INFO, "DPAA SEC devices are not available"); 159 return 0; 160 } 161 162 /* Creating SEC Devices */ 163 for (i = 0; i < RTE_LIBRTE_DPAA_MAX_CRYPTODEV; i++) { 164 dev = calloc(1, sizeof(struct rte_dpaa_device)); 165 if (!dev) { 166 DPAA_BUS_LOG(ERR, "Failed to allocate SEC devices"); 167 ret = -1; 168 goto cleanup; 169 } 170 171 dev->device_type = FSL_DPAA_CRYPTO; 172 dev->id.dev_id = rte_dpaa_bus.device_count + i; 173 174 /* Even though RTE_CRYPTODEV_NAME_MAX_LEN is valid length of 175 * crypto PMD, using RTE_ETH_NAME_MAX_LEN as that is the size 176 * allocated for dev->name/ 177 */ 178 memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN); 179 sprintf(dev->name, "dpaa-sec%d", i); 180 DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name); 181 182 dpaa_add_to_device_list(dev); 183 } 184 185 rte_dpaa_bus.device_count += i; 186 187 return 0; 188 189 cleanup: 190 dpaa_clean_device_list(); 191 return ret; 192 } 193 194 static void 195 dpaa_clean_device_list(void) 196 { 197 struct rte_dpaa_device *dev = NULL; 198 struct rte_dpaa_device *tdev = NULL; 199 200 TAILQ_FOREACH_SAFE(dev, &rte_dpaa_bus.device_list, next, tdev) { 201 TAILQ_REMOVE(&rte_dpaa_bus.device_list, dev, next); 202 free(dev); 203 dev = NULL; 204 } 205 } 206 207 /** XXX move this function into a separate file */ 208 static int 209 _dpaa_portal_init(void *arg) 210 { 211 cpu_set_t cpuset; 212 pthread_t id; 213 uint32_t cpu = rte_lcore_id(); 214 int ret; 215 struct dpaa_portal *dpaa_io_portal; 216 217 BUS_INIT_FUNC_TRACE(); 218 219 if ((uint64_t)arg == 1 || cpu == LCORE_ID_ANY) 220 cpu = rte_get_master_lcore(); 221 /* if the core id is not supported */ 222 else 223 if (cpu >= RTE_MAX_LCORE) 224 return -1; 225 226 /* Set CPU affinity for this thread */ 227 CPU_ZERO(&cpuset); 228 CPU_SET(cpu, &cpuset); 229 id = pthread_self(); 230 ret = pthread_setaffinity_np(id, sizeof(cpu_set_t), &cpuset); 231 if (ret) { 232 DPAA_BUS_LOG(ERR, "pthread_setaffinity_np failed on " 233 "core :%d with ret: %d", cpu, ret); 234 return ret; 235 } 236 237 /* Initialise bman thread portals */ 238 ret = bman_thread_init(); 239 if (ret) { 240 DPAA_BUS_LOG(ERR, "bman_thread_init failed on " 241 "core %d with ret: %d", cpu, ret); 242 return ret; 243 } 244 245 DPAA_BUS_LOG(DEBUG, "BMAN thread initialized"); 246 247 /* Initialise qman thread portals */ 248 ret = qman_thread_init(); 249 if (ret) { 250 DPAA_BUS_LOG(ERR, "bman_thread_init failed on " 251 "core %d with ret: %d", cpu, ret); 252 bman_thread_finish(); 253 return ret; 254 } 255 256 DPAA_BUS_LOG(DEBUG, "QMAN thread initialized"); 257 258 dpaa_io_portal = rte_malloc(NULL, sizeof(struct dpaa_portal), 259 RTE_CACHE_LINE_SIZE); 260 if (!dpaa_io_portal) { 261 DPAA_BUS_LOG(ERR, "Unable to allocate memory"); 262 bman_thread_finish(); 263 qman_thread_finish(); 264 return -ENOMEM; 265 } 266 267 dpaa_io_portal->qman_idx = qman_get_portal_index(); 268 dpaa_io_portal->bman_idx = bman_get_portal_index(); 269 dpaa_io_portal->tid = syscall(SYS_gettid); 270 271 ret = pthread_setspecific(dpaa_portal_key, (void *)dpaa_io_portal); 272 if (ret) { 273 DPAA_BUS_LOG(ERR, "pthread_setspecific failed on " 274 "core %d with ret: %d", cpu, ret); 275 dpaa_portal_finish(NULL); 276 277 return ret; 278 } 279 280 RTE_PER_LCORE(_dpaa_io) = true; 281 282 DPAA_BUS_LOG(DEBUG, "QMAN thread initialized"); 283 284 return 0; 285 } 286 287 /* 288 * rte_dpaa_portal_init - Wrapper over _dpaa_portal_init with thread level check 289 * XXX Complete this 290 */ 291 int 292 rte_dpaa_portal_init(void *arg) 293 { 294 if (unlikely(!RTE_PER_LCORE(_dpaa_io))) 295 return _dpaa_portal_init(arg); 296 297 return 0; 298 } 299 300 void 301 dpaa_portal_finish(void *arg) 302 { 303 struct dpaa_portal *dpaa_io_portal = (struct dpaa_portal *)arg; 304 305 if (!dpaa_io_portal) { 306 DPAA_BUS_LOG(DEBUG, "Portal already cleaned"); 307 return; 308 } 309 310 bman_thread_finish(); 311 qman_thread_finish(); 312 313 pthread_setspecific(dpaa_portal_key, NULL); 314 315 rte_free(dpaa_io_portal); 316 dpaa_io_portal = NULL; 317 318 RTE_PER_LCORE(_dpaa_io) = false; 319 } 320 321 #define DPAA_DEV_PATH1 "/sys/devices/platform/soc/soc:fsl,dpaa" 322 #define DPAA_DEV_PATH2 "/sys/devices/platform/fsl,dpaa" 323 324 static int 325 rte_dpaa_bus_scan(void) 326 { 327 int ret; 328 329 BUS_INIT_FUNC_TRACE(); 330 331 if ((access(DPAA_DEV_PATH1, F_OK) != 0) && 332 (access(DPAA_DEV_PATH2, F_OK) != 0)) { 333 RTE_LOG(DEBUG, EAL, "DPAA Bus not present. Skipping.\n"); 334 return 0; 335 } 336 337 /* Load the device-tree driver */ 338 ret = of_init(); 339 if (ret) { 340 DPAA_BUS_LOG(ERR, "of_init failed with ret: %d", ret); 341 return -1; 342 } 343 344 /* Get the interface configurations from device-tree */ 345 dpaa_netcfg = netcfg_acquire(); 346 if (!dpaa_netcfg) { 347 DPAA_BUS_LOG(ERR, "netcfg_acquire failed"); 348 return -EINVAL; 349 } 350 351 RTE_LOG(NOTICE, EAL, "DPAA Bus Detected\n"); 352 353 if (!dpaa_netcfg->num_ethports) { 354 DPAA_BUS_LOG(INFO, "no network interfaces available"); 355 /* This is not an error */ 356 return 0; 357 } 358 359 DPAA_BUS_LOG(DEBUG, "Bus: Address of netcfg=%p, Ethports=%d", 360 dpaa_netcfg, dpaa_netcfg->num_ethports); 361 362 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER 363 dump_netcfg(dpaa_netcfg); 364 #endif 365 366 DPAA_BUS_LOG(DEBUG, "Number of devices = %d\n", 367 dpaa_netcfg->num_ethports); 368 ret = dpaa_create_device_list(); 369 if (ret) { 370 DPAA_BUS_LOG(ERR, "Unable to create device list. (%d)", ret); 371 return ret; 372 } 373 374 /* create the key, supplying a function that'll be invoked 375 * when a portal affined thread will be deleted. 376 */ 377 ret = pthread_key_create(&dpaa_portal_key, dpaa_portal_finish); 378 if (ret) { 379 DPAA_BUS_LOG(DEBUG, "Unable to create pthread key. (%d)", ret); 380 dpaa_clean_device_list(); 381 return ret; 382 } 383 384 DPAA_BUS_LOG(DEBUG, "dpaa_portal_key=%u, ret=%d\n", 385 (unsigned int)dpaa_portal_key, ret); 386 387 return 0; 388 } 389 390 /* register a dpaa bus based dpaa driver */ 391 void 392 rte_dpaa_driver_register(struct rte_dpaa_driver *driver) 393 { 394 RTE_VERIFY(driver); 395 396 BUS_INIT_FUNC_TRACE(); 397 398 TAILQ_INSERT_TAIL(&rte_dpaa_bus.driver_list, driver, next); 399 /* Update Bus references */ 400 driver->dpaa_bus = &rte_dpaa_bus; 401 } 402 403 /* un-register a dpaa bus based dpaa driver */ 404 void 405 rte_dpaa_driver_unregister(struct rte_dpaa_driver *driver) 406 { 407 struct rte_dpaa_bus *dpaa_bus; 408 409 BUS_INIT_FUNC_TRACE(); 410 411 dpaa_bus = driver->dpaa_bus; 412 413 TAILQ_REMOVE(&dpaa_bus->driver_list, driver, next); 414 /* Update Bus references */ 415 driver->dpaa_bus = NULL; 416 } 417 418 static int 419 rte_dpaa_device_match(struct rte_dpaa_driver *drv, 420 struct rte_dpaa_device *dev) 421 { 422 int ret = -1; 423 424 BUS_INIT_FUNC_TRACE(); 425 426 if (!drv || !dev) { 427 DPAA_BUS_DEBUG("Invalid drv or dev received."); 428 return ret; 429 } 430 431 if (drv->drv_type == dev->device_type) { 432 DPAA_BUS_INFO("Device: %s matches for driver: %s", 433 dev->name, drv->driver.name); 434 ret = 0; /* Found a match */ 435 } 436 437 return ret; 438 } 439 440 static int 441 rte_dpaa_bus_probe(void) 442 { 443 int ret = -1; 444 struct rte_dpaa_device *dev; 445 struct rte_dpaa_driver *drv; 446 447 BUS_INIT_FUNC_TRACE(); 448 449 /* For each registered driver, and device, call the driver->probe */ 450 TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) { 451 TAILQ_FOREACH(drv, &rte_dpaa_bus.driver_list, next) { 452 ret = rte_dpaa_device_match(drv, dev); 453 if (ret) 454 continue; 455 456 if (!drv->probe) 457 continue; 458 459 ret = drv->probe(drv, dev); 460 if (ret) 461 DPAA_BUS_ERR("Unable to probe.\n"); 462 break; 463 } 464 } 465 return 0; 466 } 467 468 static struct rte_device * 469 rte_dpaa_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, 470 const void *data) 471 { 472 struct rte_dpaa_device *dev; 473 474 TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) { 475 if (start && &dev->device == start) { 476 start = NULL; /* starting point found */ 477 continue; 478 } 479 480 if (cmp(&dev->device, data) == 0) 481 return &dev->device; 482 } 483 484 return NULL; 485 } 486 487 /* 488 * Get iommu class of DPAA2 devices on the bus. 489 */ 490 static enum rte_iova_mode 491 rte_dpaa_get_iommu_class(void) 492 { 493 if ((access(DPAA_DEV_PATH1, F_OK) != 0) && 494 (access(DPAA_DEV_PATH2, F_OK) != 0)) { 495 return RTE_IOVA_DC; 496 } 497 return RTE_IOVA_PA; 498 } 499 500 struct rte_dpaa_bus rte_dpaa_bus = { 501 .bus = { 502 .scan = rte_dpaa_bus_scan, 503 .probe = rte_dpaa_bus_probe, 504 .find_device = rte_dpaa_find_device, 505 .get_iommu_class = rte_dpaa_get_iommu_class, 506 }, 507 .device_list = TAILQ_HEAD_INITIALIZER(rte_dpaa_bus.device_list), 508 .driver_list = TAILQ_HEAD_INITIALIZER(rte_dpaa_bus.driver_list), 509 .device_count = 0, 510 }; 511 512 RTE_REGISTER_BUS(FSL_DPAA_BUS_NAME, rte_dpaa_bus.bus); 513 514 RTE_INIT(dpaa_init_log); 515 static void 516 dpaa_init_log(void) 517 { 518 dpaa_logtype_bus = rte_log_register("bus.dpaa"); 519 if (dpaa_logtype_bus >= 0) 520 rte_log_set_level(dpaa_logtype_bus, RTE_LOG_NOTICE); 521 522 dpaa_logtype_mempool = rte_log_register("mempool.dpaa"); 523 if (dpaa_logtype_mempool >= 0) 524 rte_log_set_level(dpaa_logtype_mempool, RTE_LOG_NOTICE); 525 526 dpaa_logtype_pmd = rte_log_register("pmd.dpaa"); 527 if (dpaa_logtype_pmd >= 0) 528 rte_log_set_level(dpaa_logtype_pmd, RTE_LOG_NOTICE); 529 } 530