1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <rte_cycles.h> 6 #include <rte_bus_pci.h> 7 #include <rte_string_fns.h> 8 #include <rte_rawdev_pmd.h> 9 10 #include "rte_ioat_rawdev.h" 11 12 static struct rte_pci_driver ioat_pmd_drv; 13 14 #define IOAT_VENDOR_ID 0x8086 15 #define IOAT_DEVICE_ID_SKX 0x2021 16 #define IOAT_DEVICE_ID_BDX0 0x6f20 17 #define IOAT_DEVICE_ID_BDX1 0x6f21 18 #define IOAT_DEVICE_ID_BDX2 0x6f22 19 #define IOAT_DEVICE_ID_BDX3 0x6f23 20 #define IOAT_DEVICE_ID_BDX4 0x6f24 21 #define IOAT_DEVICE_ID_BDX5 0x6f25 22 #define IOAT_DEVICE_ID_BDX6 0x6f26 23 #define IOAT_DEVICE_ID_BDX7 0x6f27 24 #define IOAT_DEVICE_ID_BDXE 0x6f2E 25 #define IOAT_DEVICE_ID_BDXF 0x6f2F 26 #define IOAT_DEVICE_ID_ICX 0x0b00 27 28 RTE_LOG_REGISTER(ioat_pmd_logtype, rawdev.ioat, INFO); 29 30 #define IOAT_PMD_LOG(level, fmt, args...) rte_log(RTE_LOG_ ## level, \ 31 ioat_pmd_logtype, "%s(): " fmt "\n", __func__, ##args) 32 33 #define IOAT_PMD_DEBUG(fmt, args...) IOAT_PMD_LOG(DEBUG, fmt, ## args) 34 #define IOAT_PMD_INFO(fmt, args...) IOAT_PMD_LOG(INFO, fmt, ## args) 35 #define IOAT_PMD_ERR(fmt, args...) IOAT_PMD_LOG(ERR, fmt, ## args) 36 #define IOAT_PMD_WARN(fmt, args...) IOAT_PMD_LOG(WARNING, fmt, ## args) 37 38 #define DESC_SZ sizeof(struct rte_ioat_generic_hw_desc) 39 #define COMPLETION_SZ sizeof(__m128i) 40 41 static int 42 ioat_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) 43 { 44 struct rte_ioat_rawdev_config *params = config; 45 struct rte_ioat_rawdev *ioat = dev->dev_private; 46 char mz_name[RTE_MEMZONE_NAMESIZE]; 47 unsigned short i; 48 49 if (dev->started) 50 return -EBUSY; 51 52 if (params == NULL) 53 return -EINVAL; 54 55 if (params->ring_size > 4096 || params->ring_size < 64 || 56 !rte_is_power_of_2(params->ring_size)) 57 return -EINVAL; 58 59 ioat->ring_size = params->ring_size; 60 if (ioat->desc_ring != NULL) { 61 rte_memzone_free(ioat->desc_mz); 62 ioat->desc_ring = NULL; 63 ioat->desc_mz = NULL; 64 } 65 66 /* allocate one block of memory for both descriptors 67 * and completion handles. 68 */ 69 snprintf(mz_name, sizeof(mz_name), "rawdev%u_desc_ring", dev->dev_id); 70 ioat->desc_mz = rte_memzone_reserve(mz_name, 71 (DESC_SZ + COMPLETION_SZ) * ioat->ring_size, 72 dev->device->numa_node, RTE_MEMZONE_IOVA_CONTIG); 73 if (ioat->desc_mz == NULL) 74 return -ENOMEM; 75 ioat->desc_ring = ioat->desc_mz->addr; 76 ioat->hdls = (void *)&ioat->desc_ring[ioat->ring_size]; 77 78 ioat->ring_addr = ioat->desc_mz->iova; 79 80 /* configure descriptor ring - each one points to next */ 81 for (i = 0; i < ioat->ring_size; i++) { 82 ioat->desc_ring[i].next = ioat->ring_addr + 83 (((i + 1) % ioat->ring_size) * DESC_SZ); 84 } 85 86 return 0; 87 } 88 89 static int 90 ioat_dev_start(struct rte_rawdev *dev) 91 { 92 struct rte_ioat_rawdev *ioat = dev->dev_private; 93 94 if (ioat->ring_size == 0 || ioat->desc_ring == NULL) 95 return -EBUSY; 96 97 /* inform hardware of where the descriptor ring is */ 98 ioat->regs->chainaddr = ioat->ring_addr; 99 /* inform hardware of where to write the status/completions */ 100 ioat->regs->chancmp = ioat->status_addr; 101 102 /* prime the status register to be set to the last element */ 103 ioat->status = ioat->ring_addr + ((ioat->ring_size - 1) * DESC_SZ); 104 return 0; 105 } 106 107 static void 108 ioat_dev_stop(struct rte_rawdev *dev) 109 { 110 RTE_SET_USED(dev); 111 } 112 113 static void 114 ioat_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info) 115 { 116 struct rte_ioat_rawdev_config *cfg = dev_info; 117 struct rte_ioat_rawdev *ioat = dev->dev_private; 118 119 if (cfg != NULL) 120 cfg->ring_size = ioat->ring_size; 121 } 122 123 static const char * const xstat_names[] = { 124 "failed_enqueues", "successful_enqueues", 125 "copies_started", "copies_completed" 126 }; 127 128 static int 129 ioat_xstats_get(const struct rte_rawdev *dev, const unsigned int ids[], 130 uint64_t values[], unsigned int n) 131 { 132 const struct rte_ioat_rawdev *ioat = dev->dev_private; 133 unsigned int i; 134 135 for (i = 0; i < n; i++) { 136 switch (ids[i]) { 137 case 0: values[i] = ioat->enqueue_failed; break; 138 case 1: values[i] = ioat->enqueued; break; 139 case 2: values[i] = ioat->started; break; 140 case 3: values[i] = ioat->completed; break; 141 default: values[i] = 0; break; 142 } 143 } 144 return n; 145 } 146 147 static int 148 ioat_xstats_get_names(const struct rte_rawdev *dev, 149 struct rte_rawdev_xstats_name *names, 150 unsigned int size) 151 { 152 unsigned int i; 153 154 RTE_SET_USED(dev); 155 if (size < RTE_DIM(xstat_names)) 156 return RTE_DIM(xstat_names); 157 158 for (i = 0; i < RTE_DIM(xstat_names); i++) 159 strlcpy(names[i].name, xstat_names[i], sizeof(names[i])); 160 161 return RTE_DIM(xstat_names); 162 } 163 164 static int 165 ioat_xstats_reset(struct rte_rawdev *dev, const uint32_t *ids, uint32_t nb_ids) 166 { 167 struct rte_ioat_rawdev *ioat = dev->dev_private; 168 unsigned int i; 169 170 if (!ids) { 171 ioat->enqueue_failed = 0; 172 ioat->enqueued = 0; 173 ioat->started = 0; 174 ioat->completed = 0; 175 return 0; 176 } 177 178 for (i = 0; i < nb_ids; i++) { 179 switch (ids[i]) { 180 case 0: 181 ioat->enqueue_failed = 0; 182 break; 183 case 1: 184 ioat->enqueued = 0; 185 break; 186 case 2: 187 ioat->started = 0; 188 break; 189 case 3: 190 ioat->completed = 0; 191 break; 192 default: 193 IOAT_PMD_WARN("Invalid xstat id - cannot reset value"); 194 break; 195 } 196 } 197 198 return 0; 199 } 200 201 extern int ioat_rawdev_test(uint16_t dev_id); 202 203 static int 204 ioat_rawdev_create(const char *name, struct rte_pci_device *dev) 205 { 206 static const struct rte_rawdev_ops ioat_rawdev_ops = { 207 .dev_configure = ioat_dev_configure, 208 .dev_start = ioat_dev_start, 209 .dev_stop = ioat_dev_stop, 210 .dev_info_get = ioat_dev_info_get, 211 .xstats_get = ioat_xstats_get, 212 .xstats_get_names = ioat_xstats_get_names, 213 .xstats_reset = ioat_xstats_reset, 214 .dev_selftest = ioat_rawdev_test, 215 }; 216 217 struct rte_rawdev *rawdev = NULL; 218 struct rte_ioat_rawdev *ioat = NULL; 219 const struct rte_memzone *mz = NULL; 220 char mz_name[RTE_MEMZONE_NAMESIZE]; 221 int ret = 0; 222 int retry = 0; 223 224 if (!name) { 225 IOAT_PMD_ERR("Invalid name of the device!"); 226 ret = -EINVAL; 227 goto cleanup; 228 } 229 230 /* Allocate device structure */ 231 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct rte_ioat_rawdev), 232 dev->device.numa_node); 233 if (rawdev == NULL) { 234 IOAT_PMD_ERR("Unable to allocate raw device"); 235 ret = -ENOMEM; 236 goto cleanup; 237 } 238 239 snprintf(mz_name, sizeof(mz_name), "rawdev%u_private", rawdev->dev_id); 240 mz = rte_memzone_reserve(mz_name, sizeof(struct rte_ioat_rawdev), 241 dev->device.numa_node, RTE_MEMZONE_IOVA_CONTIG); 242 if (mz == NULL) { 243 IOAT_PMD_ERR("Unable to reserve memzone for private data\n"); 244 ret = -ENOMEM; 245 goto cleanup; 246 } 247 248 rawdev->dev_private = mz->addr; 249 rawdev->dev_ops = &ioat_rawdev_ops; 250 rawdev->device = &dev->device; 251 rawdev->driver_name = dev->device.driver->name; 252 253 ioat = rawdev->dev_private; 254 ioat->rawdev = rawdev; 255 ioat->mz = mz; 256 ioat->regs = dev->mem_resource[0].addr; 257 ioat->ring_size = 0; 258 ioat->desc_ring = NULL; 259 ioat->status_addr = ioat->mz->iova + 260 offsetof(struct rte_ioat_rawdev, status); 261 262 /* do device initialization - reset and set error behaviour */ 263 if (ioat->regs->chancnt != 1) 264 IOAT_PMD_ERR("%s: Channel count == %d\n", __func__, 265 ioat->regs->chancnt); 266 267 if (ioat->regs->chanctrl & 0x100) { /* locked by someone else */ 268 IOAT_PMD_WARN("%s: Channel appears locked\n", __func__); 269 ioat->regs->chanctrl = 0; 270 } 271 272 ioat->regs->chancmd = RTE_IOAT_CHANCMD_SUSPEND; 273 rte_delay_ms(1); 274 ioat->regs->chancmd = RTE_IOAT_CHANCMD_RESET; 275 rte_delay_ms(1); 276 while (ioat->regs->chancmd & RTE_IOAT_CHANCMD_RESET) { 277 ioat->regs->chainaddr = 0; 278 rte_delay_ms(1); 279 if (++retry >= 200) { 280 IOAT_PMD_ERR("%s: cannot reset device. CHANCMD=0x%"PRIx8", CHANSTS=0x%"PRIx64", CHANERR=0x%"PRIx32"\n", 281 __func__, 282 ioat->regs->chancmd, 283 ioat->regs->chansts, 284 ioat->regs->chanerr); 285 ret = -EIO; 286 } 287 } 288 ioat->regs->chanctrl = RTE_IOAT_CHANCTRL_ANY_ERR_ABORT_EN | 289 RTE_IOAT_CHANCTRL_ERR_COMPLETION_EN; 290 291 return 0; 292 293 cleanup: 294 if (rawdev) 295 rte_rawdev_pmd_release(rawdev); 296 297 return ret; 298 } 299 300 static int 301 ioat_rawdev_destroy(const char *name) 302 { 303 int ret; 304 struct rte_rawdev *rdev; 305 306 if (!name) { 307 IOAT_PMD_ERR("Invalid device name"); 308 return -EINVAL; 309 } 310 311 rdev = rte_rawdev_pmd_get_named_dev(name); 312 if (!rdev) { 313 IOAT_PMD_ERR("Invalid device name (%s)", name); 314 return -EINVAL; 315 } 316 317 if (rdev->dev_private != NULL) { 318 struct rte_ioat_rawdev *ioat = rdev->dev_private; 319 rdev->dev_private = NULL; 320 rte_memzone_free(ioat->desc_mz); 321 rte_memzone_free(ioat->mz); 322 } 323 324 /* rte_rawdev_close is called by pmd_release */ 325 ret = rte_rawdev_pmd_release(rdev); 326 if (ret) 327 IOAT_PMD_DEBUG("Device cleanup failed"); 328 329 return 0; 330 } 331 332 static int 333 ioat_rawdev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev) 334 { 335 char name[32]; 336 int ret = 0; 337 338 339 rte_pci_device_name(&dev->addr, name, sizeof(name)); 340 IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node); 341 342 dev->device.driver = &drv->driver; 343 ret = ioat_rawdev_create(name, dev); 344 return ret; 345 } 346 347 static int 348 ioat_rawdev_remove(struct rte_pci_device *dev) 349 { 350 char name[32]; 351 int ret; 352 353 rte_pci_device_name(&dev->addr, name, sizeof(name)); 354 355 IOAT_PMD_INFO("Closing %s on NUMA node %d", 356 name, dev->device.numa_node); 357 358 ret = ioat_rawdev_destroy(name); 359 return ret; 360 } 361 362 static const struct rte_pci_id pci_id_ioat_map[] = { 363 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_SKX) }, 364 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX0) }, 365 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX1) }, 366 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX2) }, 367 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX3) }, 368 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX4) }, 369 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX5) }, 370 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX6) }, 371 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX7) }, 372 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXE) }, 373 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXF) }, 374 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_ICX) }, 375 { .vendor_id = 0, /* sentinel */ }, 376 }; 377 378 static struct rte_pci_driver ioat_pmd_drv = { 379 .id_table = pci_id_ioat_map, 380 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 381 .probe = ioat_rawdev_probe, 382 .remove = ioat_rawdev_remove, 383 }; 384 385 RTE_PMD_REGISTER_PCI(IOAT_PMD_RAWDEV_NAME, ioat_pmd_drv); 386 RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_RAWDEV_NAME, pci_id_ioat_map); 387 RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_RAWDEV_NAME, "* igb_uio | uio_pci_generic"); 388