1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <rte_cycles.h> 6 #include <rte_bus_pci.h> 7 #include <rte_string_fns.h> 8 #include <rte_rawdev_pmd.h> 9 10 #include "rte_ioat_rawdev.h" 11 12 static struct rte_pci_driver ioat_pmd_drv; 13 14 #define IOAT_VENDOR_ID 0x8086 15 #define IOAT_DEVICE_ID_SKX 0x2021 16 #define IOAT_DEVICE_ID_BDX0 0x6f20 17 #define IOAT_DEVICE_ID_BDX1 0x6f21 18 #define IOAT_DEVICE_ID_BDX2 0x6f22 19 #define IOAT_DEVICE_ID_BDX3 0x6f23 20 #define IOAT_DEVICE_ID_BDX4 0x6f24 21 #define IOAT_DEVICE_ID_BDX5 0x6f25 22 #define IOAT_DEVICE_ID_BDX6 0x6f26 23 #define IOAT_DEVICE_ID_BDX7 0x6f27 24 #define IOAT_DEVICE_ID_BDXE 0x6f2E 25 #define IOAT_DEVICE_ID_BDXF 0x6f2F 26 #define IOAT_DEVICE_ID_ICX 0x0b00 27 28 RTE_LOG_REGISTER(ioat_pmd_logtype, rawdev.ioat, INFO); 29 30 #define IOAT_PMD_LOG(level, fmt, args...) rte_log(RTE_LOG_ ## level, \ 31 ioat_pmd_logtype, "%s(): " fmt "\n", __func__, ##args) 32 33 #define IOAT_PMD_DEBUG(fmt, args...) IOAT_PMD_LOG(DEBUG, fmt, ## args) 34 #define IOAT_PMD_INFO(fmt, args...) IOAT_PMD_LOG(INFO, fmt, ## args) 35 #define IOAT_PMD_ERR(fmt, args...) IOAT_PMD_LOG(ERR, fmt, ## args) 36 #define IOAT_PMD_WARN(fmt, args...) IOAT_PMD_LOG(WARNING, fmt, ## args) 37 38 #define DESC_SZ sizeof(struct rte_ioat_generic_hw_desc) 39 #define COMPLETION_SZ sizeof(__m128i) 40 41 static int 42 ioat_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config, 43 size_t config_size) 44 { 45 struct rte_ioat_rawdev_config *params = config; 46 struct rte_ioat_rawdev *ioat = dev->dev_private; 47 char mz_name[RTE_MEMZONE_NAMESIZE]; 48 unsigned short i; 49 50 if (dev->started) 51 return -EBUSY; 52 53 if (params == NULL || config_size != sizeof(*params)) 54 return -EINVAL; 55 56 if (params->ring_size > 4096 || params->ring_size < 64 || 57 !rte_is_power_of_2(params->ring_size)) 58 return -EINVAL; 59 60 ioat->ring_size = params->ring_size; 61 if (ioat->desc_ring != NULL) { 62 rte_memzone_free(ioat->desc_mz); 63 ioat->desc_ring = NULL; 64 ioat->desc_mz = NULL; 65 } 66 67 /* allocate one block of memory for both descriptors 68 * and completion handles. 69 */ 70 snprintf(mz_name, sizeof(mz_name), "rawdev%u_desc_ring", dev->dev_id); 71 ioat->desc_mz = rte_memzone_reserve(mz_name, 72 (DESC_SZ + COMPLETION_SZ) * ioat->ring_size, 73 dev->device->numa_node, RTE_MEMZONE_IOVA_CONTIG); 74 if (ioat->desc_mz == NULL) 75 return -ENOMEM; 76 ioat->desc_ring = ioat->desc_mz->addr; 77 ioat->hdls = (void *)&ioat->desc_ring[ioat->ring_size]; 78 79 ioat->ring_addr = ioat->desc_mz->iova; 80 81 /* configure descriptor ring - each one points to next */ 82 for (i = 0; i < ioat->ring_size; i++) { 83 ioat->desc_ring[i].next = ioat->ring_addr + 84 (((i + 1) % ioat->ring_size) * DESC_SZ); 85 } 86 87 return 0; 88 } 89 90 static int 91 ioat_dev_start(struct rte_rawdev *dev) 92 { 93 struct rte_ioat_rawdev *ioat = dev->dev_private; 94 95 if (ioat->ring_size == 0 || ioat->desc_ring == NULL) 96 return -EBUSY; 97 98 /* inform hardware of where the descriptor ring is */ 99 ioat->regs->chainaddr = ioat->ring_addr; 100 /* inform hardware of where to write the status/completions */ 101 ioat->regs->chancmp = ioat->status_addr; 102 103 /* prime the status register to be set to the last element */ 104 ioat->status = ioat->ring_addr + ((ioat->ring_size - 1) * DESC_SZ); 105 return 0; 106 } 107 108 static void 109 ioat_dev_stop(struct rte_rawdev *dev) 110 { 111 RTE_SET_USED(dev); 112 } 113 114 static int 115 ioat_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info, 116 size_t dev_info_size) 117 { 118 struct rte_ioat_rawdev_config *cfg = dev_info; 119 struct rte_ioat_rawdev *ioat = dev->dev_private; 120 121 if (dev_info == NULL || dev_info_size != sizeof(*cfg)) 122 return -EINVAL; 123 124 cfg->ring_size = ioat->ring_size; 125 return 0; 126 } 127 128 static const char * const xstat_names[] = { 129 "failed_enqueues", "successful_enqueues", 130 "copies_started", "copies_completed" 131 }; 132 133 static int 134 ioat_xstats_get(const struct rte_rawdev *dev, const unsigned int ids[], 135 uint64_t values[], unsigned int n) 136 { 137 const struct rte_ioat_rawdev *ioat = dev->dev_private; 138 unsigned int i; 139 140 for (i = 0; i < n; i++) { 141 switch (ids[i]) { 142 case 0: values[i] = ioat->enqueue_failed; break; 143 case 1: values[i] = ioat->enqueued; break; 144 case 2: values[i] = ioat->started; break; 145 case 3: values[i] = ioat->completed; break; 146 default: values[i] = 0; break; 147 } 148 } 149 return n; 150 } 151 152 static int 153 ioat_xstats_get_names(const struct rte_rawdev *dev, 154 struct rte_rawdev_xstats_name *names, 155 unsigned int size) 156 { 157 unsigned int i; 158 159 RTE_SET_USED(dev); 160 if (size < RTE_DIM(xstat_names)) 161 return RTE_DIM(xstat_names); 162 163 for (i = 0; i < RTE_DIM(xstat_names); i++) 164 strlcpy(names[i].name, xstat_names[i], sizeof(names[i])); 165 166 return RTE_DIM(xstat_names); 167 } 168 169 static int 170 ioat_xstats_reset(struct rte_rawdev *dev, const uint32_t *ids, uint32_t nb_ids) 171 { 172 struct rte_ioat_rawdev *ioat = dev->dev_private; 173 unsigned int i; 174 175 if (!ids) { 176 ioat->enqueue_failed = 0; 177 ioat->enqueued = 0; 178 ioat->started = 0; 179 ioat->completed = 0; 180 return 0; 181 } 182 183 for (i = 0; i < nb_ids; i++) { 184 switch (ids[i]) { 185 case 0: 186 ioat->enqueue_failed = 0; 187 break; 188 case 1: 189 ioat->enqueued = 0; 190 break; 191 case 2: 192 ioat->started = 0; 193 break; 194 case 3: 195 ioat->completed = 0; 196 break; 197 default: 198 IOAT_PMD_WARN("Invalid xstat id - cannot reset value"); 199 break; 200 } 201 } 202 203 return 0; 204 } 205 206 extern int ioat_rawdev_test(uint16_t dev_id); 207 208 static int 209 ioat_rawdev_create(const char *name, struct rte_pci_device *dev) 210 { 211 static const struct rte_rawdev_ops ioat_rawdev_ops = { 212 .dev_configure = ioat_dev_configure, 213 .dev_start = ioat_dev_start, 214 .dev_stop = ioat_dev_stop, 215 .dev_info_get = ioat_dev_info_get, 216 .xstats_get = ioat_xstats_get, 217 .xstats_get_names = ioat_xstats_get_names, 218 .xstats_reset = ioat_xstats_reset, 219 .dev_selftest = ioat_rawdev_test, 220 }; 221 222 struct rte_rawdev *rawdev = NULL; 223 struct rte_ioat_rawdev *ioat = NULL; 224 const struct rte_memzone *mz = NULL; 225 char mz_name[RTE_MEMZONE_NAMESIZE]; 226 int ret = 0; 227 int retry = 0; 228 229 if (!name) { 230 IOAT_PMD_ERR("Invalid name of the device!"); 231 ret = -EINVAL; 232 goto cleanup; 233 } 234 235 /* Allocate device structure */ 236 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct rte_ioat_rawdev), 237 dev->device.numa_node); 238 if (rawdev == NULL) { 239 IOAT_PMD_ERR("Unable to allocate raw device"); 240 ret = -ENOMEM; 241 goto cleanup; 242 } 243 244 snprintf(mz_name, sizeof(mz_name), "rawdev%u_private", rawdev->dev_id); 245 mz = rte_memzone_reserve(mz_name, sizeof(struct rte_ioat_rawdev), 246 dev->device.numa_node, RTE_MEMZONE_IOVA_CONTIG); 247 if (mz == NULL) { 248 IOAT_PMD_ERR("Unable to reserve memzone for private data\n"); 249 ret = -ENOMEM; 250 goto cleanup; 251 } 252 253 rawdev->dev_private = mz->addr; 254 rawdev->dev_ops = &ioat_rawdev_ops; 255 rawdev->device = &dev->device; 256 rawdev->driver_name = dev->device.driver->name; 257 258 ioat = rawdev->dev_private; 259 ioat->rawdev = rawdev; 260 ioat->mz = mz; 261 ioat->regs = dev->mem_resource[0].addr; 262 ioat->ring_size = 0; 263 ioat->desc_ring = NULL; 264 ioat->status_addr = ioat->mz->iova + 265 offsetof(struct rte_ioat_rawdev, status); 266 267 /* do device initialization - reset and set error behaviour */ 268 if (ioat->regs->chancnt != 1) 269 IOAT_PMD_ERR("%s: Channel count == %d\n", __func__, 270 ioat->regs->chancnt); 271 272 if (ioat->regs->chanctrl & 0x100) { /* locked by someone else */ 273 IOAT_PMD_WARN("%s: Channel appears locked\n", __func__); 274 ioat->regs->chanctrl = 0; 275 } 276 277 ioat->regs->chancmd = RTE_IOAT_CHANCMD_SUSPEND; 278 rte_delay_ms(1); 279 ioat->regs->chancmd = RTE_IOAT_CHANCMD_RESET; 280 rte_delay_ms(1); 281 while (ioat->regs->chancmd & RTE_IOAT_CHANCMD_RESET) { 282 ioat->regs->chainaddr = 0; 283 rte_delay_ms(1); 284 if (++retry >= 200) { 285 IOAT_PMD_ERR("%s: cannot reset device. CHANCMD=0x%"PRIx8", CHANSTS=0x%"PRIx64", CHANERR=0x%"PRIx32"\n", 286 __func__, 287 ioat->regs->chancmd, 288 ioat->regs->chansts, 289 ioat->regs->chanerr); 290 ret = -EIO; 291 } 292 } 293 ioat->regs->chanctrl = RTE_IOAT_CHANCTRL_ANY_ERR_ABORT_EN | 294 RTE_IOAT_CHANCTRL_ERR_COMPLETION_EN; 295 296 return 0; 297 298 cleanup: 299 if (rawdev) 300 rte_rawdev_pmd_release(rawdev); 301 302 return ret; 303 } 304 305 static int 306 ioat_rawdev_destroy(const char *name) 307 { 308 int ret; 309 struct rte_rawdev *rdev; 310 311 if (!name) { 312 IOAT_PMD_ERR("Invalid device name"); 313 return -EINVAL; 314 } 315 316 rdev = rte_rawdev_pmd_get_named_dev(name); 317 if (!rdev) { 318 IOAT_PMD_ERR("Invalid device name (%s)", name); 319 return -EINVAL; 320 } 321 322 if (rdev->dev_private != NULL) { 323 struct rte_ioat_rawdev *ioat = rdev->dev_private; 324 rdev->dev_private = NULL; 325 rte_memzone_free(ioat->desc_mz); 326 rte_memzone_free(ioat->mz); 327 } 328 329 /* rte_rawdev_close is called by pmd_release */ 330 ret = rte_rawdev_pmd_release(rdev); 331 if (ret) 332 IOAT_PMD_DEBUG("Device cleanup failed"); 333 334 return 0; 335 } 336 337 static int 338 ioat_rawdev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev) 339 { 340 char name[32]; 341 int ret = 0; 342 343 344 rte_pci_device_name(&dev->addr, name, sizeof(name)); 345 IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node); 346 347 dev->device.driver = &drv->driver; 348 ret = ioat_rawdev_create(name, dev); 349 return ret; 350 } 351 352 static int 353 ioat_rawdev_remove(struct rte_pci_device *dev) 354 { 355 char name[32]; 356 int ret; 357 358 rte_pci_device_name(&dev->addr, name, sizeof(name)); 359 360 IOAT_PMD_INFO("Closing %s on NUMA node %d", 361 name, dev->device.numa_node); 362 363 ret = ioat_rawdev_destroy(name); 364 return ret; 365 } 366 367 static const struct rte_pci_id pci_id_ioat_map[] = { 368 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_SKX) }, 369 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX0) }, 370 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX1) }, 371 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX2) }, 372 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX3) }, 373 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX4) }, 374 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX5) }, 375 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX6) }, 376 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX7) }, 377 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXE) }, 378 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXF) }, 379 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_ICX) }, 380 { .vendor_id = 0, /* sentinel */ }, 381 }; 382 383 static struct rte_pci_driver ioat_pmd_drv = { 384 .id_table = pci_id_ioat_map, 385 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 386 .probe = ioat_rawdev_probe, 387 .remove = ioat_rawdev_remove, 388 }; 389 390 RTE_PMD_REGISTER_PCI(IOAT_PMD_RAWDEV_NAME, ioat_pmd_drv); 391 RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_RAWDEV_NAME, pci_id_ioat_map); 392 RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_RAWDEV_NAME, "* igb_uio | uio_pci_generic"); 393