1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <rte_cycles.h> 6 #include <rte_bus_pci.h> 7 #include <rte_memzone.h> 8 #include <rte_string_fns.h> 9 #include <rte_rawdev_pmd.h> 10 11 #include "rte_ioat_rawdev.h" 12 #include "ioat_spec.h" 13 #include "ioat_private.h" 14 15 static struct rte_pci_driver ioat_pmd_drv; 16 17 #define IOAT_VENDOR_ID 0x8086 18 #define IOAT_DEVICE_ID_SKX 0x2021 19 #define IOAT_DEVICE_ID_BDX0 0x6f20 20 #define IOAT_DEVICE_ID_BDX1 0x6f21 21 #define IOAT_DEVICE_ID_BDX2 0x6f22 22 #define IOAT_DEVICE_ID_BDX3 0x6f23 23 #define IOAT_DEVICE_ID_BDX4 0x6f24 24 #define IOAT_DEVICE_ID_BDX5 0x6f25 25 #define IOAT_DEVICE_ID_BDX6 0x6f26 26 #define IOAT_DEVICE_ID_BDX7 0x6f27 27 #define IOAT_DEVICE_ID_BDXE 0x6f2E 28 #define IOAT_DEVICE_ID_BDXF 0x6f2F 29 #define IOAT_DEVICE_ID_ICX 0x0b00 30 31 RTE_LOG_REGISTER_DEFAULT(ioat_pmd_logtype, INFO); 32 33 #define DESC_SZ sizeof(struct rte_ioat_generic_hw_desc) 34 #define COMPLETION_SZ sizeof(__m128i) 35 36 static int 37 ioat_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config, 38 size_t config_size) 39 { 40 struct rte_ioat_rawdev_config *params = config; 41 struct rte_ioat_rawdev *ioat = dev->dev_private; 42 char mz_name[RTE_MEMZONE_NAMESIZE]; 43 unsigned short i; 44 45 if (dev->started) 46 return -EBUSY; 47 48 if (params == NULL || config_size != sizeof(*params)) 49 return -EINVAL; 50 51 if (params->ring_size > 4096 || params->ring_size < 64 || 52 !rte_is_power_of_2(params->ring_size)) 53 return -EINVAL; 54 55 ioat->ring_size = params->ring_size; 56 ioat->hdls_disable = params->hdls_disable; 57 if (ioat->desc_ring != NULL) { 58 rte_memzone_free(ioat->desc_mz); 59 ioat->desc_ring = NULL; 60 ioat->desc_mz = NULL; 61 } 62 63 /* allocate one block of memory for both descriptors 64 * and completion handles. 65 */ 66 snprintf(mz_name, sizeof(mz_name), "rawdev%u_desc_ring", dev->dev_id); 67 ioat->desc_mz = rte_memzone_reserve(mz_name, 68 (DESC_SZ + COMPLETION_SZ) * ioat->ring_size, 69 dev->device->numa_node, RTE_MEMZONE_IOVA_CONTIG); 70 if (ioat->desc_mz == NULL) 71 return -ENOMEM; 72 ioat->desc_ring = ioat->desc_mz->addr; 73 ioat->hdls = (void *)&ioat->desc_ring[ioat->ring_size]; 74 75 ioat->ring_addr = ioat->desc_mz->iova; 76 77 /* configure descriptor ring - each one points to next */ 78 for (i = 0; i < ioat->ring_size; i++) { 79 ioat->desc_ring[i].next = ioat->ring_addr + 80 (((i + 1) % ioat->ring_size) * DESC_SZ); 81 } 82 83 return 0; 84 } 85 86 static int 87 ioat_dev_start(struct rte_rawdev *dev) 88 { 89 struct rte_ioat_rawdev *ioat = dev->dev_private; 90 91 if (ioat->ring_size == 0 || ioat->desc_ring == NULL) 92 return -EBUSY; 93 94 /* inform hardware of where the descriptor ring is */ 95 ioat->regs->chainaddr = ioat->ring_addr; 96 /* inform hardware of where to write the status/completions */ 97 ioat->regs->chancmp = ioat->status_addr; 98 99 /* prime the status register to be set to the last element */ 100 ioat->status = ioat->ring_addr + ((ioat->ring_size - 1) * DESC_SZ); 101 return 0; 102 } 103 104 static void 105 ioat_dev_stop(struct rte_rawdev *dev) 106 { 107 RTE_SET_USED(dev); 108 } 109 110 static int 111 ioat_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info, 112 size_t dev_info_size) 113 { 114 struct rte_ioat_rawdev_config *cfg = dev_info; 115 struct rte_ioat_rawdev *ioat = dev->dev_private; 116 117 if (dev_info == NULL || dev_info_size != sizeof(*cfg)) 118 return -EINVAL; 119 120 cfg->ring_size = ioat->ring_size; 121 cfg->hdls_disable = ioat->hdls_disable; 122 return 0; 123 } 124 125 static int 126 ioat_dev_close(struct rte_rawdev *dev __rte_unused) 127 { 128 return 0; 129 } 130 131 static int 132 ioat_rawdev_create(const char *name, struct rte_pci_device *dev) 133 { 134 static const struct rte_rawdev_ops ioat_rawdev_ops = { 135 .dev_configure = ioat_dev_configure, 136 .dev_start = ioat_dev_start, 137 .dev_stop = ioat_dev_stop, 138 .dev_close = ioat_dev_close, 139 .dev_info_get = ioat_dev_info_get, 140 .xstats_get = ioat_xstats_get, 141 .xstats_get_names = ioat_xstats_get_names, 142 .xstats_reset = ioat_xstats_reset, 143 .dev_selftest = ioat_rawdev_test, 144 }; 145 146 struct rte_rawdev *rawdev = NULL; 147 struct rte_ioat_rawdev *ioat = NULL; 148 const struct rte_memzone *mz = NULL; 149 char mz_name[RTE_MEMZONE_NAMESIZE]; 150 int ret = 0; 151 int retry = 0; 152 153 if (!name) { 154 IOAT_PMD_ERR("Invalid name of the device!"); 155 ret = -EINVAL; 156 goto cleanup; 157 } 158 159 /* Allocate device structure */ 160 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct rte_ioat_rawdev), 161 dev->device.numa_node); 162 if (rawdev == NULL) { 163 IOAT_PMD_ERR("Unable to allocate raw device"); 164 ret = -ENOMEM; 165 goto cleanup; 166 } 167 168 /* Allocate memory for the primary process or else return the memory 169 * of primary memzone for the secondary process. 170 */ 171 snprintf(mz_name, sizeof(mz_name), "rawdev%u_private", rawdev->dev_id); 172 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 173 mz = rte_memzone_lookup(mz_name); 174 if (mz == NULL) { 175 IOAT_PMD_ERR("Unable lookup memzone for private data\n"); 176 ret = -ENOMEM; 177 goto cleanup; 178 } 179 rawdev->dev_private = mz->addr; 180 rawdev->dev_ops = &ioat_rawdev_ops; 181 rawdev->device = &dev->device; 182 rawdev->driver_name = dev->device.driver->name; 183 return 0; 184 } 185 mz = rte_memzone_reserve(mz_name, sizeof(struct rte_ioat_rawdev), 186 dev->device.numa_node, RTE_MEMZONE_IOVA_CONTIG); 187 if (mz == NULL) { 188 IOAT_PMD_ERR("Unable to reserve memzone for private data\n"); 189 ret = -ENOMEM; 190 goto cleanup; 191 } 192 193 rawdev->dev_private = mz->addr; 194 rawdev->dev_ops = &ioat_rawdev_ops; 195 rawdev->device = &dev->device; 196 rawdev->driver_name = dev->device.driver->name; 197 198 ioat = rawdev->dev_private; 199 ioat->type = RTE_IOAT_DEV; 200 ioat->rawdev = rawdev; 201 ioat->mz = mz; 202 ioat->regs = dev->mem_resource[0].addr; 203 ioat->doorbell = &ioat->regs->dmacount; 204 ioat->ring_size = 0; 205 ioat->desc_ring = NULL; 206 ioat->status_addr = ioat->mz->iova + 207 offsetof(struct rte_ioat_rawdev, status); 208 209 /* do device initialization - reset and set error behaviour */ 210 if (ioat->regs->chancnt != 1) 211 IOAT_PMD_ERR("%s: Channel count == %d\n", __func__, 212 ioat->regs->chancnt); 213 214 if (ioat->regs->chanctrl & 0x100) { /* locked by someone else */ 215 IOAT_PMD_WARN("%s: Channel appears locked\n", __func__); 216 ioat->regs->chanctrl = 0; 217 } 218 219 ioat->regs->chancmd = RTE_IOAT_CHANCMD_SUSPEND; 220 rte_delay_ms(1); 221 ioat->regs->chancmd = RTE_IOAT_CHANCMD_RESET; 222 rte_delay_ms(1); 223 while (ioat->regs->chancmd & RTE_IOAT_CHANCMD_RESET) { 224 ioat->regs->chainaddr = 0; 225 rte_delay_ms(1); 226 if (++retry >= 200) { 227 IOAT_PMD_ERR("%s: cannot reset device. CHANCMD=0x%"PRIx8", CHANSTS=0x%"PRIx64", CHANERR=0x%"PRIx32"\n", 228 __func__, 229 ioat->regs->chancmd, 230 ioat->regs->chansts, 231 ioat->regs->chanerr); 232 ret = -EIO; 233 } 234 } 235 ioat->regs->chanctrl = RTE_IOAT_CHANCTRL_ANY_ERR_ABORT_EN | 236 RTE_IOAT_CHANCTRL_ERR_COMPLETION_EN; 237 238 return 0; 239 240 cleanup: 241 if (rawdev) 242 rte_rawdev_pmd_release(rawdev); 243 244 return ret; 245 } 246 247 static int 248 ioat_rawdev_destroy(const char *name) 249 { 250 int ret; 251 struct rte_rawdev *rdev; 252 253 if (!name) { 254 IOAT_PMD_ERR("Invalid device name"); 255 return -EINVAL; 256 } 257 258 rdev = rte_rawdev_pmd_get_named_dev(name); 259 if (!rdev) { 260 IOAT_PMD_ERR("Invalid device name (%s)", name); 261 return -EINVAL; 262 } 263 264 if (rdev->dev_private != NULL) { 265 struct rte_ioat_rawdev *ioat = rdev->dev_private; 266 rdev->dev_private = NULL; 267 rte_memzone_free(ioat->desc_mz); 268 rte_memzone_free(ioat->mz); 269 } 270 271 /* rte_rawdev_close is called by pmd_release */ 272 ret = rte_rawdev_pmd_release(rdev); 273 if (ret) 274 IOAT_PMD_DEBUG("Device cleanup failed"); 275 276 return 0; 277 } 278 279 static int 280 ioat_rawdev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev) 281 { 282 char name[32]; 283 int ret = 0; 284 285 286 rte_pci_device_name(&dev->addr, name, sizeof(name)); 287 IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node); 288 289 dev->device.driver = &drv->driver; 290 ret = ioat_rawdev_create(name, dev); 291 return ret; 292 } 293 294 static int 295 ioat_rawdev_remove(struct rte_pci_device *dev) 296 { 297 char name[32]; 298 int ret; 299 300 rte_pci_device_name(&dev->addr, name, sizeof(name)); 301 302 IOAT_PMD_INFO("Closing %s on NUMA node %d", 303 name, dev->device.numa_node); 304 305 ret = ioat_rawdev_destroy(name); 306 return ret; 307 } 308 309 static const struct rte_pci_id pci_id_ioat_map[] = { 310 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_SKX) }, 311 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX0) }, 312 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX1) }, 313 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX2) }, 314 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX3) }, 315 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX4) }, 316 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX5) }, 317 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX6) }, 318 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX7) }, 319 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXE) }, 320 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXF) }, 321 { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_ICX) }, 322 { .vendor_id = 0, /* sentinel */ }, 323 }; 324 325 static struct rte_pci_driver ioat_pmd_drv = { 326 .id_table = pci_id_ioat_map, 327 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 328 .probe = ioat_rawdev_probe, 329 .remove = ioat_rawdev_remove, 330 }; 331 332 RTE_PMD_REGISTER_PCI(IOAT_PMD_RAWDEV_NAME, ioat_pmd_drv); 333 RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_RAWDEV_NAME, pci_id_ioat_map); 334 RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_RAWDEV_NAME, "* igb_uio | uio_pci_generic"); 335