1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include "cnxk_eventdev.h" 6 #include "cnxk_tim_evdev.h" 7 8 static struct event_timer_adapter_ops cnxk_tim_ops; 9 10 static int 11 cnxk_tim_chnk_pool_create(struct cnxk_tim_ring *tim_ring, 12 struct rte_event_timer_adapter_conf *rcfg) 13 { 14 unsigned int cache_sz = (tim_ring->nb_chunks / 1.5); 15 unsigned int mp_flags = 0; 16 char pool_name[25]; 17 int rc; 18 19 cache_sz /= rte_lcore_count(); 20 /* Create chunk pool. */ 21 if (rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT) { 22 mp_flags = RTE_MEMPOOL_F_SP_PUT | RTE_MEMPOOL_F_SC_GET; 23 plt_tim_dbg("Using single producer mode"); 24 tim_ring->prod_type_sp = true; 25 } 26 27 snprintf(pool_name, sizeof(pool_name), "cnxk_tim_chunk_pool%d", 28 tim_ring->ring_id); 29 30 if (cache_sz > CNXK_TIM_MAX_POOL_CACHE_SZ) 31 cache_sz = CNXK_TIM_MAX_POOL_CACHE_SZ; 32 cache_sz = cache_sz != 0 ? cache_sz : 2; 33 tim_ring->nb_chunks += (cache_sz * rte_lcore_count()); 34 if (!tim_ring->disable_npa) { 35 tim_ring->chunk_pool = rte_mempool_create_empty( 36 pool_name, tim_ring->nb_chunks, tim_ring->chunk_sz, 37 cache_sz, 0, rte_socket_id(), mp_flags); 38 39 if (tim_ring->chunk_pool == NULL) { 40 plt_err("Unable to create chunkpool."); 41 return -ENOMEM; 42 } 43 44 rc = rte_mempool_set_ops_byname(tim_ring->chunk_pool, 45 rte_mbuf_platform_mempool_ops(), 46 NULL); 47 if (rc < 0) { 48 plt_err("Unable to set chunkpool ops"); 49 goto free; 50 } 51 52 rc = rte_mempool_populate_default(tim_ring->chunk_pool); 53 if (rc < 0) { 54 plt_err("Unable to set populate chunkpool."); 55 goto free; 56 } 57 tim_ring->aura = roc_npa_aura_handle_to_aura( 58 tim_ring->chunk_pool->pool_id); 59 tim_ring->ena_dfb = 0; 60 } else { 61 tim_ring->chunk_pool = rte_mempool_create( 62 pool_name, tim_ring->nb_chunks, tim_ring->chunk_sz, 63 cache_sz, 0, NULL, NULL, NULL, NULL, rte_socket_id(), 64 mp_flags); 65 if (tim_ring->chunk_pool == NULL) { 66 plt_err("Unable to create chunkpool."); 67 return -ENOMEM; 68 } 69 tim_ring->ena_dfb = 1; 70 } 71 72 return 0; 73 74 free: 75 rte_mempool_free(tim_ring->chunk_pool); 76 return rc; 77 } 78 79 static void 80 cnxk_tim_set_fp_ops(struct cnxk_tim_ring *tim_ring) 81 { 82 uint8_t prod_flag = !tim_ring->prod_type_sp; 83 84 /* [STATS] [DFB/FB] [SP][MP]*/ 85 const rte_event_timer_arm_burst_t arm_burst[2][2][2] = { 86 #define FP(_name, _f3, _f2, _f1, flags) \ 87 [_f3][_f2][_f1] = cnxk_tim_arm_burst_##_name, 88 TIM_ARM_FASTPATH_MODES 89 #undef FP 90 }; 91 92 const rte_event_timer_arm_tmo_tick_burst_t arm_tmo_burst[2][2] = { 93 #define FP(_name, _f2, _f1, flags) \ 94 [_f2][_f1] = cnxk_tim_arm_tmo_tick_burst_##_name, 95 TIM_ARM_TMO_FASTPATH_MODES 96 #undef FP 97 }; 98 99 cnxk_tim_ops.arm_burst = 100 arm_burst[tim_ring->enable_stats][tim_ring->ena_dfb][prod_flag]; 101 cnxk_tim_ops.arm_tmo_tick_burst = 102 arm_tmo_burst[tim_ring->enable_stats][tim_ring->ena_dfb]; 103 cnxk_tim_ops.cancel_burst = cnxk_tim_timer_cancel_burst; 104 } 105 106 static void 107 cnxk_tim_ring_info_get(const struct rte_event_timer_adapter *adptr, 108 struct rte_event_timer_adapter_info *adptr_info) 109 { 110 struct cnxk_tim_ring *tim_ring = adptr->data->adapter_priv; 111 112 adptr_info->max_tmo_ns = tim_ring->max_tout; 113 adptr_info->min_resolution_ns = tim_ring->tck_nsec; 114 rte_memcpy(&adptr_info->conf, &adptr->data->conf, 115 sizeof(struct rte_event_timer_adapter_conf)); 116 } 117 118 static int 119 cnxk_tim_ring_create(struct rte_event_timer_adapter *adptr) 120 { 121 struct rte_event_timer_adapter_conf *rcfg = &adptr->data->conf; 122 struct cnxk_tim_evdev *dev = cnxk_tim_priv_get(); 123 struct cnxk_tim_ring *tim_ring; 124 int i, rc; 125 126 if (dev == NULL) 127 return -ENODEV; 128 129 if (adptr->data->id >= dev->nb_rings) 130 return -ENODEV; 131 132 tim_ring = rte_zmalloc("cnxk_tim_prv", sizeof(struct cnxk_tim_ring), 0); 133 if (tim_ring == NULL) 134 return -ENOMEM; 135 136 rc = roc_tim_lf_alloc(&dev->tim, adptr->data->id, NULL); 137 if (rc < 0) { 138 plt_err("Failed to create timer ring"); 139 goto tim_ring_free; 140 } 141 142 if (NSEC2TICK(RTE_ALIGN_MUL_CEIL( 143 rcfg->timer_tick_ns, 144 cnxk_tim_min_resolution_ns(cnxk_tim_cntfrq())), 145 cnxk_tim_cntfrq()) < 146 cnxk_tim_min_tmo_ticks(cnxk_tim_cntfrq())) { 147 if (rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES) 148 rcfg->timer_tick_ns = TICK2NSEC( 149 cnxk_tim_min_tmo_ticks(cnxk_tim_cntfrq()), 150 cnxk_tim_cntfrq()); 151 else { 152 rc = -ERANGE; 153 goto tim_hw_free; 154 } 155 } 156 tim_ring->ring_id = adptr->data->id; 157 tim_ring->clk_src = (int)rcfg->clk_src; 158 tim_ring->tck_nsec = RTE_ALIGN_MUL_CEIL( 159 rcfg->timer_tick_ns, 160 cnxk_tim_min_resolution_ns(cnxk_tim_cntfrq())); 161 tim_ring->max_tout = rcfg->max_tmo_ns; 162 tim_ring->nb_bkts = (tim_ring->max_tout / tim_ring->tck_nsec); 163 tim_ring->nb_timers = rcfg->nb_timers; 164 tim_ring->chunk_sz = dev->chunk_sz; 165 tim_ring->disable_npa = dev->disable_npa; 166 tim_ring->enable_stats = dev->enable_stats; 167 168 for (i = 0; i < dev->ring_ctl_cnt; i++) { 169 struct cnxk_tim_ctl *ring_ctl = &dev->ring_ctl_data[i]; 170 171 if (ring_ctl->ring == tim_ring->ring_id) { 172 tim_ring->chunk_sz = 173 ring_ctl->chunk_slots ? 174 ((uint32_t)(ring_ctl->chunk_slots + 1) * 175 CNXK_TIM_CHUNK_ALIGNMENT) : 176 tim_ring->chunk_sz; 177 tim_ring->enable_stats = ring_ctl->enable_stats; 178 tim_ring->disable_npa = ring_ctl->disable_npa; 179 } 180 } 181 182 if (tim_ring->disable_npa) { 183 tim_ring->nb_chunks = 184 tim_ring->nb_timers / 185 CNXK_TIM_NB_CHUNK_SLOTS(tim_ring->chunk_sz); 186 tim_ring->nb_chunks = tim_ring->nb_chunks * tim_ring->nb_bkts; 187 } else { 188 tim_ring->nb_chunks = tim_ring->nb_timers; 189 } 190 191 tim_ring->nb_chunk_slots = CNXK_TIM_NB_CHUNK_SLOTS(tim_ring->chunk_sz); 192 /* Create buckets. */ 193 tim_ring->bkt = 194 rte_zmalloc("cnxk_tim_bucket", 195 (tim_ring->nb_bkts) * sizeof(struct cnxk_tim_bkt), 196 RTE_CACHE_LINE_SIZE); 197 if (tim_ring->bkt == NULL) 198 goto tim_hw_free; 199 200 rc = cnxk_tim_chnk_pool_create(tim_ring, rcfg); 201 if (rc < 0) 202 goto tim_bkt_free; 203 204 rc = roc_tim_lf_config( 205 &dev->tim, tim_ring->ring_id, 206 cnxk_tim_convert_clk_src(tim_ring->clk_src), 0, 0, 207 tim_ring->nb_bkts, tim_ring->chunk_sz, 208 NSEC2TICK(tim_ring->tck_nsec, cnxk_tim_cntfrq())); 209 if (rc < 0) { 210 plt_err("Failed to configure timer ring"); 211 goto tim_chnk_free; 212 } 213 214 tim_ring->base = roc_tim_lf_base_get(&dev->tim, tim_ring->ring_id); 215 plt_write64((uint64_t)tim_ring->bkt, tim_ring->base + TIM_LF_RING_BASE); 216 plt_write64(tim_ring->aura, tim_ring->base + TIM_LF_RING_AURA); 217 218 /* Set fastpath ops. */ 219 cnxk_tim_set_fp_ops(tim_ring); 220 221 /* Update SSO xae count. */ 222 cnxk_sso_updt_xae_cnt(cnxk_sso_pmd_priv(dev->event_dev), tim_ring, 223 RTE_EVENT_TYPE_TIMER); 224 cnxk_sso_xae_reconfigure(dev->event_dev); 225 226 plt_tim_dbg( 227 "Total memory used %" PRIu64 "MB\n", 228 (uint64_t)(((tim_ring->nb_chunks * tim_ring->chunk_sz) + 229 (tim_ring->nb_bkts * sizeof(struct cnxk_tim_bkt))) / 230 BIT_ULL(20))); 231 232 adptr->data->adapter_priv = tim_ring; 233 return rc; 234 235 tim_chnk_free: 236 rte_mempool_free(tim_ring->chunk_pool); 237 tim_bkt_free: 238 rte_free(tim_ring->bkt); 239 tim_hw_free: 240 roc_tim_lf_free(&dev->tim, tim_ring->ring_id); 241 tim_ring_free: 242 rte_free(tim_ring); 243 return rc; 244 } 245 246 static int 247 cnxk_tim_ring_free(struct rte_event_timer_adapter *adptr) 248 { 249 struct cnxk_tim_ring *tim_ring = adptr->data->adapter_priv; 250 struct cnxk_tim_evdev *dev = cnxk_tim_priv_get(); 251 252 if (dev == NULL) 253 return -ENODEV; 254 255 roc_tim_lf_free(&dev->tim, tim_ring->ring_id); 256 rte_free(tim_ring->bkt); 257 rte_mempool_free(tim_ring->chunk_pool); 258 rte_free(tim_ring); 259 260 return 0; 261 } 262 263 static void 264 cnxk_tim_calibrate_start_tsc(struct cnxk_tim_ring *tim_ring) 265 { 266 #define CNXK_TIM_CALIB_ITER 1E6 267 uint32_t real_bkt, bucket; 268 int icount, ecount = 0; 269 uint64_t bkt_cyc; 270 271 for (icount = 0; icount < CNXK_TIM_CALIB_ITER; icount++) { 272 real_bkt = plt_read64(tim_ring->base + TIM_LF_RING_REL) >> 44; 273 bkt_cyc = cnxk_tim_cntvct(); 274 bucket = (bkt_cyc - tim_ring->ring_start_cyc) / 275 tim_ring->tck_int; 276 bucket = bucket % (tim_ring->nb_bkts); 277 tim_ring->ring_start_cyc = 278 bkt_cyc - (real_bkt * tim_ring->tck_int); 279 if (bucket != real_bkt) 280 ecount++; 281 } 282 tim_ring->last_updt_cyc = bkt_cyc; 283 plt_tim_dbg("Bucket mispredict %3.2f distance %d\n", 284 100 - (((double)(icount - ecount) / (double)icount) * 100), 285 bucket - real_bkt); 286 } 287 288 static int 289 cnxk_tim_ring_start(const struct rte_event_timer_adapter *adptr) 290 { 291 struct cnxk_tim_ring *tim_ring = adptr->data->adapter_priv; 292 struct cnxk_tim_evdev *dev = cnxk_tim_priv_get(); 293 int rc; 294 295 if (dev == NULL) 296 return -ENODEV; 297 298 rc = roc_tim_lf_enable(&dev->tim, tim_ring->ring_id, 299 &tim_ring->ring_start_cyc, NULL); 300 if (rc < 0) 301 return rc; 302 303 tim_ring->tck_int = NSEC2TICK(tim_ring->tck_nsec, cnxk_tim_cntfrq()); 304 tim_ring->tot_int = tim_ring->tck_int * tim_ring->nb_bkts; 305 tim_ring->fast_div = rte_reciprocal_value_u64(tim_ring->tck_int); 306 tim_ring->fast_bkt = rte_reciprocal_value_u64(tim_ring->nb_bkts); 307 308 cnxk_tim_calibrate_start_tsc(tim_ring); 309 310 return rc; 311 } 312 313 static int 314 cnxk_tim_ring_stop(const struct rte_event_timer_adapter *adptr) 315 { 316 struct cnxk_tim_ring *tim_ring = adptr->data->adapter_priv; 317 struct cnxk_tim_evdev *dev = cnxk_tim_priv_get(); 318 int rc; 319 320 if (dev == NULL) 321 return -ENODEV; 322 323 rc = roc_tim_lf_disable(&dev->tim, tim_ring->ring_id); 324 if (rc < 0) 325 plt_err("Failed to disable timer ring"); 326 327 return rc; 328 } 329 330 static int 331 cnxk_tim_stats_get(const struct rte_event_timer_adapter *adapter, 332 struct rte_event_timer_adapter_stats *stats) 333 { 334 struct cnxk_tim_ring *tim_ring = adapter->data->adapter_priv; 335 uint64_t bkt_cyc = cnxk_tim_cntvct() - tim_ring->ring_start_cyc; 336 337 stats->evtim_exp_count = 338 __atomic_load_n(&tim_ring->arm_cnt, __ATOMIC_RELAXED); 339 stats->ev_enq_count = stats->evtim_exp_count; 340 stats->adapter_tick_count = 341 rte_reciprocal_divide_u64(bkt_cyc, &tim_ring->fast_div); 342 return 0; 343 } 344 345 static int 346 cnxk_tim_stats_reset(const struct rte_event_timer_adapter *adapter) 347 { 348 struct cnxk_tim_ring *tim_ring = adapter->data->adapter_priv; 349 350 __atomic_store_n(&tim_ring->arm_cnt, 0, __ATOMIC_RELAXED); 351 return 0; 352 } 353 354 int 355 cnxk_tim_caps_get(const struct rte_eventdev *evdev, uint64_t flags, 356 uint32_t *caps, const struct event_timer_adapter_ops **ops) 357 { 358 struct cnxk_tim_evdev *dev = cnxk_tim_priv_get(); 359 360 RTE_SET_USED(flags); 361 362 if (dev == NULL) 363 return -ENODEV; 364 365 cnxk_tim_ops.init = cnxk_tim_ring_create; 366 cnxk_tim_ops.uninit = cnxk_tim_ring_free; 367 cnxk_tim_ops.start = cnxk_tim_ring_start; 368 cnxk_tim_ops.stop = cnxk_tim_ring_stop; 369 cnxk_tim_ops.get_info = cnxk_tim_ring_info_get; 370 371 if (dev->enable_stats) { 372 cnxk_tim_ops.stats_get = cnxk_tim_stats_get; 373 cnxk_tim_ops.stats_reset = cnxk_tim_stats_reset; 374 } 375 376 /* Store evdev pointer for later use. */ 377 dev->event_dev = (struct rte_eventdev *)(uintptr_t)evdev; 378 *caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT; 379 *ops = &cnxk_tim_ops; 380 381 return 0; 382 } 383 384 static void 385 cnxk_tim_parse_ring_param(char *value, void *opaque) 386 { 387 struct cnxk_tim_evdev *dev = opaque; 388 struct cnxk_tim_ctl ring_ctl = {0}; 389 char *tok = strtok(value, "-"); 390 struct cnxk_tim_ctl *old_ptr; 391 uint16_t *val; 392 393 val = (uint16_t *)&ring_ctl; 394 395 if (!strlen(value)) 396 return; 397 398 while (tok != NULL) { 399 *val = atoi(tok); 400 tok = strtok(NULL, "-"); 401 val++; 402 } 403 404 if (val != (&ring_ctl.enable_stats + 1)) { 405 plt_err("Invalid ring param expected [ring-chunk_sz-disable_npa-enable_stats]"); 406 return; 407 } 408 409 dev->ring_ctl_cnt++; 410 old_ptr = dev->ring_ctl_data; 411 dev->ring_ctl_data = 412 rte_realloc(dev->ring_ctl_data, 413 sizeof(struct cnxk_tim_ctl) * dev->ring_ctl_cnt, 0); 414 if (dev->ring_ctl_data == NULL) { 415 dev->ring_ctl_data = old_ptr; 416 dev->ring_ctl_cnt--; 417 return; 418 } 419 420 dev->ring_ctl_data[dev->ring_ctl_cnt - 1] = ring_ctl; 421 } 422 423 static void 424 cnxk_tim_parse_ring_ctl_list(const char *value, void *opaque) 425 { 426 char *s = strdup(value); 427 char *start = NULL; 428 char *end = NULL; 429 char *f = s; 430 431 while (*s) { 432 if (*s == '[') 433 start = s; 434 else if (*s == ']') 435 end = s; 436 437 if (start && start < end) { 438 *end = 0; 439 cnxk_tim_parse_ring_param(start + 1, opaque); 440 start = end; 441 s = end; 442 } 443 s++; 444 } 445 446 free(f); 447 } 448 449 static int 450 cnxk_tim_parse_kvargs_dict(const char *key, const char *value, void *opaque) 451 { 452 RTE_SET_USED(key); 453 454 /* Dict format [ring-chunk_sz-disable_npa-enable_stats] use '-' as ',' 455 * isn't allowed. 0 represents default. 456 */ 457 cnxk_tim_parse_ring_ctl_list(value, opaque); 458 459 return 0; 460 } 461 462 static void 463 cnxk_tim_parse_devargs(struct rte_devargs *devargs, struct cnxk_tim_evdev *dev) 464 { 465 struct rte_kvargs *kvlist; 466 467 if (devargs == NULL) 468 return; 469 470 kvlist = rte_kvargs_parse(devargs->args, NULL); 471 if (kvlist == NULL) 472 return; 473 474 rte_kvargs_process(kvlist, CNXK_TIM_DISABLE_NPA, &parse_kvargs_flag, 475 &dev->disable_npa); 476 rte_kvargs_process(kvlist, CNXK_TIM_CHNK_SLOTS, &parse_kvargs_value, 477 &dev->chunk_slots); 478 rte_kvargs_process(kvlist, CNXK_TIM_STATS_ENA, &parse_kvargs_flag, 479 &dev->enable_stats); 480 rte_kvargs_process(kvlist, CNXK_TIM_RINGS_LMT, &parse_kvargs_value, 481 &dev->min_ring_cnt); 482 rte_kvargs_process(kvlist, CNXK_TIM_RING_CTL, 483 &cnxk_tim_parse_kvargs_dict, &dev); 484 485 rte_kvargs_free(kvlist); 486 } 487 488 void 489 cnxk_tim_init(struct roc_sso *sso) 490 { 491 const struct rte_memzone *mz; 492 struct cnxk_tim_evdev *dev; 493 int rc; 494 495 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 496 return; 497 498 mz = rte_memzone_reserve(RTE_STR(CNXK_TIM_EVDEV_NAME), 499 sizeof(struct cnxk_tim_evdev), 0, 0); 500 if (mz == NULL) { 501 plt_tim_dbg("Unable to allocate memory for TIM Event device"); 502 return; 503 } 504 dev = mz->addr; 505 506 cnxk_tim_parse_devargs(sso->pci_dev->device.devargs, dev); 507 508 dev->tim.roc_sso = sso; 509 dev->tim.nb_lfs = dev->min_ring_cnt; 510 rc = roc_tim_init(&dev->tim); 511 if (rc < 0) { 512 plt_err("Failed to initialize roc tim resources"); 513 rte_memzone_free(mz); 514 return; 515 } 516 dev->nb_rings = rc; 517 518 if (dev->chunk_slots && dev->chunk_slots <= CNXK_TIM_MAX_CHUNK_SLOTS && 519 dev->chunk_slots >= CNXK_TIM_MIN_CHUNK_SLOTS) { 520 dev->chunk_sz = 521 (dev->chunk_slots + 1) * CNXK_TIM_CHUNK_ALIGNMENT; 522 } else { 523 dev->chunk_sz = CNXK_TIM_RING_DEF_CHUNK_SZ; 524 } 525 } 526 527 void 528 cnxk_tim_fini(void) 529 { 530 struct cnxk_tim_evdev *dev = cnxk_tim_priv_get(); 531 532 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 533 return; 534 535 roc_tim_fini(&dev->tim); 536 rte_memzone_free(rte_memzone_lookup(RTE_STR(CNXK_TIM_EVDEV_NAME))); 537 } 538