1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Cavium, Inc
3 */
4
5 #include "ssovf_evdev.h"
6 #include "timvf_evdev.h"
7
8 RTE_LOG_REGISTER(otx_logtype_timvf, pmd.event.octeontx.timer, NOTICE);
9
10 static struct rte_eventdev *event_dev;
11
12 struct __rte_packed timvf_mbox_dev_info {
13 uint64_t ring_active[4];
14 uint64_t clk_freq;
15 };
16
17 /* Response messages */
18 enum {
19 MBOX_RET_SUCCESS,
20 MBOX_RET_INVALID,
21 MBOX_RET_INTERNAL_ERR,
22 };
23
24 static int
timvf_mbox_dev_info_get(struct timvf_mbox_dev_info * info)25 timvf_mbox_dev_info_get(struct timvf_mbox_dev_info *info)
26 {
27 struct octeontx_mbox_hdr hdr = {0};
28 uint16_t len = sizeof(struct timvf_mbox_dev_info);
29
30 hdr.coproc = TIM_COPROC;
31 hdr.msg = TIM_GET_DEV_INFO;
32 hdr.vfid = 0; /* TIM DEV is always 0. TIM RING ID changes. */
33
34 memset(info, 0, len);
35 return octeontx_mbox_send(&hdr, NULL, 0, info, len);
36 }
37
38 static void
timvf_ring_info_get(const struct rte_event_timer_adapter * adptr,struct rte_event_timer_adapter_info * adptr_info)39 timvf_ring_info_get(const struct rte_event_timer_adapter *adptr,
40 struct rte_event_timer_adapter_info *adptr_info)
41 {
42 struct timvf_ring *timr = adptr->data->adapter_priv;
43 adptr_info->max_tmo_ns = timr->max_tout;
44 adptr_info->min_resolution_ns = timr->tck_nsec;
45 rte_memcpy(&adptr_info->conf, &adptr->data->conf,
46 sizeof(struct rte_event_timer_adapter_conf));
47 }
48
49 static int
timvf_ring_conf_set(struct timvf_ctrl_reg * rctl,uint8_t ring_id)50 timvf_ring_conf_set(struct timvf_ctrl_reg *rctl, uint8_t ring_id)
51 {
52 struct octeontx_mbox_hdr hdr = {0};
53 uint16_t len = sizeof(struct timvf_ctrl_reg);
54 int ret;
55
56 hdr.coproc = TIM_COPROC;
57 hdr.msg = TIM_SET_RING_INFO;
58 hdr.vfid = ring_id;
59
60 ret = octeontx_mbox_send(&hdr, rctl, len, NULL, 0);
61 if (ret < 0 || hdr.res_code != MBOX_RET_SUCCESS)
62 return -EACCES;
63 return 0;
64 }
65
66 static int
timvf_get_start_cyc(uint64_t * now,uint8_t ring_id)67 timvf_get_start_cyc(uint64_t *now, uint8_t ring_id)
68 {
69 struct octeontx_mbox_hdr hdr = {0};
70
71 hdr.coproc = TIM_COPROC;
72 hdr.msg = TIM_RING_START_CYC_GET;
73 hdr.vfid = ring_id;
74 *now = 0;
75 return octeontx_mbox_send(&hdr, NULL, 0, now, sizeof(uint64_t));
76 }
77
78 static int
optimize_bucket_parameters(struct timvf_ring * timr)79 optimize_bucket_parameters(struct timvf_ring *timr)
80 {
81 uint32_t hbkts;
82 uint32_t lbkts;
83 uint64_t tck_nsec;
84
85 hbkts = rte_align32pow2(timr->nb_bkts);
86 tck_nsec = RTE_ALIGN_MUL_CEIL(timr->max_tout / (hbkts - 1), 10);
87
88 if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS))
89 hbkts = 0;
90
91 lbkts = rte_align32prevpow2(timr->nb_bkts);
92 tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout / (lbkts - 1)), 10);
93
94 if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS))
95 lbkts = 0;
96
97 if (!hbkts && !lbkts)
98 return 0;
99
100 if (!hbkts) {
101 timr->nb_bkts = lbkts;
102 goto end;
103 } else if (!lbkts) {
104 timr->nb_bkts = hbkts;
105 goto end;
106 }
107
108 timr->nb_bkts = (hbkts - timr->nb_bkts) <
109 (timr->nb_bkts - lbkts) ? hbkts : lbkts;
110 end:
111 timr->get_target_bkt = bkt_and;
112 timr->tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout /
113 (timr->nb_bkts - 1)), 10);
114 return 1;
115 }
116
117 static int
timvf_ring_start(const struct rte_event_timer_adapter * adptr)118 timvf_ring_start(const struct rte_event_timer_adapter *adptr)
119 {
120 int ret;
121 uint8_t use_fpa = 0;
122 uint64_t interval;
123 uintptr_t pool;
124 struct timvf_ctrl_reg rctrl;
125 struct timvf_mbox_dev_info dinfo;
126 struct timvf_ring *timr = adptr->data->adapter_priv;
127
128 ret = timvf_mbox_dev_info_get(&dinfo);
129 if (ret < 0 || ret != sizeof(struct timvf_mbox_dev_info))
130 return -EINVAL;
131
132 /* Calculate the interval cycles according to clock source. */
133 switch (timr->clk_src) {
134 case TIM_CLK_SRC_SCLK:
135 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
136 break;
137 case TIM_CLK_SRC_GPIO:
138 /* GPIO doesn't work on tck_nsec. */
139 interval = 0;
140 break;
141 case TIM_CLK_SRC_GTI:
142 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
143 break;
144 case TIM_CLK_SRC_PTP:
145 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
146 break;
147 default:
148 timvf_log_err("Unsupported clock source configured %d",
149 timr->clk_src);
150 return -EINVAL;
151 }
152
153 if (!strcmp(rte_mbuf_best_mempool_ops(), "octeontx_fpavf"))
154 use_fpa = 1;
155
156 /*CTRL0 register.*/
157 rctrl.rctrl0 = interval;
158
159 /*CTRL1 register.*/
160 rctrl.rctrl1 = (uint64_t)(timr->clk_src) << 51 |
161 1ull << 48 /* LOCK_EN (Enable hw bucket lock mechanism) */ |
162 1ull << 47 /* ENA */ |
163 1ull << 44 /* ENA_LDWB */ |
164 (timr->nb_bkts - 1);
165
166 rctrl.rctrl2 = (uint64_t)(TIM_CHUNK_SIZE / 16) << 40;
167
168 if (use_fpa) {
169 pool = (uintptr_t)((struct rte_mempool *)
170 timr->chunk_pool)->pool_id;
171 ret = octeontx_fpa_bufpool_gaura(pool);
172 if (ret < 0) {
173 timvf_log_dbg("Unable to get gaura id");
174 ret = -ENOMEM;
175 goto error;
176 }
177 timvf_write64((uint64_t)ret,
178 (uint8_t *)timr->vbar0 + TIM_VRING_AURA);
179 } else {
180 rctrl.rctrl1 |= 1ull << 43 /* ENA_DFB (Enable don't free) */;
181 }
182
183 timvf_write64((uintptr_t)timr->bkt,
184 (uint8_t *)timr->vbar0 + TIM_VRING_BASE);
185 timvf_set_chunk_refill(timr, use_fpa);
186 if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id)) {
187 ret = -EACCES;
188 goto error;
189 }
190
191 if (timvf_get_start_cyc(&timr->ring_start_cyc,
192 timr->tim_ring_id) < 0) {
193 ret = -EACCES;
194 goto error;
195 }
196 timr->tck_int = NSEC2CLK(timr->tck_nsec, rte_get_timer_hz());
197 timr->fast_div = rte_reciprocal_value_u64(timr->tck_int);
198 timvf_log_info("nb_bkts %d min_ns %"PRIu64" min_cyc %"PRIu64""
199 " maxtmo %"PRIu64"\n",
200 timr->nb_bkts, timr->tck_nsec, interval,
201 timr->max_tout);
202
203 return 0;
204 error:
205 rte_free(timr->bkt);
206 rte_mempool_free(timr->chunk_pool);
207 return ret;
208 }
209
210 static int
timvf_ring_stop(const struct rte_event_timer_adapter * adptr)211 timvf_ring_stop(const struct rte_event_timer_adapter *adptr)
212 {
213 struct timvf_ring *timr = adptr->data->adapter_priv;
214 struct timvf_ctrl_reg rctrl = {0};
215 rctrl.rctrl0 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL0);
216 rctrl.rctrl1 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL1);
217 rctrl.rctrl1 &= ~(1ull << 47); /* Disable */
218 rctrl.rctrl2 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL2);
219
220 if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id))
221 return -EACCES;
222 return 0;
223 }
224
225 static int
timvf_ring_create(struct rte_event_timer_adapter * adptr)226 timvf_ring_create(struct rte_event_timer_adapter *adptr)
227 {
228 struct rte_event_timer_adapter_conf *rcfg = &adptr->data->conf;
229 uint16_t free_idx = UINT16_MAX;
230 unsigned int mp_flags = 0;
231 struct ssovf_evdev *edev;
232 struct timvf_ring *timr;
233 const char *mempool_ops;
234 uint8_t tim_ring_id;
235 char pool_name[25];
236 int i, ret;
237
238 tim_ring_id = timvf_get_ring();
239 if (tim_ring_id == UINT8_MAX)
240 return -ENODEV;
241
242 edev = ssovf_pmd_priv(event_dev);
243 timr = rte_zmalloc("octeontx_timvf_priv",
244 sizeof(struct timvf_ring), 0);
245 if (timr == NULL)
246 return -ENOMEM;
247
248 adptr->data->adapter_priv = timr;
249 /* Check config parameters. */
250 if ((rcfg->clk_src != RTE_EVENT_TIMER_ADAPTER_CPU_CLK) &&
251 (!rcfg->timer_tick_ns ||
252 rcfg->timer_tick_ns < TIM_MIN_INTERVAL)) {
253 timvf_log_err("Too low timer ticks");
254 goto cfg_err;
255 }
256
257 timr->clk_src = (int) rcfg->clk_src;
258 timr->tim_ring_id = tim_ring_id;
259 timr->tck_nsec = RTE_ALIGN_MUL_CEIL(rcfg->timer_tick_ns, 10);
260 timr->max_tout = rcfg->max_tmo_ns;
261 timr->nb_bkts = (timr->max_tout / timr->tck_nsec);
262 timr->vbar0 = timvf_bar(timr->tim_ring_id, 0);
263 timr->bkt_pos = (uint8_t *)timr->vbar0 + TIM_VRING_REL;
264 timr->nb_timers = rcfg->nb_timers;
265 timr->get_target_bkt = bkt_mod;
266
267 if (edev->available_events < timr->nb_timers) {
268 timvf_log_err(
269 "Max available events %"PRIu32" requested timer events %"PRIu64"",
270 edev->available_events, timr->nb_timers);
271 return -ENOMEM;
272 }
273
274 for (i = 0; i < edev->tim_ring_cnt; i++) {
275 if (edev->tim_ring_ids[i] == UINT16_MAX)
276 free_idx = i;
277 }
278
279 if (free_idx == UINT16_MAX) {
280 void *old_ptr;
281
282 edev->tim_ring_cnt++;
283 old_ptr = edev->tim_ring_ids;
284 edev->tim_ring_ids =
285 rte_realloc(edev->tim_ring_ids,
286 sizeof(uint16_t) * edev->tim_ring_cnt, 0);
287 if (edev->tim_ring_ids == NULL) {
288 edev->tim_ring_ids = old_ptr;
289 edev->tim_ring_cnt--;
290 return -ENOMEM;
291 }
292
293 edev->available_events -= timr->nb_timers;
294 } else {
295 edev->tim_ring_ids[free_idx] = tim_ring_id;
296 edev->available_events -= timr->nb_timers;
297 }
298
299 timr->nb_chunks = timr->nb_timers / nb_chunk_slots;
300
301 /* Try to optimize the bucket parameters. */
302 if ((rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES)
303 && !rte_is_power_of_2(timr->nb_bkts)) {
304 if (optimize_bucket_parameters(timr)) {
305 timvf_log_info("Optimized configured values");
306 timvf_log_dbg("nb_bkts : %"PRIu32"", timr->nb_bkts);
307 timvf_log_dbg("tck_nsec : %"PRIu64"", timr->tck_nsec);
308 } else
309 timvf_log_info("Failed to Optimize configured values");
310 }
311
312 if (rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT) {
313 mp_flags = MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET;
314 timvf_log_info("Using single producer mode");
315 }
316
317 timr->bkt = rte_zmalloc("octeontx_timvf_bucket",
318 (timr->nb_bkts) * sizeof(struct tim_mem_bucket),
319 0);
320 if (timr->bkt == NULL)
321 goto mem_err;
322
323 snprintf(pool_name, sizeof(pool_name), "timvf_chunk_pool%d",
324 timr->tim_ring_id);
325 timr->chunk_pool = (void *)rte_mempool_create_empty(pool_name,
326 timr->nb_chunks, TIM_CHUNK_SIZE, 0, 0, rte_socket_id(),
327 mp_flags);
328
329 if (!timr->chunk_pool) {
330 rte_free(timr->bkt);
331 timvf_log_err("Unable to create chunkpool.");
332 return -ENOMEM;
333 }
334
335 mempool_ops = rte_mbuf_best_mempool_ops();
336 ret = rte_mempool_set_ops_byname(timr->chunk_pool,
337 mempool_ops, NULL);
338
339 if (ret != 0) {
340 timvf_log_err("Unable to set chunkpool ops.");
341 goto mem_err;
342 }
343
344 ret = rte_mempool_populate_default(timr->chunk_pool);
345 if (ret < 0) {
346 timvf_log_err("Unable to set populate chunkpool.");
347 goto mem_err;
348 }
349 timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VRING_BASE);
350 timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT);
351 timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT_W1S);
352 timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1C);
353 timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1S);
354
355 return 0;
356 mem_err:
357 rte_free(timr);
358 return -ENOMEM;
359 cfg_err:
360 rte_free(timr);
361 return -EINVAL;
362 }
363
364 static int
timvf_ring_free(struct rte_event_timer_adapter * adptr)365 timvf_ring_free(struct rte_event_timer_adapter *adptr)
366 {
367 struct timvf_ring *timr = adptr->data->adapter_priv;
368 struct ssovf_evdev *edev;
369 int i;
370
371 edev = ssovf_pmd_priv(event_dev);
372 for (i = 0; i < edev->tim_ring_cnt; i++) {
373 if (edev->tim_ring_ids[i] == timr->tim_ring_id) {
374 edev->available_events += timr->nb_timers;
375 edev->tim_ring_ids[i] = UINT16_MAX;
376 break;
377 }
378 }
379
380 rte_mempool_free(timr->chunk_pool);
381 rte_free(timr->bkt);
382 timvf_release_ring(timr->tim_ring_id);
383 rte_free(adptr->data->adapter_priv);
384 return 0;
385 }
386
387 static int
timvf_stats_get(const struct rte_event_timer_adapter * adapter,struct rte_event_timer_adapter_stats * stats)388 timvf_stats_get(const struct rte_event_timer_adapter *adapter,
389 struct rte_event_timer_adapter_stats *stats)
390 {
391 struct timvf_ring *timr = adapter->data->adapter_priv;
392 uint64_t bkt_cyc = rte_rdtsc() - timr->ring_start_cyc;
393
394 stats->evtim_exp_count = timr->tim_arm_cnt;
395 stats->ev_enq_count = timr->tim_arm_cnt;
396 stats->adapter_tick_count = rte_reciprocal_divide_u64(bkt_cyc,
397 &timr->fast_div);
398 return 0;
399 }
400
401 static int
timvf_stats_reset(const struct rte_event_timer_adapter * adapter)402 timvf_stats_reset(const struct rte_event_timer_adapter *adapter)
403 {
404 struct timvf_ring *timr = adapter->data->adapter_priv;
405
406 timr->tim_arm_cnt = 0;
407 return 0;
408 }
409
410 static struct rte_event_timer_adapter_ops timvf_ops = {
411 .init = timvf_ring_create,
412 .uninit = timvf_ring_free,
413 .start = timvf_ring_start,
414 .stop = timvf_ring_stop,
415 .get_info = timvf_ring_info_get,
416 };
417
418 int
timvf_timer_adapter_caps_get(const struct rte_eventdev * dev,uint64_t flags,uint32_t * caps,const struct rte_event_timer_adapter_ops ** ops,uint8_t enable_stats)419 timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
420 uint32_t *caps, const struct rte_event_timer_adapter_ops **ops,
421 uint8_t enable_stats)
422 {
423 RTE_SET_USED(dev);
424
425 if (enable_stats) {
426 timvf_ops.stats_get = timvf_stats_get;
427 timvf_ops.stats_reset = timvf_stats_reset;
428 }
429
430 if (flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT)
431 timvf_ops.arm_burst = enable_stats ?
432 timvf_timer_arm_burst_sp_stats :
433 timvf_timer_arm_burst_sp;
434 else
435 timvf_ops.arm_burst = enable_stats ?
436 timvf_timer_arm_burst_mp_stats :
437 timvf_timer_arm_burst_mp;
438
439 timvf_ops.arm_tmo_tick_burst = enable_stats ?
440 timvf_timer_arm_tmo_brst_stats :
441 timvf_timer_arm_tmo_brst;
442 timvf_ops.cancel_burst = timvf_timer_cancel_burst;
443 *caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT;
444 *ops = &timvf_ops;
445 return 0;
446 }
447
448 void
timvf_set_eventdevice(struct rte_eventdev * dev)449 timvf_set_eventdevice(struct rte_eventdev *dev)
450 {
451 event_dev = dev;
452 }
453