1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4
5 #include <stdio.h>
6 #include <unistd.h>
7 #include <inttypes.h>
8 #include <limits.h>
9 #include <string.h>
10
11 #include <rte_compat.h>
12 #include <rte_service.h>
13 #include <rte_service_component.h>
14
15 #include <rte_eal.h>
16 #include <rte_lcore.h>
17 #include <rte_common.h>
18 #include <rte_debug.h>
19 #include <rte_cycles.h>
20 #include <rte_atomic.h>
21 #include <rte_memory.h>
22 #include <rte_malloc.h>
23 #include <rte_spinlock.h>
24
25 #include "eal_private.h"
26
27 #define RTE_SERVICE_NUM_MAX 64
28
29 #define SERVICE_F_REGISTERED (1 << 0)
30 #define SERVICE_F_STATS_ENABLED (1 << 1)
31 #define SERVICE_F_START_CHECK (1 << 2)
32
33 /* runstates for services and lcores, denoting if they are active or not */
34 #define RUNSTATE_STOPPED 0
35 #define RUNSTATE_RUNNING 1
36
37 /* internal representation of a service */
38 struct rte_service_spec_impl {
39 /* public part of the struct */
40 struct rte_service_spec spec;
41
42 /* spin lock that when set indicates a service core is currently
43 * running this service callback. When not set, a core may take the
44 * lock and then run the service callback.
45 */
46 rte_spinlock_t execute_lock;
47
48 /* API set/get-able variables */
49 int8_t app_runstate;
50 int8_t comp_runstate;
51 uint8_t internal_flags;
52
53 /* per service statistics */
54 /* Indicates how many cores the service is mapped to run on.
55 * It does not indicate the number of cores the service is running
56 * on currently.
57 */
58 uint32_t num_mapped_cores;
59 uint64_t calls;
60 uint64_t cycles_spent;
61 } __rte_cache_aligned;
62
63 /* the internal values of a service core */
64 struct core_state {
65 /* map of services IDs are run on this core */
66 uint64_t service_mask;
67 uint8_t runstate; /* running or stopped */
68 uint8_t thread_active; /* indicates when thread is in service_run() */
69 uint8_t is_service_core; /* set if core is currently a service core */
70 uint8_t service_active_on_lcore[RTE_SERVICE_NUM_MAX];
71 uint64_t loops;
72 uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
73 } __rte_cache_aligned;
74
75 static uint32_t rte_service_count;
76 static struct rte_service_spec_impl *rte_services;
77 static struct core_state *lcore_states;
78 static uint32_t rte_service_library_initialized;
79
80 int32_t
rte_service_init(void)81 rte_service_init(void)
82 {
83 if (rte_service_library_initialized) {
84 RTE_LOG(NOTICE, EAL,
85 "service library init() called, init flag %d\n",
86 rte_service_library_initialized);
87 return -EALREADY;
88 }
89
90 rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX,
91 sizeof(struct rte_service_spec_impl),
92 RTE_CACHE_LINE_SIZE);
93 if (!rte_services) {
94 RTE_LOG(ERR, EAL, "error allocating rte services array\n");
95 goto fail_mem;
96 }
97
98 lcore_states = rte_calloc("rte_service_core_states", RTE_MAX_LCORE,
99 sizeof(struct core_state), RTE_CACHE_LINE_SIZE);
100 if (!lcore_states) {
101 RTE_LOG(ERR, EAL, "error allocating core states array\n");
102 goto fail_mem;
103 }
104
105 int i;
106 int count = 0;
107 struct rte_config *cfg = rte_eal_get_configuration();
108 for (i = 0; i < RTE_MAX_LCORE; i++) {
109 if (lcore_config[i].core_role == ROLE_SERVICE) {
110 if ((unsigned int)i == cfg->main_lcore)
111 continue;
112 rte_service_lcore_add(i);
113 count++;
114 }
115 }
116
117 rte_service_library_initialized = 1;
118 return 0;
119 fail_mem:
120 rte_free(rte_services);
121 rte_free(lcore_states);
122 return -ENOMEM;
123 }
124
125 void
rte_service_finalize(void)126 rte_service_finalize(void)
127 {
128 if (!rte_service_library_initialized)
129 return;
130
131 rte_service_lcore_reset_all();
132 rte_eal_mp_wait_lcore();
133
134 rte_free(rte_services);
135 rte_free(lcore_states);
136
137 rte_service_library_initialized = 0;
138 }
139
140 /* returns 1 if service is registered and has not been unregistered
141 * Returns 0 if service never registered, or has been unregistered
142 */
143 static inline int
service_valid(uint32_t id)144 service_valid(uint32_t id)
145 {
146 return !!(rte_services[id].internal_flags & SERVICE_F_REGISTERED);
147 }
148
149 static struct rte_service_spec_impl *
service_get(uint32_t id)150 service_get(uint32_t id)
151 {
152 return &rte_services[id];
153 }
154
155 /* validate ID and retrieve service pointer, or return error value */
156 #define SERVICE_VALID_GET_OR_ERR_RET(id, service, retval) do { \
157 if (id >= RTE_SERVICE_NUM_MAX || !service_valid(id)) \
158 return retval; \
159 service = &rte_services[id]; \
160 } while (0)
161
162 /* returns 1 if statistics should be collected for service
163 * Returns 0 if statistics should not be collected for service
164 */
165 static inline int
service_stats_enabled(struct rte_service_spec_impl * impl)166 service_stats_enabled(struct rte_service_spec_impl *impl)
167 {
168 return !!(impl->internal_flags & SERVICE_F_STATS_ENABLED);
169 }
170
171 static inline int
service_mt_safe(struct rte_service_spec_impl * s)172 service_mt_safe(struct rte_service_spec_impl *s)
173 {
174 return !!(s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE);
175 }
176
177 int32_t
rte_service_set_stats_enable(uint32_t id,int32_t enabled)178 rte_service_set_stats_enable(uint32_t id, int32_t enabled)
179 {
180 struct rte_service_spec_impl *s;
181 SERVICE_VALID_GET_OR_ERR_RET(id, s, 0);
182
183 if (enabled)
184 s->internal_flags |= SERVICE_F_STATS_ENABLED;
185 else
186 s->internal_flags &= ~(SERVICE_F_STATS_ENABLED);
187
188 return 0;
189 }
190
191 int32_t
rte_service_set_runstate_mapped_check(uint32_t id,int32_t enabled)192 rte_service_set_runstate_mapped_check(uint32_t id, int32_t enabled)
193 {
194 struct rte_service_spec_impl *s;
195 SERVICE_VALID_GET_OR_ERR_RET(id, s, 0);
196
197 if (enabled)
198 s->internal_flags |= SERVICE_F_START_CHECK;
199 else
200 s->internal_flags &= ~(SERVICE_F_START_CHECK);
201
202 return 0;
203 }
204
205 uint32_t
rte_service_get_count(void)206 rte_service_get_count(void)
207 {
208 return rte_service_count;
209 }
210
211 int32_t
rte_service_get_by_name(const char * name,uint32_t * service_id)212 rte_service_get_by_name(const char *name, uint32_t *service_id)
213 {
214 if (!service_id)
215 return -EINVAL;
216
217 int i;
218 for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
219 if (service_valid(i) &&
220 strcmp(name, rte_services[i].spec.name) == 0) {
221 *service_id = i;
222 return 0;
223 }
224 }
225
226 return -ENODEV;
227 }
228
229 const char *
rte_service_get_name(uint32_t id)230 rte_service_get_name(uint32_t id)
231 {
232 struct rte_service_spec_impl *s;
233 SERVICE_VALID_GET_OR_ERR_RET(id, s, 0);
234 return s->spec.name;
235 }
236
237 int32_t
rte_service_probe_capability(uint32_t id,uint32_t capability)238 rte_service_probe_capability(uint32_t id, uint32_t capability)
239 {
240 struct rte_service_spec_impl *s;
241 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
242 return !!(s->spec.capabilities & capability);
243 }
244
245 int32_t
rte_service_component_register(const struct rte_service_spec * spec,uint32_t * id_ptr)246 rte_service_component_register(const struct rte_service_spec *spec,
247 uint32_t *id_ptr)
248 {
249 uint32_t i;
250 int32_t free_slot = -1;
251
252 if (spec->callback == NULL || strlen(spec->name) == 0)
253 return -EINVAL;
254
255 for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
256 if (!service_valid(i)) {
257 free_slot = i;
258 break;
259 }
260 }
261
262 if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
263 return -ENOSPC;
264
265 struct rte_service_spec_impl *s = &rte_services[free_slot];
266 s->spec = *spec;
267 s->internal_flags |= SERVICE_F_REGISTERED | SERVICE_F_START_CHECK;
268
269 rte_service_count++;
270
271 if (id_ptr)
272 *id_ptr = free_slot;
273
274 return 0;
275 }
276
277 int32_t
rte_service_component_unregister(uint32_t id)278 rte_service_component_unregister(uint32_t id)
279 {
280 uint32_t i;
281 struct rte_service_spec_impl *s;
282 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
283
284 rte_service_count--;
285
286 s->internal_flags &= ~(SERVICE_F_REGISTERED);
287
288 /* clear the run-bit in all cores */
289 for (i = 0; i < RTE_MAX_LCORE; i++)
290 lcore_states[i].service_mask &= ~(UINT64_C(1) << id);
291
292 memset(&rte_services[id], 0, sizeof(struct rte_service_spec_impl));
293
294 return 0;
295 }
296
297 int32_t
rte_service_component_runstate_set(uint32_t id,uint32_t runstate)298 rte_service_component_runstate_set(uint32_t id, uint32_t runstate)
299 {
300 struct rte_service_spec_impl *s;
301 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
302
303 /* comp_runstate act as the guard variable. Use store-release
304 * memory order. This synchronizes with load-acquire in
305 * service_run and service_runstate_get function.
306 */
307 if (runstate)
308 __atomic_store_n(&s->comp_runstate, RUNSTATE_RUNNING,
309 __ATOMIC_RELEASE);
310 else
311 __atomic_store_n(&s->comp_runstate, RUNSTATE_STOPPED,
312 __ATOMIC_RELEASE);
313
314 return 0;
315 }
316
317 int32_t
rte_service_runstate_set(uint32_t id,uint32_t runstate)318 rte_service_runstate_set(uint32_t id, uint32_t runstate)
319 {
320 struct rte_service_spec_impl *s;
321 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
322
323 /* app_runstate act as the guard variable. Use store-release
324 * memory order. This synchronizes with load-acquire in
325 * service_run runstate_get function.
326 */
327 if (runstate)
328 __atomic_store_n(&s->app_runstate, RUNSTATE_RUNNING,
329 __ATOMIC_RELEASE);
330 else
331 __atomic_store_n(&s->app_runstate, RUNSTATE_STOPPED,
332 __ATOMIC_RELEASE);
333
334 return 0;
335 }
336
337 int32_t
rte_service_runstate_get(uint32_t id)338 rte_service_runstate_get(uint32_t id)
339 {
340 struct rte_service_spec_impl *s;
341 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
342
343 /* comp_runstate and app_runstate act as the guard variables.
344 * Use load-acquire memory order. This synchronizes with
345 * store-release in service state set functions.
346 */
347 if (__atomic_load_n(&s->comp_runstate, __ATOMIC_ACQUIRE) ==
348 RUNSTATE_RUNNING &&
349 __atomic_load_n(&s->app_runstate, __ATOMIC_ACQUIRE) ==
350 RUNSTATE_RUNNING) {
351 int check_disabled = !(s->internal_flags &
352 SERVICE_F_START_CHECK);
353 int lcore_mapped = (__atomic_load_n(&s->num_mapped_cores,
354 __ATOMIC_RELAXED) > 0);
355
356 return (check_disabled | lcore_mapped);
357 } else
358 return 0;
359
360 }
361
362 static inline void
service_runner_do_callback(struct rte_service_spec_impl * s,struct core_state * cs,uint32_t service_idx)363 service_runner_do_callback(struct rte_service_spec_impl *s,
364 struct core_state *cs, uint32_t service_idx)
365 {
366 void *userdata = s->spec.callback_userdata;
367
368 if (service_stats_enabled(s)) {
369 uint64_t start = rte_rdtsc();
370 s->spec.callback(userdata);
371 uint64_t end = rte_rdtsc();
372 s->cycles_spent += end - start;
373 cs->calls_per_service[service_idx]++;
374 s->calls++;
375 } else
376 s->spec.callback(userdata);
377 }
378
379
380 /* Expects the service 's' is valid. */
381 static int32_t
service_run(uint32_t i,struct core_state * cs,uint64_t service_mask,struct rte_service_spec_impl * s,uint32_t serialize_mt_unsafe)382 service_run(uint32_t i, struct core_state *cs, uint64_t service_mask,
383 struct rte_service_spec_impl *s, uint32_t serialize_mt_unsafe)
384 {
385 if (!s)
386 return -EINVAL;
387
388 /* comp_runstate and app_runstate act as the guard variables.
389 * Use load-acquire memory order. This synchronizes with
390 * store-release in service state set functions.
391 */
392 if (__atomic_load_n(&s->comp_runstate, __ATOMIC_ACQUIRE) !=
393 RUNSTATE_RUNNING ||
394 __atomic_load_n(&s->app_runstate, __ATOMIC_ACQUIRE) !=
395 RUNSTATE_RUNNING ||
396 !(service_mask & (UINT64_C(1) << i))) {
397 cs->service_active_on_lcore[i] = 0;
398 return -ENOEXEC;
399 }
400
401 cs->service_active_on_lcore[i] = 1;
402
403 if ((service_mt_safe(s) == 0) && (serialize_mt_unsafe == 1)) {
404 if (!rte_spinlock_trylock(&s->execute_lock))
405 return -EBUSY;
406
407 service_runner_do_callback(s, cs, i);
408 rte_spinlock_unlock(&s->execute_lock);
409 } else
410 service_runner_do_callback(s, cs, i);
411
412 return 0;
413 }
414
415 int32_t
rte_service_may_be_active(uint32_t id)416 rte_service_may_be_active(uint32_t id)
417 {
418 uint32_t ids[RTE_MAX_LCORE] = {0};
419 int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
420 int i;
421
422 if (id >= RTE_SERVICE_NUM_MAX || !service_valid(id))
423 return -EINVAL;
424
425 for (i = 0; i < lcore_count; i++) {
426 if (lcore_states[ids[i]].service_active_on_lcore[id])
427 return 1;
428 }
429
430 return 0;
431 }
432
433 int32_t
rte_service_run_iter_on_app_lcore(uint32_t id,uint32_t serialize_mt_unsafe)434 rte_service_run_iter_on_app_lcore(uint32_t id, uint32_t serialize_mt_unsafe)
435 {
436 struct core_state *cs = &lcore_states[rte_lcore_id()];
437 struct rte_service_spec_impl *s;
438
439 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
440
441 /* Increment num_mapped_cores to reflect that this core is
442 * now mapped capable of running the service.
443 */
444 __atomic_add_fetch(&s->num_mapped_cores, 1, __ATOMIC_RELAXED);
445
446 int ret = service_run(id, cs, UINT64_MAX, s, serialize_mt_unsafe);
447
448 __atomic_sub_fetch(&s->num_mapped_cores, 1, __ATOMIC_RELAXED);
449
450 return ret;
451 }
452
453 static int32_t
service_runner_func(void * arg)454 service_runner_func(void *arg)
455 {
456 RTE_SET_USED(arg);
457 uint32_t i;
458 const int lcore = rte_lcore_id();
459 struct core_state *cs = &lcore_states[lcore];
460
461 __atomic_store_n(&cs->thread_active, 1, __ATOMIC_SEQ_CST);
462
463 /* runstate act as the guard variable. Use load-acquire
464 * memory order here to synchronize with store-release
465 * in runstate update functions.
466 */
467 while (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) ==
468 RUNSTATE_RUNNING) {
469 const uint64_t service_mask = cs->service_mask;
470
471 for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
472 if (!service_valid(i))
473 continue;
474 /* return value ignored as no change to code flow */
475 service_run(i, cs, service_mask, service_get(i), 1);
476 }
477
478 cs->loops++;
479 }
480
481 /* Use SEQ CST memory ordering to avoid any re-ordering around
482 * this store, ensuring that once this store is visible, the service
483 * lcore thread really is done in service cores code.
484 */
485 __atomic_store_n(&cs->thread_active, 0, __ATOMIC_SEQ_CST);
486 return 0;
487 }
488
489 int32_t
rte_service_lcore_may_be_active(uint32_t lcore)490 rte_service_lcore_may_be_active(uint32_t lcore)
491 {
492 if (lcore >= RTE_MAX_LCORE || !lcore_states[lcore].is_service_core)
493 return -EINVAL;
494
495 /* Load thread_active using ACQUIRE to avoid instructions dependent on
496 * the result being re-ordered before this load completes.
497 */
498 return __atomic_load_n(&lcore_states[lcore].thread_active,
499 __ATOMIC_ACQUIRE);
500 }
501
502 int32_t
rte_service_lcore_count(void)503 rte_service_lcore_count(void)
504 {
505 int32_t count = 0;
506 uint32_t i;
507 for (i = 0; i < RTE_MAX_LCORE; i++)
508 count += lcore_states[i].is_service_core;
509 return count;
510 }
511
512 int32_t
rte_service_lcore_list(uint32_t array[],uint32_t n)513 rte_service_lcore_list(uint32_t array[], uint32_t n)
514 {
515 uint32_t count = rte_service_lcore_count();
516 if (count > n)
517 return -ENOMEM;
518
519 if (!array)
520 return -EINVAL;
521
522 uint32_t i;
523 uint32_t idx = 0;
524 for (i = 0; i < RTE_MAX_LCORE; i++) {
525 struct core_state *cs = &lcore_states[i];
526 if (cs->is_service_core) {
527 array[idx] = i;
528 idx++;
529 }
530 }
531
532 return count;
533 }
534
535 int32_t
rte_service_lcore_count_services(uint32_t lcore)536 rte_service_lcore_count_services(uint32_t lcore)
537 {
538 if (lcore >= RTE_MAX_LCORE)
539 return -EINVAL;
540
541 struct core_state *cs = &lcore_states[lcore];
542 if (!cs->is_service_core)
543 return -ENOTSUP;
544
545 return __builtin_popcountll(cs->service_mask);
546 }
547
548 int32_t
rte_service_start_with_defaults(void)549 rte_service_start_with_defaults(void)
550 {
551 /* create a default mapping from cores to services, then start the
552 * services to make them transparent to unaware applications.
553 */
554 uint32_t i;
555 int ret;
556 uint32_t count = rte_service_get_count();
557
558 int32_t lcore_iter = 0;
559 uint32_t ids[RTE_MAX_LCORE] = {0};
560 int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
561
562 if (lcore_count == 0)
563 return -ENOTSUP;
564
565 for (i = 0; (int)i < lcore_count; i++)
566 rte_service_lcore_start(ids[i]);
567
568 for (i = 0; i < count; i++) {
569 /* do 1:1 core mapping here, with each service getting
570 * assigned a single core by default. Adding multiple services
571 * should multiplex to a single core, or 1:1 if there are the
572 * same amount of services as service-cores
573 */
574 ret = rte_service_map_lcore_set(i, ids[lcore_iter], 1);
575 if (ret)
576 return -ENODEV;
577
578 lcore_iter++;
579 if (lcore_iter >= lcore_count)
580 lcore_iter = 0;
581
582 ret = rte_service_runstate_set(i, 1);
583 if (ret)
584 return -ENOEXEC;
585 }
586
587 return 0;
588 }
589
590 static int32_t
service_update(uint32_t sid,uint32_t lcore,uint32_t * set,uint32_t * enabled)591 service_update(uint32_t sid, uint32_t lcore, uint32_t *set, uint32_t *enabled)
592 {
593 /* validate ID, or return error value */
594 if (sid >= RTE_SERVICE_NUM_MAX || !service_valid(sid) ||
595 lcore >= RTE_MAX_LCORE || !lcore_states[lcore].is_service_core)
596 return -EINVAL;
597
598 uint64_t sid_mask = UINT64_C(1) << sid;
599 if (set) {
600 uint64_t lcore_mapped = lcore_states[lcore].service_mask &
601 sid_mask;
602
603 if (*set && !lcore_mapped) {
604 lcore_states[lcore].service_mask |= sid_mask;
605 __atomic_add_fetch(&rte_services[sid].num_mapped_cores,
606 1, __ATOMIC_RELAXED);
607 }
608 if (!*set && lcore_mapped) {
609 lcore_states[lcore].service_mask &= ~(sid_mask);
610 __atomic_sub_fetch(&rte_services[sid].num_mapped_cores,
611 1, __ATOMIC_RELAXED);
612 }
613 }
614
615 if (enabled)
616 *enabled = !!(lcore_states[lcore].service_mask & (sid_mask));
617
618 return 0;
619 }
620
621 int32_t
rte_service_map_lcore_set(uint32_t id,uint32_t lcore,uint32_t enabled)622 rte_service_map_lcore_set(uint32_t id, uint32_t lcore, uint32_t enabled)
623 {
624 uint32_t on = enabled > 0;
625 return service_update(id, lcore, &on, 0);
626 }
627
628 int32_t
rte_service_map_lcore_get(uint32_t id,uint32_t lcore)629 rte_service_map_lcore_get(uint32_t id, uint32_t lcore)
630 {
631 uint32_t enabled;
632 int ret = service_update(id, lcore, 0, &enabled);
633 if (ret == 0)
634 return enabled;
635 return ret;
636 }
637
638 static void
set_lcore_state(uint32_t lcore,int32_t state)639 set_lcore_state(uint32_t lcore, int32_t state)
640 {
641 /* mark core state in hugepage backed config */
642 struct rte_config *cfg = rte_eal_get_configuration();
643 cfg->lcore_role[lcore] = state;
644
645 /* mark state in process local lcore_config */
646 lcore_config[lcore].core_role = state;
647
648 /* update per-lcore optimized state tracking */
649 lcore_states[lcore].is_service_core = (state == ROLE_SERVICE);
650 }
651
652 int32_t
rte_service_lcore_reset_all(void)653 rte_service_lcore_reset_all(void)
654 {
655 /* loop over cores, reset all to mask 0 */
656 uint32_t i;
657 for (i = 0; i < RTE_MAX_LCORE; i++) {
658 if (lcore_states[i].is_service_core) {
659 lcore_states[i].service_mask = 0;
660 set_lcore_state(i, ROLE_RTE);
661 /* runstate act as guard variable Use
662 * store-release memory order here to synchronize
663 * with load-acquire in runstate read functions.
664 */
665 __atomic_store_n(&lcore_states[i].runstate,
666 RUNSTATE_STOPPED, __ATOMIC_RELEASE);
667 }
668 }
669 for (i = 0; i < RTE_SERVICE_NUM_MAX; i++)
670 __atomic_store_n(&rte_services[i].num_mapped_cores, 0,
671 __ATOMIC_RELAXED);
672
673 return 0;
674 }
675
676 int32_t
rte_service_lcore_add(uint32_t lcore)677 rte_service_lcore_add(uint32_t lcore)
678 {
679 if (lcore >= RTE_MAX_LCORE)
680 return -EINVAL;
681 if (lcore_states[lcore].is_service_core)
682 return -EALREADY;
683
684 set_lcore_state(lcore, ROLE_SERVICE);
685
686 /* ensure that after adding a core the mask and state are defaults */
687 lcore_states[lcore].service_mask = 0;
688 /* Use store-release memory order here to synchronize with
689 * load-acquire in runstate read functions.
690 */
691 __atomic_store_n(&lcore_states[lcore].runstate, RUNSTATE_STOPPED,
692 __ATOMIC_RELEASE);
693
694 return rte_eal_wait_lcore(lcore);
695 }
696
697 int32_t
rte_service_lcore_del(uint32_t lcore)698 rte_service_lcore_del(uint32_t lcore)
699 {
700 if (lcore >= RTE_MAX_LCORE)
701 return -EINVAL;
702
703 struct core_state *cs = &lcore_states[lcore];
704 if (!cs->is_service_core)
705 return -EINVAL;
706
707 /* runstate act as the guard variable. Use load-acquire
708 * memory order here to synchronize with store-release
709 * in runstate update functions.
710 */
711 if (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) !=
712 RUNSTATE_STOPPED)
713 return -EBUSY;
714
715 set_lcore_state(lcore, ROLE_RTE);
716
717 rte_smp_wmb();
718 return 0;
719 }
720
721 int32_t
rte_service_lcore_start(uint32_t lcore)722 rte_service_lcore_start(uint32_t lcore)
723 {
724 if (lcore >= RTE_MAX_LCORE)
725 return -EINVAL;
726
727 struct core_state *cs = &lcore_states[lcore];
728 if (!cs->is_service_core)
729 return -EINVAL;
730
731 /* runstate act as the guard variable. Use load-acquire
732 * memory order here to synchronize with store-release
733 * in runstate update functions.
734 */
735 if (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) ==
736 RUNSTATE_RUNNING)
737 return -EALREADY;
738
739 /* set core to run state first, and then launch otherwise it will
740 * return immediately as runstate keeps it in the service poll loop
741 */
742 /* Use load-acquire memory order here to synchronize with
743 * store-release in runstate update functions.
744 */
745 __atomic_store_n(&cs->runstate, RUNSTATE_RUNNING, __ATOMIC_RELEASE);
746
747 int ret = rte_eal_remote_launch(service_runner_func, 0, lcore);
748 /* returns -EBUSY if the core is already launched, 0 on success */
749 return ret;
750 }
751
752 int32_t
rte_service_lcore_stop(uint32_t lcore)753 rte_service_lcore_stop(uint32_t lcore)
754 {
755 if (lcore >= RTE_MAX_LCORE)
756 return -EINVAL;
757
758 /* runstate act as the guard variable. Use load-acquire
759 * memory order here to synchronize with store-release
760 * in runstate update functions.
761 */
762 if (__atomic_load_n(&lcore_states[lcore].runstate, __ATOMIC_ACQUIRE) ==
763 RUNSTATE_STOPPED)
764 return -EALREADY;
765
766 uint32_t i;
767 uint64_t service_mask = lcore_states[lcore].service_mask;
768 for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
769 int32_t enabled = service_mask & (UINT64_C(1) << i);
770 int32_t service_running = rte_service_runstate_get(i);
771 int32_t only_core = (1 ==
772 __atomic_load_n(&rte_services[i].num_mapped_cores,
773 __ATOMIC_RELAXED));
774
775 /* if the core is mapped, and the service is running, and this
776 * is the only core that is mapped, the service would cease to
777 * run if this core stopped, so fail instead.
778 */
779 if (enabled && service_running && only_core)
780 return -EBUSY;
781 }
782
783 /* Use store-release memory order here to synchronize with
784 * load-acquire in runstate read functions.
785 */
786 __atomic_store_n(&lcore_states[lcore].runstate, RUNSTATE_STOPPED,
787 __ATOMIC_RELEASE);
788
789 return 0;
790 }
791
792 int32_t
rte_service_attr_get(uint32_t id,uint32_t attr_id,uint64_t * attr_value)793 rte_service_attr_get(uint32_t id, uint32_t attr_id, uint64_t *attr_value)
794 {
795 struct rte_service_spec_impl *s;
796 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
797
798 if (!attr_value)
799 return -EINVAL;
800
801 switch (attr_id) {
802 case RTE_SERVICE_ATTR_CYCLES:
803 *attr_value = s->cycles_spent;
804 return 0;
805 case RTE_SERVICE_ATTR_CALL_COUNT:
806 *attr_value = s->calls;
807 return 0;
808 default:
809 return -EINVAL;
810 }
811 }
812
813 int32_t
rte_service_lcore_attr_get(uint32_t lcore,uint32_t attr_id,uint64_t * attr_value)814 rte_service_lcore_attr_get(uint32_t lcore, uint32_t attr_id,
815 uint64_t *attr_value)
816 {
817 struct core_state *cs;
818
819 if (lcore >= RTE_MAX_LCORE || !attr_value)
820 return -EINVAL;
821
822 cs = &lcore_states[lcore];
823 if (!cs->is_service_core)
824 return -ENOTSUP;
825
826 switch (attr_id) {
827 case RTE_SERVICE_LCORE_ATTR_LOOPS:
828 *attr_value = cs->loops;
829 return 0;
830 default:
831 return -EINVAL;
832 }
833 }
834
835 int32_t
rte_service_attr_reset_all(uint32_t id)836 rte_service_attr_reset_all(uint32_t id)
837 {
838 struct rte_service_spec_impl *s;
839 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
840
841 s->cycles_spent = 0;
842 s->calls = 0;
843 return 0;
844 }
845
846 int32_t
rte_service_lcore_attr_reset_all(uint32_t lcore)847 rte_service_lcore_attr_reset_all(uint32_t lcore)
848 {
849 struct core_state *cs;
850
851 if (lcore >= RTE_MAX_LCORE)
852 return -EINVAL;
853
854 cs = &lcore_states[lcore];
855 if (!cs->is_service_core)
856 return -ENOTSUP;
857
858 cs->loops = 0;
859
860 return 0;
861 }
862
863 static void
service_dump_one(FILE * f,struct rte_service_spec_impl * s)864 service_dump_one(FILE *f, struct rte_service_spec_impl *s)
865 {
866 /* avoid divide by zero */
867 int calls = 1;
868
869 if (s->calls != 0)
870 calls = s->calls;
871 fprintf(f, " %s: stats %d\tcalls %"PRIu64"\tcycles %"
872 PRIu64"\tavg: %"PRIu64"\n",
873 s->spec.name, service_stats_enabled(s), s->calls,
874 s->cycles_spent, s->cycles_spent / calls);
875 }
876
877 static void
service_dump_calls_per_lcore(FILE * f,uint32_t lcore)878 service_dump_calls_per_lcore(FILE *f, uint32_t lcore)
879 {
880 uint32_t i;
881 struct core_state *cs = &lcore_states[lcore];
882
883 fprintf(f, "%02d\t", lcore);
884 for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
885 if (!service_valid(i))
886 continue;
887 fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]);
888 }
889 fprintf(f, "\n");
890 }
891
892 int32_t
rte_service_dump(FILE * f,uint32_t id)893 rte_service_dump(FILE *f, uint32_t id)
894 {
895 uint32_t i;
896 int print_one = (id != UINT32_MAX);
897
898 /* print only the specified service */
899 if (print_one) {
900 struct rte_service_spec_impl *s;
901 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
902 fprintf(f, "Service %s Summary\n", s->spec.name);
903 service_dump_one(f, s);
904 return 0;
905 }
906
907 /* print all services, as UINT32_MAX was passed as id */
908 fprintf(f, "Services Summary\n");
909 for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
910 if (!service_valid(i))
911 continue;
912 service_dump_one(f, &rte_services[i]);
913 }
914
915 fprintf(f, "Service Cores Summary\n");
916 for (i = 0; i < RTE_MAX_LCORE; i++) {
917 if (lcore_config[i].core_role != ROLE_SERVICE)
918 continue;
919
920 service_dump_calls_per_lcore(f, i);
921 }
922
923 return 0;
924 }
925