1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <stdint.h>
8 #include <unistd.h>
9 #include <pthread.h>
10 #include <signal.h>
11 #include <sched.h>
12 #include <assert.h>
13 #include <string.h>
14
15 #include <rte_errno.h>
16 #include <rte_lcore.h>
17 #include <rte_log.h>
18 #include <rte_memory.h>
19 #include <rte_trace_point.h>
20
21 #include "eal_internal_cfg.h"
22 #include "eal_private.h"
23 #include "eal_thread.h"
24 #include "eal_trace.h"
25
26 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
27 RTE_DEFINE_PER_LCORE(int, _thread_id) = -1;
28 static RTE_DEFINE_PER_LCORE(unsigned int, _socket_id) =
29 (unsigned int)SOCKET_ID_ANY;
30 static RTE_DEFINE_PER_LCORE(rte_cpuset_t, _cpuset);
31
rte_socket_id(void)32 unsigned rte_socket_id(void)
33 {
34 return RTE_PER_LCORE(_socket_id);
35 }
36
37 static int
eal_cpuset_socket_id(rte_cpuset_t * cpusetp)38 eal_cpuset_socket_id(rte_cpuset_t *cpusetp)
39 {
40 unsigned cpu = 0;
41 int socket_id = SOCKET_ID_ANY;
42 int sid;
43
44 if (cpusetp == NULL)
45 return SOCKET_ID_ANY;
46
47 do {
48 if (!CPU_ISSET(cpu, cpusetp))
49 continue;
50
51 if (socket_id == SOCKET_ID_ANY)
52 socket_id = eal_cpu_socket_id(cpu);
53
54 sid = eal_cpu_socket_id(cpu);
55 if (socket_id != sid) {
56 socket_id = SOCKET_ID_ANY;
57 break;
58 }
59
60 } while (++cpu < CPU_SETSIZE);
61
62 return socket_id;
63 }
64
65 static void
thread_update_affinity(rte_cpuset_t * cpusetp)66 thread_update_affinity(rte_cpuset_t *cpusetp)
67 {
68 unsigned int lcore_id = rte_lcore_id();
69
70 /* store socket_id in TLS for quick access */
71 RTE_PER_LCORE(_socket_id) =
72 eal_cpuset_socket_id(cpusetp);
73
74 /* store cpuset in TLS for quick access */
75 memmove(&RTE_PER_LCORE(_cpuset), cpusetp,
76 sizeof(rte_cpuset_t));
77
78 if (lcore_id != (unsigned)LCORE_ID_ANY) {
79 /* EAL thread will update lcore_config */
80 lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id);
81 memmove(&lcore_config[lcore_id].cpuset, cpusetp,
82 sizeof(rte_cpuset_t));
83 }
84 }
85
86 int
rte_thread_set_affinity(rte_cpuset_t * cpusetp)87 rte_thread_set_affinity(rte_cpuset_t *cpusetp)
88 {
89 if (pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
90 cpusetp) != 0) {
91 RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n");
92 return -1;
93 }
94
95 thread_update_affinity(cpusetp);
96 return 0;
97 }
98
99 void
rte_thread_get_affinity(rte_cpuset_t * cpusetp)100 rte_thread_get_affinity(rte_cpuset_t *cpusetp)
101 {
102 assert(cpusetp);
103 memmove(cpusetp, &RTE_PER_LCORE(_cpuset),
104 sizeof(rte_cpuset_t));
105 }
106
107 int
eal_thread_dump_affinity(rte_cpuset_t * cpuset,char * str,unsigned int size)108 eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size)
109 {
110 unsigned cpu;
111 int ret;
112 unsigned int out = 0;
113
114 for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
115 if (!CPU_ISSET(cpu, cpuset))
116 continue;
117
118 ret = snprintf(str + out,
119 size - out, "%u,", cpu);
120 if (ret < 0 || (unsigned)ret >= size - out) {
121 /* string will be truncated */
122 ret = -1;
123 goto exit;
124 }
125
126 out += ret;
127 }
128
129 ret = 0;
130 exit:
131 /* remove the last separator */
132 if (out > 0)
133 str[out - 1] = '\0';
134
135 return ret;
136 }
137
138 int
eal_thread_dump_current_affinity(char * str,unsigned int size)139 eal_thread_dump_current_affinity(char *str, unsigned int size)
140 {
141 rte_cpuset_t cpuset;
142
143 rte_thread_get_affinity(&cpuset);
144 return eal_thread_dump_affinity(&cpuset, str, size);
145 }
146
147 void
__rte_thread_init(unsigned int lcore_id,rte_cpuset_t * cpuset)148 __rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset)
149 {
150 /* set the lcore ID in per-lcore memory area */
151 RTE_PER_LCORE(_lcore_id) = lcore_id;
152
153 /* acquire system unique id */
154 rte_gettid();
155
156 thread_update_affinity(cpuset);
157
158 __rte_trace_mem_per_thread_alloc();
159 }
160
161 void
__rte_thread_uninit(void)162 __rte_thread_uninit(void)
163 {
164 trace_mem_per_thread_free();
165
166 RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY;
167 }
168
169 struct rte_thread_ctrl_params {
170 void *(*start_routine)(void *);
171 void *arg;
172 pthread_barrier_t configured;
173 };
174
ctrl_thread_init(void * arg)175 static void *ctrl_thread_init(void *arg)
176 {
177 int ret;
178 struct internal_config *internal_conf =
179 eal_get_internal_configuration();
180 rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
181 struct rte_thread_ctrl_params *params = arg;
182 void *(*start_routine)(void *) = params->start_routine;
183 void *routine_arg = params->arg;
184
185 __rte_thread_init(rte_lcore_id(), cpuset);
186
187 ret = pthread_barrier_wait(¶ms->configured);
188 if (ret == PTHREAD_BARRIER_SERIAL_THREAD) {
189 pthread_barrier_destroy(¶ms->configured);
190 free(params);
191 }
192
193 return start_routine(routine_arg);
194 }
195
196 int
rte_ctrl_thread_create(pthread_t * thread,const char * name,const pthread_attr_t * attr,void * (* start_routine)(void *),void * arg)197 rte_ctrl_thread_create(pthread_t *thread, const char *name,
198 const pthread_attr_t *attr,
199 void *(*start_routine)(void *), void *arg)
200 {
201 struct internal_config *internal_conf =
202 eal_get_internal_configuration();
203 rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
204 struct rte_thread_ctrl_params *params;
205 int ret;
206
207 params = malloc(sizeof(*params));
208 if (!params)
209 return -ENOMEM;
210
211 params->start_routine = start_routine;
212 params->arg = arg;
213
214 pthread_barrier_init(¶ms->configured, NULL, 2);
215
216 ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params);
217 if (ret != 0) {
218 free(params);
219 return -ret;
220 }
221
222 if (name != NULL) {
223 ret = rte_thread_setname(*thread, name);
224 if (ret < 0)
225 RTE_LOG(DEBUG, EAL,
226 "Cannot set name for ctrl thread\n");
227 }
228
229 ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset);
230 if (ret)
231 goto fail;
232
233 ret = pthread_barrier_wait(¶ms->configured);
234 if (ret == PTHREAD_BARRIER_SERIAL_THREAD) {
235 pthread_barrier_destroy(¶ms->configured);
236 free(params);
237 }
238
239 return 0;
240
241 fail:
242 if (PTHREAD_BARRIER_SERIAL_THREAD ==
243 pthread_barrier_wait(¶ms->configured)) {
244 pthread_barrier_destroy(¶ms->configured);
245 free(params);
246 }
247 pthread_cancel(*thread);
248 pthread_join(*thread, NULL);
249 return -ret;
250 }
251
252 int
rte_thread_register(void)253 rte_thread_register(void)
254 {
255 unsigned int lcore_id;
256 rte_cpuset_t cpuset;
257
258 /* EAL init flushes all lcores, we can't register before. */
259 if (eal_get_internal_configuration()->init_complete != 1) {
260 RTE_LOG(DEBUG, EAL, "Called %s before EAL init.\n", __func__);
261 rte_errno = EINVAL;
262 return -1;
263 }
264 if (!rte_mp_disable()) {
265 RTE_LOG(ERR, EAL, "Multiprocess in use, registering non-EAL threads is not supported.\n");
266 rte_errno = EINVAL;
267 return -1;
268 }
269 if (pthread_getaffinity_np(pthread_self(), sizeof(cpuset),
270 &cpuset) != 0)
271 CPU_ZERO(&cpuset);
272 lcore_id = eal_lcore_non_eal_allocate();
273 if (lcore_id >= RTE_MAX_LCORE)
274 lcore_id = LCORE_ID_ANY;
275 __rte_thread_init(lcore_id, &cpuset);
276 if (lcore_id == LCORE_ID_ANY) {
277 rte_errno = ENOMEM;
278 return -1;
279 }
280 RTE_LOG(DEBUG, EAL, "Registered non-EAL thread as lcore %u.\n",
281 lcore_id);
282 return 0;
283 }
284
285 void
rte_thread_unregister(void)286 rte_thread_unregister(void)
287 {
288 unsigned int lcore_id = rte_lcore_id();
289
290 if (lcore_id != LCORE_ID_ANY)
291 eal_lcore_non_eal_release(lcore_id);
292 __rte_thread_uninit();
293 if (lcore_id != LCORE_ID_ANY)
294 RTE_LOG(DEBUG, EAL, "Unregistered non-EAL thread (was lcore %u).\n",
295 lcore_id);
296 }
297