1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
3 */
4 #include <fcntl.h>
5 #include <pthread.h>
6 #include <sys/ioctl.h>
7 #include <sys/mman.h>
8 #include <sys/queue.h>
9 #include <unistd.h>
10
11 #include "roc_api.h"
12 #include "roc_bphy_irq.h"
13
14 #define roc_cpuset_t cpu_set_t
15
16 struct roc_bphy_irq_usr_data {
17 uint64_t isr_base;
18 uint64_t sp;
19 uint64_t cpu;
20 uint64_t irq_num;
21 };
22
23 struct roc_bphy_irq_stack {
24 STAILQ_ENTRY(roc_bphy_irq_stack) entries;
25 void *sp_buffer;
26 int cpu;
27 int inuse;
28 };
29
30 #define ROC_BPHY_MEMZONE_NAME "roc_bphy_mz"
31 #define ROC_BPHY_CTR_DEV_PATH "/dev/otx-bphy-ctr"
32
33 #define ROC_BPHY_IOC_MAGIC 0xF3
34 #define ROC_BPHY_IOC_SET_BPHY_HANDLER \
35 _IOW(ROC_BPHY_IOC_MAGIC, 1, struct roc_bphy_irq_usr_data)
36 #define ROC_BPHY_IOC_CLR_BPHY_HANDLER _IO(ROC_BPHY_IOC_MAGIC, 2)
37 #define ROC_BPHY_IOC_GET_BPHY_MAX_IRQ _IOR(ROC_BPHY_IOC_MAGIC, 3, uint64_t)
38 #define ROC_BPHY_IOC_GET_BPHY_BMASK_IRQ _IOR(ROC_BPHY_IOC_MAGIC, 4, uint64_t)
39
40 static STAILQ_HEAD(slisthead, roc_bphy_irq_stack)
41 irq_stacks = STAILQ_HEAD_INITIALIZER(irq_stacks);
42
43 /* Note: it is assumed that as for now there is no multiprocess support */
44 static pthread_mutex_t stacks_mutex = PTHREAD_MUTEX_INITIALIZER;
45
46 struct roc_bphy_irq_chip *
roc_bphy_intr_init(void)47 roc_bphy_intr_init(void)
48 {
49 struct roc_bphy_irq_chip *irq_chip;
50 uint64_t max_irq, i, avail_irqs;
51 int fd, ret;
52
53 fd = open(ROC_BPHY_CTR_DEV_PATH, O_RDWR | O_SYNC);
54 if (fd < 0) {
55 plt_err("Failed to open %s", ROC_BPHY_CTR_DEV_PATH);
56 return NULL;
57 }
58
59 ret = ioctl(fd, ROC_BPHY_IOC_GET_BPHY_MAX_IRQ, &max_irq);
60 if (ret < 0) {
61 plt_err("Failed to get max irq number via ioctl");
62 goto err_ioctl;
63 }
64
65 ret = ioctl(fd, ROC_BPHY_IOC_GET_BPHY_BMASK_IRQ, &avail_irqs);
66 if (ret < 0) {
67 plt_err("Failed to get available irqs bitmask via ioctl");
68 goto err_ioctl;
69 }
70
71 irq_chip = plt_zmalloc(sizeof(*irq_chip), 0);
72 if (irq_chip == NULL) {
73 plt_err("Failed to alloc irq_chip");
74 goto err_alloc_chip;
75 }
76
77 irq_chip->intfd = fd;
78 irq_chip->max_irq = max_irq;
79 irq_chip->avail_irq_bmask = avail_irqs;
80 irq_chip->irq_vecs =
81 plt_zmalloc(irq_chip->max_irq * sizeof(*irq_chip->irq_vecs), 0);
82 if (irq_chip->irq_vecs == NULL) {
83 plt_err("Failed to alloc irq_chip irq_vecs");
84 goto err_alloc_irq;
85 }
86
87 irq_chip->mz_name = plt_zmalloc(strlen(ROC_BPHY_MEMZONE_NAME) + 1, 0);
88 if (irq_chip->mz_name == NULL) {
89 plt_err("Failed to alloc irq_chip name");
90 goto err_alloc_name;
91 }
92 plt_strlcpy(irq_chip->mz_name, ROC_BPHY_MEMZONE_NAME,
93 strlen(ROC_BPHY_MEMZONE_NAME) + 1);
94
95 for (i = 0; i < irq_chip->max_irq; i++) {
96 irq_chip->irq_vecs[i].fd = -1;
97 irq_chip->irq_vecs[i].handler_cpu = -1;
98 }
99
100 return irq_chip;
101
102 err_alloc_name:
103 plt_free(irq_chip->irq_vecs);
104
105 err_alloc_irq:
106 plt_free(irq_chip);
107
108 err_ioctl:
109 err_alloc_chip:
110 close(fd);
111 return NULL;
112 }
113
114 void
roc_bphy_intr_fini(struct roc_bphy_irq_chip * irq_chip)115 roc_bphy_intr_fini(struct roc_bphy_irq_chip *irq_chip)
116 {
117 if (irq_chip == NULL)
118 return;
119
120 close(irq_chip->intfd);
121 plt_free(irq_chip->mz_name);
122 plt_free(irq_chip->irq_vecs);
123 plt_free(irq_chip);
124 }
125
126 static void
roc_bphy_irq_stack_remove(int cpu)127 roc_bphy_irq_stack_remove(int cpu)
128 {
129 struct roc_bphy_irq_stack *curr_stack;
130
131 if (pthread_mutex_lock(&stacks_mutex))
132 return;
133
134 STAILQ_FOREACH(curr_stack, &irq_stacks, entries) {
135 if (curr_stack->cpu == cpu)
136 break;
137 }
138
139 if (curr_stack == NULL)
140 goto leave;
141
142 if (curr_stack->inuse > 0)
143 curr_stack->inuse--;
144
145 if (curr_stack->inuse == 0) {
146 STAILQ_REMOVE(&irq_stacks, curr_stack, roc_bphy_irq_stack,
147 entries);
148 plt_free(curr_stack->sp_buffer);
149 plt_free(curr_stack);
150 }
151
152 leave:
153 pthread_mutex_unlock(&stacks_mutex);
154 }
155
156 static void *
roc_bphy_irq_stack_get(int cpu)157 roc_bphy_irq_stack_get(int cpu)
158 {
159 #define ARM_STACK_ALIGNMENT (2 * sizeof(void *))
160 #define IRQ_ISR_STACK_SIZE 0x200000
161
162 struct roc_bphy_irq_stack *curr_stack;
163 void *retval = NULL;
164
165 if (pthread_mutex_lock(&stacks_mutex))
166 return NULL;
167
168 STAILQ_FOREACH(curr_stack, &irq_stacks, entries) {
169 if (curr_stack->cpu == cpu) {
170 curr_stack->inuse++;
171 retval = ((char *)curr_stack->sp_buffer) +
172 IRQ_ISR_STACK_SIZE;
173 goto found_stack;
174 }
175 }
176
177 curr_stack = plt_zmalloc(sizeof(struct roc_bphy_irq_stack), 0);
178 if (curr_stack == NULL)
179 goto err_stack;
180
181 curr_stack->sp_buffer =
182 plt_zmalloc(IRQ_ISR_STACK_SIZE * 2, ARM_STACK_ALIGNMENT);
183 if (curr_stack->sp_buffer == NULL)
184 goto err_buffer;
185
186 curr_stack->cpu = cpu;
187 curr_stack->inuse = 0;
188 STAILQ_INSERT_TAIL(&irq_stacks, curr_stack, entries);
189 retval = ((char *)curr_stack->sp_buffer) + IRQ_ISR_STACK_SIZE;
190
191 found_stack:
192 pthread_mutex_unlock(&stacks_mutex);
193 return retval;
194
195 err_buffer:
196 plt_free(curr_stack);
197
198 err_stack:
199 pthread_mutex_unlock(&stacks_mutex);
200 return NULL;
201 }
202
203 void
roc_bphy_intr_handler(unsigned int irq_num)204 roc_bphy_intr_handler(unsigned int irq_num)
205 {
206 struct roc_bphy_irq_chip *irq_chip;
207 const struct plt_memzone *mz;
208
209 mz = plt_memzone_lookup(ROC_BPHY_MEMZONE_NAME);
210 if (mz == NULL)
211 return;
212
213 irq_chip = *(struct roc_bphy_irq_chip **)mz->addr;
214 if (irq_chip == NULL)
215 return;
216
217 if (irq_chip->irq_vecs[irq_num].handler != NULL)
218 irq_chip->irq_vecs[irq_num].handler(
219 (int)irq_num, irq_chip->irq_vecs[irq_num].isr_data);
220
221 roc_atf_ret();
222 }
223
224 static int
roc_bphy_irq_handler_set(struct roc_bphy_irq_chip * chip,int irq_num,void (* isr)(int irq_num,void * isr_data),void * isr_data)225 roc_bphy_irq_handler_set(struct roc_bphy_irq_chip *chip, int irq_num,
226 void (*isr)(int irq_num, void *isr_data),
227 void *isr_data)
228 {
229 roc_cpuset_t orig_cpuset, intr_cpuset;
230 struct roc_bphy_irq_usr_data irq_usr;
231 const struct plt_memzone *mz;
232 int i, retval, curr_cpu, rc;
233 char *env;
234
235 mz = plt_memzone_lookup(chip->mz_name);
236 if (mz == NULL) {
237 /* what we want is just a pointer to chip, not object itself */
238 mz = plt_memzone_reserve_cache_align(chip->mz_name,
239 sizeof(chip));
240 if (mz == NULL)
241 return -ENOMEM;
242 }
243
244 if (chip->irq_vecs[irq_num].handler != NULL)
245 return -EINVAL;
246
247 rc = pthread_getaffinity_np(pthread_self(), sizeof(orig_cpuset),
248 &orig_cpuset);
249 if (rc < 0) {
250 plt_err("Failed to get affinity mask");
251 return rc;
252 }
253
254 for (curr_cpu = -1, i = 0; i < CPU_SETSIZE; i++)
255 if (CPU_ISSET(i, &orig_cpuset))
256 curr_cpu = i;
257 if (curr_cpu < 0)
258 return -ENOENT;
259
260 CPU_ZERO(&intr_cpuset);
261 CPU_SET(curr_cpu, &intr_cpuset);
262 rc = pthread_setaffinity_np(pthread_self(), sizeof(intr_cpuset),
263 &intr_cpuset);
264 if (rc < 0) {
265 plt_err("Failed to set affinity mask");
266 return rc;
267 }
268
269 irq_usr.isr_base = (uint64_t)roc_bphy_intr_handler;
270 irq_usr.sp = (uint64_t)roc_bphy_irq_stack_get(curr_cpu);
271 irq_usr.cpu = curr_cpu;
272 if (irq_usr.sp == 0) {
273 rc = pthread_setaffinity_np(pthread_self(), sizeof(orig_cpuset),
274 &orig_cpuset);
275 if (rc < 0)
276 plt_err("Failed to restore affinity mask");
277 return rc;
278 }
279
280 /* On simulator memory locking operation takes much time. We want
281 * to skip this when running in such an environment.
282 */
283 env = getenv("BPHY_INTR_MLOCK_DISABLE");
284 if (env == NULL) {
285 rc = mlockall(MCL_CURRENT | MCL_FUTURE);
286 if (rc < 0)
287 plt_warn("Failed to lock memory into RAM");
288 }
289
290 *((struct roc_bphy_irq_chip **)(mz->addr)) = chip;
291 irq_usr.irq_num = irq_num;
292 chip->irq_vecs[irq_num].handler_cpu = curr_cpu;
293 chip->irq_vecs[irq_num].handler = isr;
294 chip->irq_vecs[irq_num].isr_data = isr_data;
295 retval = ioctl(chip->intfd, ROC_BPHY_IOC_SET_BPHY_HANDLER, &irq_usr);
296 if (retval != 0) {
297 roc_bphy_irq_stack_remove(curr_cpu);
298 chip->irq_vecs[irq_num].handler = NULL;
299 chip->irq_vecs[irq_num].handler_cpu = -1;
300 } else {
301 chip->n_handlers++;
302 }
303
304 rc = pthread_setaffinity_np(pthread_self(), sizeof(orig_cpuset),
305 &orig_cpuset);
306 if (rc < 0)
307 plt_warn("Failed to restore affinity mask");
308
309 return retval;
310 }
311
312 bool
roc_bphy_intr_available(struct roc_bphy_irq_chip * irq_chip,int irq_num)313 roc_bphy_intr_available(struct roc_bphy_irq_chip *irq_chip, int irq_num)
314 {
315 if (irq_num < 0 || (uint64_t)irq_num >= irq_chip->max_irq)
316 return false;
317
318 return irq_chip->avail_irq_bmask & BIT(irq_num);
319 }
320
321 uint64_t
roc_bphy_intr_max_get(struct roc_bphy_irq_chip * irq_chip)322 roc_bphy_intr_max_get(struct roc_bphy_irq_chip *irq_chip)
323 {
324 return irq_chip->max_irq;
325 }
326
327 int
roc_bphy_intr_clear(struct roc_bphy_irq_chip * chip,int irq_num)328 roc_bphy_intr_clear(struct roc_bphy_irq_chip *chip, int irq_num)
329 {
330 roc_cpuset_t orig_cpuset, intr_cpuset;
331 const struct plt_memzone *mz;
332 int retval;
333
334 if (chip == NULL)
335 return -EINVAL;
336 if ((uint64_t)irq_num >= chip->max_irq || irq_num < 0)
337 return -EINVAL;
338 if (!roc_bphy_intr_available(chip, irq_num))
339 return -ENOTSUP;
340 if (chip->irq_vecs[irq_num].handler == NULL)
341 return -EINVAL;
342 mz = plt_memzone_lookup(chip->mz_name);
343 if (mz == NULL)
344 return -ENXIO;
345
346 retval = pthread_getaffinity_np(pthread_self(), sizeof(orig_cpuset),
347 &orig_cpuset);
348 if (retval < 0) {
349 plt_warn("Failed to get affinity mask");
350 CPU_ZERO(&orig_cpuset);
351 CPU_SET(0, &orig_cpuset);
352 }
353
354 CPU_ZERO(&intr_cpuset);
355 CPU_SET(chip->irq_vecs[irq_num].handler_cpu, &intr_cpuset);
356 retval = pthread_setaffinity_np(pthread_self(), sizeof(intr_cpuset),
357 &intr_cpuset);
358 if (retval < 0) {
359 plt_warn("Failed to set affinity mask");
360 CPU_ZERO(&orig_cpuset);
361 CPU_SET(0, &orig_cpuset);
362 }
363
364 retval = ioctl(chip->intfd, ROC_BPHY_IOC_CLR_BPHY_HANDLER, irq_num);
365 if (retval == 0) {
366 roc_bphy_irq_stack_remove(chip->irq_vecs[irq_num].handler_cpu);
367 chip->n_handlers--;
368 chip->irq_vecs[irq_num].isr_data = NULL;
369 chip->irq_vecs[irq_num].handler = NULL;
370 chip->irq_vecs[irq_num].handler_cpu = -1;
371 if (chip->n_handlers == 0) {
372 retval = plt_memzone_free(mz);
373 if (retval < 0)
374 plt_err("Failed to free memzone: irq %d",
375 irq_num);
376 }
377 } else {
378 plt_err("Failed to clear bphy interrupt handler");
379 }
380
381 retval = pthread_setaffinity_np(pthread_self(), sizeof(orig_cpuset),
382 &orig_cpuset);
383 if (retval < 0) {
384 plt_warn("Failed to restore affinity mask");
385 CPU_ZERO(&orig_cpuset);
386 CPU_SET(0, &orig_cpuset);
387 }
388
389 return retval;
390 }
391
392 int
roc_bphy_intr_register(struct roc_bphy_irq_chip * irq_chip,struct roc_bphy_intr * intr)393 roc_bphy_intr_register(struct roc_bphy_irq_chip *irq_chip,
394 struct roc_bphy_intr *intr)
395 {
396 roc_cpuset_t orig_cpuset, intr_cpuset;
397 int retval;
398 int ret;
399
400 if (!roc_bphy_intr_available(irq_chip, intr->irq_num))
401 return -ENOTSUP;
402
403 retval = pthread_getaffinity_np(pthread_self(), sizeof(orig_cpuset),
404 &orig_cpuset);
405 if (retval < 0) {
406 plt_err("Failed to get affinity mask");
407 return retval;
408 }
409
410 CPU_ZERO(&intr_cpuset);
411 CPU_SET(intr->cpu, &intr_cpuset);
412 retval = pthread_setaffinity_np(pthread_self(), sizeof(intr_cpuset),
413 &intr_cpuset);
414 if (retval < 0) {
415 plt_err("Failed to set affinity mask");
416 return retval;
417 }
418
419 ret = roc_bphy_irq_handler_set(irq_chip, intr->irq_num,
420 intr->intr_handler, intr->isr_data);
421
422 retval = pthread_setaffinity_np(pthread_self(), sizeof(orig_cpuset),
423 &orig_cpuset);
424 if (retval < 0)
425 plt_warn("Failed to restore affinity mask");
426
427 return ret;
428 }
429