1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020-2021 NXP
3 */
4
5 #include <string.h>
6 #include <unistd.h>
7 #include <fcntl.h>
8 #include <sys/ioctl.h>
9 #include <sys/mman.h>
10 #include <dirent.h>
11
12 #include <rte_common.h>
13 #include <rte_bus_vdev.h>
14 #include <rte_malloc.h>
15 #include <rte_ring.h>
16 #include <rte_kvargs.h>
17
18 #include <rte_bbdev.h>
19 #include <rte_bbdev_pmd.h>
20
21 #include <bbdev_la12xx_pmd_logs.h>
22 #include <bbdev_la12xx_ipc.h>
23 #include <bbdev_la12xx.h>
24
25 #define DRIVER_NAME baseband_la12xx
26
27 /* Initialisation params structure that can be used by LA12xx BBDEV driver */
28 struct bbdev_la12xx_params {
29 uint8_t queues_num; /*< LA12xx BBDEV queues number */
30 int8_t modem_id; /*< LA12xx modem instance id */
31 };
32
33 #define LA12XX_MAX_NB_QUEUES_ARG "max_nb_queues"
34 #define LA12XX_VDEV_MODEM_ID_ARG "modem"
35 #define LA12XX_MAX_MODEM 4
36
37 #define LA12XX_MAX_CORES 4
38 #define LA12XX_LDPC_ENC_CORE 0
39 #define LA12XX_LDPC_DEC_CORE 1
40
41 #define LA12XX_MAX_LDPC_ENC_QUEUES 4
42 #define LA12XX_MAX_LDPC_DEC_QUEUES 4
43
44 static const char * const bbdev_la12xx_valid_params[] = {
45 LA12XX_MAX_NB_QUEUES_ARG,
46 LA12XX_VDEV_MODEM_ID_ARG,
47 };
48
49 static const struct rte_bbdev_op_cap bbdev_capabilities[] = {
50 {
51 .type = RTE_BBDEV_OP_LDPC_ENC,
52 .cap.ldpc_enc = {
53 .capability_flags =
54 RTE_BBDEV_LDPC_RATE_MATCH |
55 RTE_BBDEV_LDPC_CRC_24A_ATTACH |
56 RTE_BBDEV_LDPC_CRC_24B_ATTACH,
57 .num_buffers_src =
58 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
59 .num_buffers_dst =
60 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
61 }
62 },
63 {
64 .type = RTE_BBDEV_OP_LDPC_DEC,
65 .cap.ldpc_dec = {
66 .capability_flags =
67 RTE_BBDEV_LDPC_CRC_TYPE_24A_CHECK |
68 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |
69 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP,
70 .num_buffers_src =
71 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
72 .num_buffers_hard_out =
73 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
74 .llr_size = 8,
75 .llr_decimals = 1,
76 }
77 },
78 RTE_BBDEV_END_OF_CAPABILITIES_LIST()
79 };
80
81 static struct rte_bbdev_queue_conf default_queue_conf = {
82 .queue_size = MAX_CHANNEL_DEPTH,
83 };
84
85 /* Get device info */
86 static void
la12xx_info_get(struct rte_bbdev * dev __rte_unused,struct rte_bbdev_driver_info * dev_info)87 la12xx_info_get(struct rte_bbdev *dev __rte_unused,
88 struct rte_bbdev_driver_info *dev_info)
89 {
90 PMD_INIT_FUNC_TRACE();
91
92 dev_info->driver_name = RTE_STR(DRIVER_NAME);
93 dev_info->max_num_queues = LA12XX_MAX_QUEUES;
94 dev_info->queue_size_lim = MAX_CHANNEL_DEPTH;
95 dev_info->hardware_accelerated = true;
96 dev_info->max_dl_queue_priority = 0;
97 dev_info->max_ul_queue_priority = 0;
98 dev_info->data_endianness = RTE_BIG_ENDIAN;
99 dev_info->default_queue_conf = default_queue_conf;
100 dev_info->capabilities = bbdev_capabilities;
101 dev_info->cpu_flag_reqs = NULL;
102 dev_info->min_alignment = 64;
103
104 rte_bbdev_log_debug("got device info from %u", dev->data->dev_id);
105 }
106
107 /* Release queue */
108 static int
la12xx_queue_release(struct rte_bbdev * dev,uint16_t q_id)109 la12xx_queue_release(struct rte_bbdev *dev, uint16_t q_id)
110 {
111 RTE_SET_USED(dev);
112 RTE_SET_USED(q_id);
113
114 PMD_INIT_FUNC_TRACE();
115
116 return 0;
117 }
118
119 #define HUGEPG_OFFSET(A) \
120 ((uint64_t) ((unsigned long) (A) \
121 - ((uint64_t)ipc_priv->hugepg_start.host_vaddr)))
122
123 #define MODEM_P2V(A) \
124 ((uint64_t) ((unsigned long) (A) \
125 + (unsigned long)(ipc_priv->peb_start.host_vaddr)))
126
127 static int
ipc_queue_configure(uint32_t channel_id,ipc_t instance,struct bbdev_la12xx_q_priv * q_priv)128 ipc_queue_configure(uint32_t channel_id,
129 ipc_t instance,
130 struct bbdev_la12xx_q_priv *q_priv)
131 {
132 ipc_userspace_t *ipc_priv = (ipc_userspace_t *)instance;
133 ipc_instance_t *ipc_instance = ipc_priv->instance;
134 ipc_ch_t *ch;
135 void *vaddr;
136 uint32_t i = 0;
137 uint32_t msg_size = sizeof(struct bbdev_ipc_enqueue_op);
138
139 PMD_INIT_FUNC_TRACE();
140
141 rte_bbdev_log_debug("%x %p", ipc_instance->initialized,
142 ipc_priv->instance);
143 ch = &(ipc_instance->ch_list[channel_id]);
144
145 rte_bbdev_log_debug("channel: %u, depth: %u, msg size: %u",
146 channel_id, q_priv->queue_size, msg_size);
147
148 /* Start init of channel */
149 ch->md.ring_size = rte_cpu_to_be_32(q_priv->queue_size);
150 ch->md.pi = 0;
151 ch->md.ci = 0;
152 ch->md.msg_size = msg_size;
153 for (i = 0; i < q_priv->queue_size; i++) {
154 vaddr = rte_malloc(NULL, msg_size, RTE_CACHE_LINE_SIZE);
155 if (!vaddr)
156 return IPC_HOST_BUF_ALLOC_FAIL;
157 /* Only offset now */
158 ch->bd_h[i].modem_ptr =
159 rte_cpu_to_be_32(HUGEPG_OFFSET(vaddr));
160 ch->bd_h[i].host_virt_l = lower_32_bits(vaddr);
161 ch->bd_h[i].host_virt_h = upper_32_bits(vaddr);
162 q_priv->msg_ch_vaddr[i] = vaddr;
163 /* Not sure use of this len may be for CRC*/
164 ch->bd_h[i].len = 0;
165 }
166 ch->host_ipc_params =
167 rte_cpu_to_be_32(HUGEPG_OFFSET(q_priv->host_params));
168
169 rte_bbdev_log_debug("Channel configured");
170 return IPC_SUCCESS;
171 }
172
173 static int
la12xx_e200_queue_setup(struct rte_bbdev * dev,struct bbdev_la12xx_q_priv * q_priv)174 la12xx_e200_queue_setup(struct rte_bbdev *dev,
175 struct bbdev_la12xx_q_priv *q_priv)
176 {
177 struct bbdev_la12xx_private *priv = dev->data->dev_private;
178 ipc_userspace_t *ipc_priv = priv->ipc_priv;
179 struct gul_hif *mhif;
180 ipc_metadata_t *ipc_md;
181 ipc_ch_t *ch;
182 int instance_id = 0, i;
183 int ret;
184
185 PMD_INIT_FUNC_TRACE();
186
187 switch (q_priv->op_type) {
188 case RTE_BBDEV_OP_LDPC_ENC:
189 q_priv->la12xx_core_id = LA12XX_LDPC_ENC_CORE;
190 break;
191 case RTE_BBDEV_OP_LDPC_DEC:
192 q_priv->la12xx_core_id = LA12XX_LDPC_DEC_CORE;
193 break;
194 default:
195 rte_bbdev_log(ERR, "Unsupported op type\n");
196 return -1;
197 }
198
199 mhif = (struct gul_hif *)ipc_priv->mhif_start.host_vaddr;
200 /* offset is from start of PEB */
201 ipc_md = (ipc_metadata_t *)((uintptr_t)ipc_priv->peb_start.host_vaddr +
202 mhif->ipc_regs.ipc_mdata_offset);
203 ch = &ipc_md->instance_list[instance_id].ch_list[q_priv->q_id];
204
205 if (q_priv->q_id < priv->num_valid_queues) {
206 ipc_br_md_t *md = &(ch->md);
207
208 q_priv->feca_blk_id = rte_cpu_to_be_32(ch->feca_blk_id);
209 q_priv->feca_blk_id_be32 = ch->feca_blk_id;
210 q_priv->host_pi = rte_be_to_cpu_32(md->pi);
211 q_priv->host_ci = rte_be_to_cpu_32(md->ci);
212 q_priv->host_params = (host_ipc_params_t *)(uintptr_t)
213 (rte_be_to_cpu_32(ch->host_ipc_params) +
214 ((uint64_t)ipc_priv->hugepg_start.host_vaddr));
215
216 for (i = 0; i < q_priv->queue_size; i++) {
217 uint32_t h, l;
218
219 h = ch->bd_h[i].host_virt_h;
220 l = ch->bd_h[i].host_virt_l;
221 q_priv->msg_ch_vaddr[i] = (void *)join_32_bits(h, l);
222 }
223
224 rte_bbdev_log(WARNING,
225 "Queue [%d] already configured, not configuring again",
226 q_priv->q_id);
227 return 0;
228 }
229
230 rte_bbdev_log_debug("setting up queue %d", q_priv->q_id);
231
232 /* Call ipc_configure_channel */
233 ret = ipc_queue_configure(q_priv->q_id, ipc_priv, q_priv);
234 if (ret) {
235 rte_bbdev_log(ERR, "Unable to setup queue (%d) (err=%d)",
236 q_priv->q_id, ret);
237 return ret;
238 }
239
240 /* Set queue properties for LA12xx device */
241 switch (q_priv->op_type) {
242 case RTE_BBDEV_OP_LDPC_ENC:
243 if (priv->num_ldpc_enc_queues >= LA12XX_MAX_LDPC_ENC_QUEUES) {
244 rte_bbdev_log(ERR,
245 "num_ldpc_enc_queues reached max value");
246 return -1;
247 }
248 ch->la12xx_core_id =
249 rte_cpu_to_be_32(LA12XX_LDPC_ENC_CORE);
250 ch->feca_blk_id = rte_cpu_to_be_32(priv->num_ldpc_enc_queues++);
251 break;
252 case RTE_BBDEV_OP_LDPC_DEC:
253 if (priv->num_ldpc_dec_queues >= LA12XX_MAX_LDPC_DEC_QUEUES) {
254 rte_bbdev_log(ERR,
255 "num_ldpc_dec_queues reached max value");
256 return -1;
257 }
258 ch->la12xx_core_id =
259 rte_cpu_to_be_32(LA12XX_LDPC_DEC_CORE);
260 ch->feca_blk_id = rte_cpu_to_be_32(priv->num_ldpc_dec_queues++);
261 break;
262 default:
263 rte_bbdev_log(ERR, "Not supported op type\n");
264 return -1;
265 }
266 ch->op_type = rte_cpu_to_be_32(q_priv->op_type);
267 ch->depth = rte_cpu_to_be_32(q_priv->queue_size);
268
269 /* Store queue config here */
270 q_priv->feca_blk_id = rte_cpu_to_be_32(ch->feca_blk_id);
271 q_priv->feca_blk_id_be32 = ch->feca_blk_id;
272
273 return 0;
274 }
275
276 /* Setup a queue */
277 static int
la12xx_queue_setup(struct rte_bbdev * dev,uint16_t q_id,const struct rte_bbdev_queue_conf * queue_conf)278 la12xx_queue_setup(struct rte_bbdev *dev, uint16_t q_id,
279 const struct rte_bbdev_queue_conf *queue_conf)
280 {
281 struct bbdev_la12xx_private *priv = dev->data->dev_private;
282 struct rte_bbdev_queue_data *q_data;
283 struct bbdev_la12xx_q_priv *q_priv;
284 int ret;
285
286 PMD_INIT_FUNC_TRACE();
287
288 /* Move to setup_queues callback */
289 q_data = &dev->data->queues[q_id];
290 q_data->queue_private = rte_zmalloc(NULL,
291 sizeof(struct bbdev_la12xx_q_priv), 0);
292 if (!q_data->queue_private) {
293 rte_bbdev_log(ERR, "Memory allocation failed for qpriv");
294 return -ENOMEM;
295 }
296 q_priv = q_data->queue_private;
297 q_priv->q_id = q_id;
298 q_priv->bbdev_priv = dev->data->dev_private;
299 q_priv->queue_size = queue_conf->queue_size;
300 q_priv->op_type = queue_conf->op_type;
301
302 ret = la12xx_e200_queue_setup(dev, q_priv);
303 if (ret) {
304 rte_bbdev_log(ERR, "e200_queue_setup failed for qid: %d",
305 q_id);
306 return ret;
307 }
308
309 /* Store queue config here */
310 priv->num_valid_queues++;
311
312 return 0;
313 }
314
315 static int
la12xx_start(struct rte_bbdev * dev)316 la12xx_start(struct rte_bbdev *dev)
317 {
318 struct bbdev_la12xx_private *priv = dev->data->dev_private;
319 ipc_userspace_t *ipc_priv = priv->ipc_priv;
320 int ready = 0;
321 struct gul_hif *hif_start;
322
323 PMD_INIT_FUNC_TRACE();
324
325 hif_start = (struct gul_hif *)ipc_priv->mhif_start.host_vaddr;
326
327 /* Set Host Read bit */
328 SET_HIF_HOST_RDY(hif_start, HIF_HOST_READY_IPC_APP);
329
330 /* Now wait for modem ready bit */
331 while (!ready)
332 ready = CHK_HIF_MOD_RDY(hif_start, HIF_MOD_READY_IPC_APP);
333
334 return 0;
335 }
336
337 static const struct rte_bbdev_ops pmd_ops = {
338 .info_get = la12xx_info_get,
339 .queue_setup = la12xx_queue_setup,
340 .queue_release = la12xx_queue_release,
341 .start = la12xx_start
342 };
343
344 static inline int
is_bd_ring_full(uint32_t ci,uint32_t ci_flag,uint32_t pi,uint32_t pi_flag)345 is_bd_ring_full(uint32_t ci, uint32_t ci_flag,
346 uint32_t pi, uint32_t pi_flag)
347 {
348 if (pi == ci) {
349 if (pi_flag != ci_flag)
350 return 1; /* Ring is Full */
351 }
352 return 0;
353 }
354
355 static inline int
prepare_ldpc_enc_op(struct rte_bbdev_enc_op * bbdev_enc_op,struct bbdev_la12xx_q_priv * q_priv __rte_unused,struct rte_bbdev_op_data * in_op_data __rte_unused,struct rte_bbdev_op_data * out_op_data)356 prepare_ldpc_enc_op(struct rte_bbdev_enc_op *bbdev_enc_op,
357 struct bbdev_la12xx_q_priv *q_priv __rte_unused,
358 struct rte_bbdev_op_data *in_op_data __rte_unused,
359 struct rte_bbdev_op_data *out_op_data)
360 {
361 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &bbdev_enc_op->ldpc_enc;
362 uint32_t total_out_bits;
363
364 total_out_bits = (ldpc_enc->tb_params.cab *
365 ldpc_enc->tb_params.ea) + (ldpc_enc->tb_params.c -
366 ldpc_enc->tb_params.cab) * ldpc_enc->tb_params.eb;
367
368 ldpc_enc->output.length = (total_out_bits + 7)/8;
369
370 rte_pktmbuf_append(out_op_data->data, ldpc_enc->output.length);
371
372 return 0;
373 }
374
375 static inline int
prepare_ldpc_dec_op(struct rte_bbdev_dec_op * bbdev_dec_op,struct bbdev_ipc_dequeue_op * bbdev_ipc_op,struct bbdev_la12xx_q_priv * q_priv __rte_unused,struct rte_bbdev_op_data * out_op_data __rte_unused)376 prepare_ldpc_dec_op(struct rte_bbdev_dec_op *bbdev_dec_op,
377 struct bbdev_ipc_dequeue_op *bbdev_ipc_op,
378 struct bbdev_la12xx_q_priv *q_priv __rte_unused,
379 struct rte_bbdev_op_data *out_op_data __rte_unused)
380 {
381 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &bbdev_dec_op->ldpc_dec;
382 uint32_t total_out_bits;
383 uint32_t num_code_blocks = 0;
384 uint16_t sys_cols;
385
386 sys_cols = (ldpc_dec->basegraph == 1) ? 22 : 10;
387 if (ldpc_dec->tb_params.c == 1) {
388 total_out_bits = ((sys_cols * ldpc_dec->z_c) -
389 ldpc_dec->n_filler);
390 /* 5G-NR protocol uses 16 bit CRC when output packet
391 * size <= 3824 (bits). Otherwise 24 bit CRC is used.
392 * Adjust the output bits accordingly
393 */
394 if (total_out_bits - 16 <= 3824)
395 total_out_bits -= 16;
396 else
397 total_out_bits -= 24;
398 ldpc_dec->hard_output.length = (total_out_bits / 8);
399 } else {
400 total_out_bits = (((sys_cols * ldpc_dec->z_c) -
401 ldpc_dec->n_filler - 24) *
402 ldpc_dec->tb_params.c);
403 ldpc_dec->hard_output.length = (total_out_bits / 8) - 3;
404 }
405
406 num_code_blocks = ldpc_dec->tb_params.c;
407
408 bbdev_ipc_op->num_code_blocks = rte_cpu_to_be_32(num_code_blocks);
409
410 return 0;
411 }
412
413 static int
enqueue_single_op(struct bbdev_la12xx_q_priv * q_priv,void * bbdev_op)414 enqueue_single_op(struct bbdev_la12xx_q_priv *q_priv, void *bbdev_op)
415 {
416 struct bbdev_la12xx_private *priv = q_priv->bbdev_priv;
417 ipc_userspace_t *ipc_priv = priv->ipc_priv;
418 ipc_instance_t *ipc_instance = ipc_priv->instance;
419 struct bbdev_ipc_dequeue_op *bbdev_ipc_op;
420 struct rte_bbdev_op_ldpc_enc *ldpc_enc;
421 struct rte_bbdev_op_ldpc_dec *ldpc_dec;
422 uint32_t q_id = q_priv->q_id;
423 uint32_t ci, ci_flag, pi, pi_flag;
424 ipc_ch_t *ch = &(ipc_instance->ch_list[q_id]);
425 ipc_br_md_t *md = &(ch->md);
426 size_t virt;
427 char *huge_start_addr =
428 (char *)q_priv->bbdev_priv->ipc_priv->hugepg_start.host_vaddr;
429 struct rte_bbdev_op_data *in_op_data, *out_op_data;
430 char *data_ptr;
431 uint32_t l1_pcie_addr;
432 int ret;
433
434 ci = IPC_GET_CI_INDEX(q_priv->host_ci);
435 ci_flag = IPC_GET_CI_FLAG(q_priv->host_ci);
436
437 pi = IPC_GET_PI_INDEX(q_priv->host_pi);
438 pi_flag = IPC_GET_PI_FLAG(q_priv->host_pi);
439
440 rte_bbdev_dp_log(DEBUG, "before bd_ring_full: pi: %u, ci: %u,"
441 "pi_flag: %u, ci_flag: %u, ring size: %u",
442 pi, ci, pi_flag, ci_flag, q_priv->queue_size);
443
444 if (is_bd_ring_full(ci, ci_flag, pi, pi_flag)) {
445 rte_bbdev_dp_log(DEBUG, "bd ring full for queue id: %d", q_id);
446 return IPC_CH_FULL;
447 }
448
449 virt = MODEM_P2V(q_priv->host_params->bd_m_modem_ptr[pi]);
450 bbdev_ipc_op = (struct bbdev_ipc_dequeue_op *)virt;
451 q_priv->bbdev_op[pi] = bbdev_op;
452
453 switch (q_priv->op_type) {
454 case RTE_BBDEV_OP_LDPC_ENC:
455 ldpc_enc = &(((struct rte_bbdev_enc_op *)bbdev_op)->ldpc_enc);
456 in_op_data = &ldpc_enc->input;
457 out_op_data = &ldpc_enc->output;
458
459 ret = prepare_ldpc_enc_op(bbdev_op, q_priv,
460 in_op_data, out_op_data);
461 if (ret) {
462 rte_bbdev_log(ERR, "process_ldpc_enc_op fail, ret: %d",
463 ret);
464 return ret;
465 }
466 break;
467
468 case RTE_BBDEV_OP_LDPC_DEC:
469 ldpc_dec = &(((struct rte_bbdev_dec_op *)bbdev_op)->ldpc_dec);
470 in_op_data = &ldpc_dec->input;
471
472 out_op_data = &ldpc_dec->hard_output;
473
474 ret = prepare_ldpc_dec_op(bbdev_op, bbdev_ipc_op,
475 q_priv, out_op_data);
476 if (ret) {
477 rte_bbdev_log(ERR, "process_ldpc_dec_op fail, ret: %d",
478 ret);
479 return ret;
480 }
481 break;
482
483 default:
484 rte_bbdev_log(ERR, "unsupported bbdev_ipc op type");
485 return -1;
486 }
487
488 if (in_op_data->data) {
489 data_ptr = rte_pktmbuf_mtod(in_op_data->data, char *);
490 l1_pcie_addr = (uint32_t)GUL_USER_HUGE_PAGE_ADDR +
491 data_ptr - huge_start_addr;
492 bbdev_ipc_op->in_addr = l1_pcie_addr;
493 bbdev_ipc_op->in_len = in_op_data->length;
494 }
495
496 if (out_op_data->data) {
497 data_ptr = rte_pktmbuf_mtod(out_op_data->data, char *);
498 l1_pcie_addr = (uint32_t)GUL_USER_HUGE_PAGE_ADDR +
499 data_ptr - huge_start_addr;
500 bbdev_ipc_op->out_addr = rte_cpu_to_be_32(l1_pcie_addr);
501 bbdev_ipc_op->out_len = rte_cpu_to_be_32(out_op_data->length);
502 }
503
504 /* Move Producer Index forward */
505 pi++;
506 /* Flip the PI flag, if wrapping */
507 if (unlikely(q_priv->queue_size == pi)) {
508 pi = 0;
509 pi_flag = pi_flag ? 0 : 1;
510 }
511
512 if (pi_flag)
513 IPC_SET_PI_FLAG(pi);
514 else
515 IPC_RESET_PI_FLAG(pi);
516 q_priv->host_pi = pi;
517
518 /* Wait for Data Copy & pi_flag update to complete before updating pi */
519 rte_mb();
520 /* now update pi */
521 md->pi = rte_cpu_to_be_32(pi);
522
523 rte_bbdev_dp_log(DEBUG, "enter: pi: %u, ci: %u,"
524 "pi_flag: %u, ci_flag: %u, ring size: %u",
525 pi, ci, pi_flag, ci_flag, q_priv->queue_size);
526
527 return 0;
528 }
529
530 /* Enqueue decode burst */
531 static uint16_t
enqueue_dec_ops(struct rte_bbdev_queue_data * q_data,struct rte_bbdev_dec_op ** ops,uint16_t nb_ops)532 enqueue_dec_ops(struct rte_bbdev_queue_data *q_data,
533 struct rte_bbdev_dec_op **ops, uint16_t nb_ops)
534 {
535 struct bbdev_la12xx_q_priv *q_priv = q_data->queue_private;
536 int nb_enqueued, ret;
537
538 for (nb_enqueued = 0; nb_enqueued < nb_ops; nb_enqueued++) {
539 ret = enqueue_single_op(q_priv, ops[nb_enqueued]);
540 if (ret)
541 break;
542 }
543
544 q_data->queue_stats.enqueue_err_count += nb_ops - nb_enqueued;
545 q_data->queue_stats.enqueued_count += nb_enqueued;
546
547 return nb_enqueued;
548 }
549
550 /* Enqueue encode burst */
551 static uint16_t
enqueue_enc_ops(struct rte_bbdev_queue_data * q_data,struct rte_bbdev_enc_op ** ops,uint16_t nb_ops)552 enqueue_enc_ops(struct rte_bbdev_queue_data *q_data,
553 struct rte_bbdev_enc_op **ops, uint16_t nb_ops)
554 {
555 struct bbdev_la12xx_q_priv *q_priv = q_data->queue_private;
556 int nb_enqueued, ret;
557
558 for (nb_enqueued = 0; nb_enqueued < nb_ops; nb_enqueued++) {
559 ret = enqueue_single_op(q_priv, ops[nb_enqueued]);
560 if (ret)
561 break;
562 }
563
564 q_data->queue_stats.enqueue_err_count += nb_ops - nb_enqueued;
565 q_data->queue_stats.enqueued_count += nb_enqueued;
566
567 return nb_enqueued;
568 }
569
570 /* Dequeue encode burst */
571 static void *
dequeue_single_op(struct bbdev_la12xx_q_priv * q_priv,void * dst)572 dequeue_single_op(struct bbdev_la12xx_q_priv *q_priv, void *dst)
573 {
574 void *op;
575 uint32_t ci, ci_flag;
576 uint32_t temp_ci;
577
578 temp_ci = q_priv->host_params->ci;
579 if (temp_ci == q_priv->host_ci)
580 return NULL;
581
582 ci = IPC_GET_CI_INDEX(q_priv->host_ci);
583 ci_flag = IPC_GET_CI_FLAG(q_priv->host_ci);
584
585 rte_bbdev_dp_log(DEBUG,
586 "ci: %u, ci_flag: %u, ring size: %u",
587 ci, ci_flag, q_priv->queue_size);
588
589 op = q_priv->bbdev_op[ci];
590
591 rte_memcpy(dst, q_priv->msg_ch_vaddr[ci],
592 sizeof(struct bbdev_ipc_enqueue_op));
593
594 /* Move Consumer Index forward */
595 ci++;
596 /* Flip the CI flag, if wrapping */
597 if (q_priv->queue_size == ci) {
598 ci = 0;
599 ci_flag = ci_flag ? 0 : 1;
600 }
601 if (ci_flag)
602 IPC_SET_CI_FLAG(ci);
603 else
604 IPC_RESET_CI_FLAG(ci);
605
606 q_priv->host_ci = ci;
607
608 rte_bbdev_dp_log(DEBUG,
609 "exit: ci: %u, ci_flag: %u, ring size: %u",
610 ci, ci_flag, q_priv->queue_size);
611
612 return op;
613 }
614
615 /* Dequeue decode burst */
616 static uint16_t
dequeue_dec_ops(struct rte_bbdev_queue_data * q_data,struct rte_bbdev_dec_op ** ops,uint16_t nb_ops)617 dequeue_dec_ops(struct rte_bbdev_queue_data *q_data,
618 struct rte_bbdev_dec_op **ops, uint16_t nb_ops)
619 {
620 struct bbdev_la12xx_q_priv *q_priv = q_data->queue_private;
621 struct bbdev_ipc_enqueue_op bbdev_ipc_op;
622 int nb_dequeued;
623
624 for (nb_dequeued = 0; nb_dequeued < nb_ops; nb_dequeued++) {
625 ops[nb_dequeued] = dequeue_single_op(q_priv, &bbdev_ipc_op);
626 if (!ops[nb_dequeued])
627 break;
628 ops[nb_dequeued]->status = bbdev_ipc_op.status;
629 }
630 q_data->queue_stats.dequeued_count += nb_dequeued;
631
632 return nb_dequeued;
633 }
634
635 /* Dequeue encode burst */
636 static uint16_t
dequeue_enc_ops(struct rte_bbdev_queue_data * q_data,struct rte_bbdev_enc_op ** ops,uint16_t nb_ops)637 dequeue_enc_ops(struct rte_bbdev_queue_data *q_data,
638 struct rte_bbdev_enc_op **ops, uint16_t nb_ops)
639 {
640 struct bbdev_la12xx_q_priv *q_priv = q_data->queue_private;
641 struct bbdev_ipc_enqueue_op bbdev_ipc_op;
642 int nb_enqueued;
643
644 for (nb_enqueued = 0; nb_enqueued < nb_ops; nb_enqueued++) {
645 ops[nb_enqueued] = dequeue_single_op(q_priv, &bbdev_ipc_op);
646 if (!ops[nb_enqueued])
647 break;
648 ops[nb_enqueued]->status = bbdev_ipc_op.status;
649 }
650 q_data->queue_stats.enqueued_count += nb_enqueued;
651
652 return nb_enqueued;
653 }
654
655 static struct hugepage_info *
get_hugepage_info(void)656 get_hugepage_info(void)
657 {
658 struct hugepage_info *hp_info;
659 struct rte_memseg *mseg;
660
661 PMD_INIT_FUNC_TRACE();
662
663 /* TODO - find a better way */
664 hp_info = rte_malloc(NULL, sizeof(struct hugepage_info), 0);
665 if (!hp_info) {
666 rte_bbdev_log(ERR, "Unable to allocate on local heap");
667 return NULL;
668 }
669
670 mseg = rte_mem_virt2memseg(hp_info, NULL);
671 hp_info->vaddr = mseg->addr;
672 hp_info->paddr = rte_mem_virt2phy(mseg->addr);
673 hp_info->len = mseg->len;
674
675 return hp_info;
676 }
677
678 static int
open_ipc_dev(int modem_id)679 open_ipc_dev(int modem_id)
680 {
681 char dev_initials[16], dev_path[PATH_MAX];
682 struct dirent *entry;
683 int dev_ipc = 0;
684 DIR *dir;
685
686 dir = opendir("/dev/");
687 if (!dir) {
688 rte_bbdev_log(ERR, "Unable to open /dev/");
689 return -1;
690 }
691
692 sprintf(dev_initials, "gulipcgul%d", modem_id);
693
694 while ((entry = readdir(dir)) != NULL) {
695 if (!strncmp(dev_initials, entry->d_name,
696 sizeof(dev_initials) - 1))
697 break;
698 }
699
700 if (!entry) {
701 rte_bbdev_log(ERR, "Error: No gulipcgul%d device", modem_id);
702 return -1;
703 }
704
705 sprintf(dev_path, "/dev/%s", entry->d_name);
706 dev_ipc = open(dev_path, O_RDWR);
707 if (dev_ipc < 0) {
708 rte_bbdev_log(ERR, "Error: Cannot open %s", dev_path);
709 return -errno;
710 }
711
712 return dev_ipc;
713 }
714
715 static int
setup_la12xx_dev(struct rte_bbdev * dev)716 setup_la12xx_dev(struct rte_bbdev *dev)
717 {
718 struct bbdev_la12xx_private *priv = dev->data->dev_private;
719 ipc_userspace_t *ipc_priv = priv->ipc_priv;
720 struct hugepage_info *hp = NULL;
721 ipc_channel_us_t *ipc_priv_ch = NULL;
722 int dev_ipc = 0, dev_mem = 0, i;
723 ipc_metadata_t *ipc_md;
724 struct gul_hif *mhif;
725 uint32_t phy_align = 0;
726 int ret;
727
728 PMD_INIT_FUNC_TRACE();
729
730 if (!ipc_priv) {
731 /* TODO - get a better way */
732 /* Get the hugepage info against it */
733 hp = get_hugepage_info();
734 if (!hp) {
735 rte_bbdev_log(ERR, "Unable to get hugepage info");
736 ret = -ENOMEM;
737 goto err;
738 }
739
740 rte_bbdev_log_debug("0x%" PRIx64 " %p 0x%" PRIx64,
741 hp->paddr, hp->vaddr, hp->len);
742
743 ipc_priv = rte_zmalloc(0, sizeof(ipc_userspace_t), 0);
744 if (ipc_priv == NULL) {
745 rte_bbdev_log(ERR,
746 "Unable to allocate memory for ipc priv");
747 ret = -ENOMEM;
748 goto err;
749 }
750
751 for (i = 0; i < IPC_MAX_CHANNEL_COUNT; i++) {
752 ipc_priv_ch = rte_zmalloc(0,
753 sizeof(ipc_channel_us_t), 0);
754 if (ipc_priv_ch == NULL) {
755 rte_bbdev_log(ERR,
756 "Unable to allocate memory for channels");
757 ret = -ENOMEM;
758 }
759 ipc_priv->channels[i] = ipc_priv_ch;
760 }
761
762 dev_mem = open("/dev/mem", O_RDWR);
763 if (dev_mem < 0) {
764 rte_bbdev_log(ERR, "Error: Cannot open /dev/mem");
765 ret = -errno;
766 goto err;
767 }
768
769 ipc_priv->instance_id = 0;
770 ipc_priv->dev_mem = dev_mem;
771
772 rte_bbdev_log_debug("hugepg input 0x%" PRIx64 "%p 0x%" PRIx64,
773 hp->paddr, hp->vaddr, hp->len);
774
775 ipc_priv->sys_map.hugepg_start.host_phys = hp->paddr;
776 ipc_priv->sys_map.hugepg_start.size = hp->len;
777
778 ipc_priv->hugepg_start.host_phys = hp->paddr;
779 ipc_priv->hugepg_start.host_vaddr = hp->vaddr;
780 ipc_priv->hugepg_start.size = hp->len;
781
782 rte_free(hp);
783 }
784
785 dev_ipc = open_ipc_dev(priv->modem_id);
786 if (dev_ipc < 0) {
787 rte_bbdev_log(ERR, "Error: open_ipc_dev failed");
788 goto err;
789 }
790 ipc_priv->dev_ipc = dev_ipc;
791
792 ret = ioctl(ipc_priv->dev_ipc, IOCTL_GUL_IPC_GET_SYS_MAP,
793 &ipc_priv->sys_map);
794 if (ret) {
795 rte_bbdev_log(ERR,
796 "IOCTL_GUL_IPC_GET_SYS_MAP ioctl failed");
797 goto err;
798 }
799
800 phy_align = (ipc_priv->sys_map.mhif_start.host_phys % 0x1000);
801 ipc_priv->mhif_start.host_vaddr =
802 mmap(0, ipc_priv->sys_map.mhif_start.size + phy_align,
803 (PROT_READ | PROT_WRITE), MAP_SHARED, ipc_priv->dev_mem,
804 (ipc_priv->sys_map.mhif_start.host_phys - phy_align));
805 if (ipc_priv->mhif_start.host_vaddr == MAP_FAILED) {
806 rte_bbdev_log(ERR, "MAP failed:");
807 ret = -errno;
808 goto err;
809 }
810
811 ipc_priv->mhif_start.host_vaddr = (void *) ((uintptr_t)
812 (ipc_priv->mhif_start.host_vaddr) + phy_align);
813
814 phy_align = (ipc_priv->sys_map.peb_start.host_phys % 0x1000);
815 ipc_priv->peb_start.host_vaddr =
816 mmap(0, ipc_priv->sys_map.peb_start.size + phy_align,
817 (PROT_READ | PROT_WRITE), MAP_SHARED, ipc_priv->dev_mem,
818 (ipc_priv->sys_map.peb_start.host_phys - phy_align));
819 if (ipc_priv->peb_start.host_vaddr == MAP_FAILED) {
820 rte_bbdev_log(ERR, "MAP failed:");
821 ret = -errno;
822 goto err;
823 }
824
825 ipc_priv->peb_start.host_vaddr = (void *)((uintptr_t)
826 (ipc_priv->peb_start.host_vaddr) + phy_align);
827
828 phy_align = (ipc_priv->sys_map.modem_ccsrbar.host_phys % 0x1000);
829 ipc_priv->modem_ccsrbar.host_vaddr =
830 mmap(0, ipc_priv->sys_map.modem_ccsrbar.size + phy_align,
831 (PROT_READ | PROT_WRITE), MAP_SHARED, ipc_priv->dev_mem,
832 (ipc_priv->sys_map.modem_ccsrbar.host_phys - phy_align));
833 if (ipc_priv->modem_ccsrbar.host_vaddr == MAP_FAILED) {
834 rte_bbdev_log(ERR, "MAP failed:");
835 ret = -errno;
836 goto err;
837 }
838
839 ipc_priv->modem_ccsrbar.host_vaddr = (void *)((uintptr_t)
840 (ipc_priv->modem_ccsrbar.host_vaddr) + phy_align);
841
842 ipc_priv->hugepg_start.modem_phys =
843 ipc_priv->sys_map.hugepg_start.modem_phys;
844
845 ipc_priv->mhif_start.host_phys =
846 ipc_priv->sys_map.mhif_start.host_phys;
847 ipc_priv->mhif_start.size = ipc_priv->sys_map.mhif_start.size;
848 ipc_priv->peb_start.host_phys = ipc_priv->sys_map.peb_start.host_phys;
849 ipc_priv->peb_start.size = ipc_priv->sys_map.peb_start.size;
850
851 rte_bbdev_log(INFO, "peb 0x%" PRIx64 "%p 0x%" PRIx32,
852 ipc_priv->peb_start.host_phys,
853 ipc_priv->peb_start.host_vaddr,
854 ipc_priv->peb_start.size);
855 rte_bbdev_log(INFO, "hugepg 0x%" PRIx64 "%p 0x%" PRIx32,
856 ipc_priv->hugepg_start.host_phys,
857 ipc_priv->hugepg_start.host_vaddr,
858 ipc_priv->hugepg_start.size);
859 rte_bbdev_log(INFO, "mhif 0x%" PRIx64 "%p 0x%" PRIx32,
860 ipc_priv->mhif_start.host_phys,
861 ipc_priv->mhif_start.host_vaddr,
862 ipc_priv->mhif_start.size);
863 mhif = (struct gul_hif *)ipc_priv->mhif_start.host_vaddr;
864
865 /* offset is from start of PEB */
866 ipc_md = (ipc_metadata_t *)((uintptr_t)ipc_priv->peb_start.host_vaddr +
867 mhif->ipc_regs.ipc_mdata_offset);
868
869 if (sizeof(ipc_metadata_t) != mhif->ipc_regs.ipc_mdata_size) {
870 rte_bbdev_log(ERR,
871 "ipc_metadata_t =0x%" PRIx64
872 ", mhif->ipc_regs.ipc_mdata_size=0x%" PRIx32,
873 (uint64_t)(sizeof(ipc_metadata_t)),
874 mhif->ipc_regs.ipc_mdata_size);
875 rte_bbdev_log(ERR, "--> mhif->ipc_regs.ipc_mdata_offset= 0x%"
876 PRIx32, mhif->ipc_regs.ipc_mdata_offset);
877 rte_bbdev_log(ERR, "gul_hif size=0x%" PRIx64,
878 (uint64_t)(sizeof(struct gul_hif)));
879 return IPC_MD_SZ_MISS_MATCH;
880 }
881
882 ipc_priv->instance = (ipc_instance_t *)
883 (&ipc_md->instance_list[ipc_priv->instance_id]);
884
885 rte_bbdev_log_debug("finish host init");
886
887 priv->ipc_priv = ipc_priv;
888
889 return 0;
890
891 err:
892 rte_free(hp);
893 rte_free(ipc_priv);
894 rte_free(ipc_priv_ch);
895 if (dev_mem)
896 close(dev_mem);
897 if (dev_ipc)
898 close(dev_ipc);
899
900 return ret;
901 }
902
903 static inline int
parse_u16_arg(const char * key,const char * value,void * extra_args)904 parse_u16_arg(const char *key, const char *value, void *extra_args)
905 {
906 uint16_t *u16 = extra_args;
907
908 uint64_t result;
909 if ((value == NULL) || (extra_args == NULL))
910 return -EINVAL;
911 errno = 0;
912 result = strtoul(value, NULL, 0);
913 if ((result >= (1 << 16)) || (errno != 0)) {
914 rte_bbdev_log(ERR, "Invalid value %" PRIu64 " for %s",
915 result, key);
916 return -ERANGE;
917 }
918 *u16 = (uint16_t)result;
919 return 0;
920 }
921
922 /* Parse integer from integer argument */
923 static int
parse_integer_arg(const char * key __rte_unused,const char * value,void * extra_args)924 parse_integer_arg(const char *key __rte_unused,
925 const char *value, void *extra_args)
926 {
927 int i;
928 char *end;
929
930 errno = 0;
931
932 i = strtol(value, &end, 10);
933 if (*end != 0 || errno != 0 || i < 0 || i > LA12XX_MAX_MODEM) {
934 rte_bbdev_log(ERR, "Supported Port IDS are 0 to %d",
935 LA12XX_MAX_MODEM - 1);
936 return -EINVAL;
937 }
938
939 *((uint32_t *)extra_args) = i;
940
941 return 0;
942 }
943
944 /* Parse parameters used to create device */
945 static int
parse_bbdev_la12xx_params(struct bbdev_la12xx_params * params,const char * input_args)946 parse_bbdev_la12xx_params(struct bbdev_la12xx_params *params,
947 const char *input_args)
948 {
949 struct rte_kvargs *kvlist = NULL;
950 int ret = 0;
951
952 if (params == NULL)
953 return -EINVAL;
954 if (input_args) {
955 kvlist = rte_kvargs_parse(input_args,
956 bbdev_la12xx_valid_params);
957 if (kvlist == NULL)
958 return -EFAULT;
959
960 ret = rte_kvargs_process(kvlist, bbdev_la12xx_valid_params[0],
961 &parse_u16_arg, ¶ms->queues_num);
962 if (ret < 0)
963 goto exit;
964
965 ret = rte_kvargs_process(kvlist,
966 bbdev_la12xx_valid_params[1],
967 &parse_integer_arg,
968 ¶ms->modem_id);
969
970 if (params->modem_id >= LA12XX_MAX_MODEM) {
971 rte_bbdev_log(ERR, "Invalid modem id, must be < %u",
972 LA12XX_MAX_MODEM);
973 goto exit;
974 }
975 }
976
977 exit:
978 rte_kvargs_free(kvlist);
979 return ret;
980 }
981
982 /* Create device */
983 static int
la12xx_bbdev_create(struct rte_vdev_device * vdev,struct bbdev_la12xx_params * init_params)984 la12xx_bbdev_create(struct rte_vdev_device *vdev,
985 struct bbdev_la12xx_params *init_params)
986 {
987 struct rte_bbdev *bbdev;
988 const char *name = rte_vdev_device_name(vdev);
989 struct bbdev_la12xx_private *priv;
990 int ret;
991
992 PMD_INIT_FUNC_TRACE();
993
994 bbdev = rte_bbdev_allocate(name);
995 if (bbdev == NULL)
996 return -ENODEV;
997
998 bbdev->data->dev_private = rte_zmalloc(name,
999 sizeof(struct bbdev_la12xx_private),
1000 RTE_CACHE_LINE_SIZE);
1001 if (bbdev->data->dev_private == NULL) {
1002 rte_bbdev_release(bbdev);
1003 return -ENOMEM;
1004 }
1005
1006 priv = bbdev->data->dev_private;
1007 priv->modem_id = init_params->modem_id;
1008 /* if modem id is not configured */
1009 if (priv->modem_id == -1)
1010 priv->modem_id = bbdev->data->dev_id;
1011
1012 /* Reset Global variables */
1013 priv->num_ldpc_enc_queues = 0;
1014 priv->num_ldpc_dec_queues = 0;
1015 priv->num_valid_queues = 0;
1016 priv->max_nb_queues = init_params->queues_num;
1017
1018 rte_bbdev_log(INFO, "Setting Up %s: DevId=%d, ModemId=%d",
1019 name, bbdev->data->dev_id, priv->modem_id);
1020 ret = setup_la12xx_dev(bbdev);
1021 if (ret) {
1022 rte_bbdev_log(ERR, "IPC Setup failed for %s", name);
1023 rte_free(bbdev->data->dev_private);
1024 return ret;
1025 }
1026 bbdev->dev_ops = &pmd_ops;
1027 bbdev->device = &vdev->device;
1028 bbdev->data->socket_id = 0;
1029 bbdev->intr_handle = NULL;
1030
1031 /* register rx/tx burst functions for data path */
1032 bbdev->dequeue_enc_ops = NULL;
1033 bbdev->dequeue_dec_ops = NULL;
1034 bbdev->enqueue_enc_ops = NULL;
1035 bbdev->enqueue_dec_ops = NULL;
1036 bbdev->dequeue_ldpc_enc_ops = dequeue_enc_ops;
1037 bbdev->dequeue_ldpc_dec_ops = dequeue_dec_ops;
1038 bbdev->enqueue_ldpc_enc_ops = enqueue_enc_ops;
1039 bbdev->enqueue_ldpc_dec_ops = enqueue_dec_ops;
1040
1041 return 0;
1042 }
1043
1044 /* Initialise device */
1045 static int
la12xx_bbdev_probe(struct rte_vdev_device * vdev)1046 la12xx_bbdev_probe(struct rte_vdev_device *vdev)
1047 {
1048 struct bbdev_la12xx_params init_params = {
1049 8, -1,
1050 };
1051 const char *name;
1052 const char *input_args;
1053
1054 PMD_INIT_FUNC_TRACE();
1055
1056 if (vdev == NULL)
1057 return -EINVAL;
1058
1059 name = rte_vdev_device_name(vdev);
1060 if (name == NULL)
1061 return -EINVAL;
1062
1063 input_args = rte_vdev_device_args(vdev);
1064 parse_bbdev_la12xx_params(&init_params, input_args);
1065
1066 return la12xx_bbdev_create(vdev, &init_params);
1067 }
1068
1069 /* Uninitialise device */
1070 static int
la12xx_bbdev_remove(struct rte_vdev_device * vdev)1071 la12xx_bbdev_remove(struct rte_vdev_device *vdev)
1072 {
1073 struct rte_bbdev *bbdev;
1074 const char *name;
1075
1076 PMD_INIT_FUNC_TRACE();
1077
1078 if (vdev == NULL)
1079 return -EINVAL;
1080
1081 name = rte_vdev_device_name(vdev);
1082 if (name == NULL)
1083 return -EINVAL;
1084
1085 bbdev = rte_bbdev_get_named_dev(name);
1086 if (bbdev == NULL)
1087 return -EINVAL;
1088
1089 rte_free(bbdev->data->dev_private);
1090
1091 return rte_bbdev_release(bbdev);
1092 }
1093
1094 static struct rte_vdev_driver bbdev_la12xx_pmd_drv = {
1095 .probe = la12xx_bbdev_probe,
1096 .remove = la12xx_bbdev_remove
1097 };
1098
1099 RTE_PMD_REGISTER_VDEV(DRIVER_NAME, bbdev_la12xx_pmd_drv);
1100 RTE_PMD_REGISTER_PARAM_STRING(DRIVER_NAME,
1101 LA12XX_MAX_NB_QUEUES_ARG"=<int>"
1102 LA12XX_VDEV_MODEM_ID_ARG "=<int> ");
1103 RTE_LOG_REGISTER_DEFAULT(bbdev_la12xx_logtype, NOTICE);
1104