1 /*
2 * BSD LICENSE
3 *
4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Cavium, Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33 /*$FreeBSD$*/
34
35 #include "lio_bsd.h"
36 #include "lio_common.h"
37 #include "lio_droq.h"
38 #include "lio_iq.h"
39 #include "lio_response_manager.h"
40 #include "lio_device.h"
41 #include "lio_main.h"
42 #include "lio_network.h"
43 #include "cn23xx_pf_device.h"
44 #include "lio_rxtx.h"
45
46 struct lio_iq_post_status {
47 int status;
48 int index;
49 };
50
51 static void lio_check_db_timeout(void *arg, int pending);
52 static void __lio_check_db_timeout(struct octeon_device *oct,
53 uint64_t iq_no);
54
55 /* Return 0 on success, 1 on failure */
56 int
lio_init_instr_queue(struct octeon_device * oct,union octeon_txpciq txpciq,uint32_t num_descs)57 lio_init_instr_queue(struct octeon_device *oct, union octeon_txpciq txpciq,
58 uint32_t num_descs)
59 {
60 struct lio_instr_queue *iq;
61 struct lio_iq_config *conf = NULL;
62 struct lio_tq *db_tq;
63 struct lio_request_list *request_buf;
64 bus_size_t max_size;
65 uint32_t iq_no = (uint32_t)txpciq.s.q_no;
66 uint32_t q_size;
67 int error, i;
68
69 if (LIO_CN23XX_PF(oct))
70 conf = &(LIO_GET_IQ_CFG(LIO_CHIP_CONF(oct, cn23xx_pf)));
71 if (conf == NULL) {
72 lio_dev_err(oct, "Unsupported Chip %x\n", oct->chip_id);
73 return (1);
74 }
75
76 q_size = (uint32_t)conf->instr_type * num_descs;
77 iq = oct->instr_queue[iq_no];
78 iq->oct_dev = oct;
79
80 max_size = LIO_CN23XX_PKI_MAX_FRAME_SIZE * num_descs;
81
82 error = bus_dma_tag_create(bus_get_dma_tag(oct->device), /* parent */
83 1, 0, /* alignment, bounds */
84 BUS_SPACE_MAXADDR, /* lowaddr */
85 BUS_SPACE_MAXADDR, /* highaddr */
86 NULL, NULL, /* filter, filterarg */
87 max_size, /* maxsize */
88 LIO_MAX_SG, /* nsegments */
89 PAGE_SIZE, /* maxsegsize */
90 0, /* flags */
91 NULL, /* lockfunc */
92 NULL, /* lockfuncarg */
93 &iq->txtag);
94 if (error) {
95 lio_dev_err(oct, "Cannot allocate memory for instr queue %d\n",
96 iq_no);
97 return (1);
98 }
99
100 iq->base_addr = lio_dma_alloc(q_size, (vm_paddr_t *)&iq->base_addr_dma);
101 if (!iq->base_addr) {
102 lio_dev_err(oct, "Cannot allocate memory for instr queue %d\n",
103 iq_no);
104 return (1);
105 }
106
107 iq->max_count = num_descs;
108
109 /*
110 * Initialize a list to holds requests that have been posted to
111 * Octeon but has yet to be fetched by octeon
112 */
113 iq->request_list = malloc(sizeof(*iq->request_list) * num_descs,
114 M_DEVBUF, M_NOWAIT | M_ZERO);
115 if (iq->request_list == NULL) {
116 lio_dev_err(oct, "Alloc failed for IQ[%d] nr free list\n",
117 iq_no);
118 return (1);
119 }
120
121 lio_dev_dbg(oct, "IQ[%d]: base: %p basedma: %llx count: %d\n",
122 iq_no, iq->base_addr, LIO_CAST64(iq->base_addr_dma),
123 iq->max_count);
124
125 /* Create the descriptor buffer dma maps */
126 request_buf = iq->request_list;
127 for (i = 0; i < num_descs; i++, request_buf++) {
128 error = bus_dmamap_create(iq->txtag, 0, &request_buf->map);
129 if (error) {
130 lio_dev_err(oct, "Unable to create TX DMA map\n");
131 return (1);
132 }
133 }
134
135 iq->txpciq.txpciq64 = txpciq.txpciq64;
136 iq->fill_cnt = 0;
137 iq->host_write_index = 0;
138 iq->octeon_read_index = 0;
139 iq->flush_index = 0;
140 iq->last_db_time = 0;
141 iq->db_timeout = (uint32_t)conf->db_timeout;
142 atomic_store_rel_int(&iq->instr_pending, 0);
143
144 /* Initialize the lock for this instruction queue */
145 mtx_init(&iq->lock, "Tx_lock", NULL, MTX_DEF);
146 mtx_init(&iq->post_lock, "iq_post_lock", NULL, MTX_DEF);
147 mtx_init(&iq->enq_lock, "enq_lock", NULL, MTX_DEF);
148
149 mtx_init(&iq->iq_flush_running_lock, "iq_flush_running_lock", NULL,
150 MTX_DEF);
151
152 oct->io_qmask.iq |= BIT_ULL(iq_no);
153
154 /* Set the 32B/64B mode for each input queue */
155 oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);
156 iq->iqcmd_64B = (conf->instr_type == 64);
157
158 oct->fn_list.setup_iq_regs(oct, iq_no);
159
160 db_tq = &oct->check_db_tq[iq_no];
161 db_tq->tq = taskqueue_create("lio_check_db_timeout", M_WAITOK,
162 taskqueue_thread_enqueue, &db_tq->tq);
163 if (db_tq->tq == NULL) {
164 lio_dev_err(oct, "check db wq create failed for iq %d\n",
165 iq_no);
166 return (1);
167 }
168
169 TIMEOUT_TASK_INIT(db_tq->tq, &db_tq->work, 0, lio_check_db_timeout,
170 (void *)db_tq);
171 db_tq->ctxul = iq_no;
172 db_tq->ctxptr = oct;
173
174 taskqueue_start_threads(&db_tq->tq, 1, PI_NET,
175 "lio%d_check_db_timeout:%d",
176 oct->octeon_id, iq_no);
177 taskqueue_enqueue_timeout(db_tq->tq, &db_tq->work, 1);
178
179 /* Allocate a buf ring */
180 oct->instr_queue[iq_no]->br =
181 buf_ring_alloc(LIO_BR_SIZE, M_DEVBUF, M_WAITOK,
182 &oct->instr_queue[iq_no]->enq_lock);
183 if (oct->instr_queue[iq_no]->br == NULL) {
184 lio_dev_err(oct, "Critical Failure setting up buf ring\n");
185 return (1);
186 }
187
188 return (0);
189 }
190
191 int
lio_delete_instr_queue(struct octeon_device * oct,uint32_t iq_no)192 lio_delete_instr_queue(struct octeon_device *oct, uint32_t iq_no)
193 {
194 struct lio_instr_queue *iq = oct->instr_queue[iq_no];
195 struct lio_request_list *request_buf;
196 struct lio_mbuf_free_info *finfo;
197 uint64_t desc_size = 0, q_size;
198 int i;
199
200 lio_dev_dbg(oct, "%s[%d]\n", __func__, iq_no);
201
202 if (oct->check_db_tq[iq_no].tq != NULL) {
203 while (taskqueue_cancel_timeout(oct->check_db_tq[iq_no].tq,
204 &oct->check_db_tq[iq_no].work,
205 NULL))
206 taskqueue_drain_timeout(oct->check_db_tq[iq_no].tq,
207 &oct->check_db_tq[iq_no].work);
208 taskqueue_free(oct->check_db_tq[iq_no].tq);
209 oct->check_db_tq[iq_no].tq = NULL;
210 }
211
212 if (LIO_CN23XX_PF(oct))
213 desc_size =
214 LIO_GET_IQ_INSTR_TYPE_CFG(LIO_CHIP_CONF(oct, cn23xx_pf));
215
216 request_buf = iq->request_list;
217 for (i = 0; i < iq->max_count; i++, request_buf++) {
218 if ((request_buf->reqtype == LIO_REQTYPE_NORESP_NET) ||
219 (request_buf->reqtype == LIO_REQTYPE_NORESP_NET_SG)) {
220 if (request_buf->buf != NULL) {
221 finfo = request_buf->buf;
222 bus_dmamap_sync(iq->txtag, request_buf->map,
223 BUS_DMASYNC_POSTWRITE);
224 bus_dmamap_unload(iq->txtag,
225 request_buf->map);
226 m_freem(finfo->mb);
227 request_buf->buf = NULL;
228 if (request_buf->map != NULL) {
229 bus_dmamap_destroy(iq->txtag,
230 request_buf->map);
231 request_buf->map = NULL;
232 }
233 } else if (request_buf->map != NULL) {
234 bus_dmamap_unload(iq->txtag, request_buf->map);
235 bus_dmamap_destroy(iq->txtag, request_buf->map);
236 request_buf->map = NULL;
237 }
238 }
239 }
240
241 if (iq->br != NULL) {
242 buf_ring_free(iq->br, M_DEVBUF);
243 iq->br = NULL;
244 }
245
246 if (iq->request_list != NULL) {
247 free(iq->request_list, M_DEVBUF);
248 iq->request_list = NULL;
249 }
250
251 if (iq->txtag != NULL) {
252 bus_dma_tag_destroy(iq->txtag);
253 iq->txtag = NULL;
254 }
255
256 if (iq->base_addr) {
257 q_size = iq->max_count * desc_size;
258 lio_dma_free((uint32_t)q_size, iq->base_addr);
259
260 oct->io_qmask.iq &= ~(1ULL << iq_no);
261 bzero(oct->instr_queue[iq_no], sizeof(struct lio_instr_queue));
262 oct->num_iqs--;
263
264 return (0);
265 }
266
267 return (1);
268 }
269
270 /* Return 0 on success, 1 on failure */
271 int
lio_setup_iq(struct octeon_device * oct,int ifidx,int q_index,union octeon_txpciq txpciq,uint32_t num_descs)272 lio_setup_iq(struct octeon_device *oct, int ifidx, int q_index,
273 union octeon_txpciq txpciq, uint32_t num_descs)
274 {
275 uint32_t iq_no = (uint32_t)txpciq.s.q_no;
276
277 if (oct->instr_queue[iq_no]->oct_dev != NULL) {
278 lio_dev_dbg(oct, "IQ is in use. Cannot create the IQ: %d again\n",
279 iq_no);
280 oct->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64;
281 return (0);
282 }
283
284 oct->instr_queue[iq_no]->q_index = q_index;
285 oct->instr_queue[iq_no]->ifidx = ifidx;
286
287 if (lio_init_instr_queue(oct, txpciq, num_descs)) {
288 lio_delete_instr_queue(oct, iq_no);
289 return (1);
290 }
291
292 oct->num_iqs++;
293 if (oct->fn_list.enable_io_queues(oct))
294 return (1);
295
296 return (0);
297 }
298
299 int
lio_wait_for_instr_fetch(struct octeon_device * oct)300 lio_wait_for_instr_fetch(struct octeon_device *oct)
301 {
302 int i, retry = 1000, pending, instr_cnt = 0;
303
304 do {
305 instr_cnt = 0;
306
307 for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {
308 if (!(oct->io_qmask.iq & BIT_ULL(i)))
309 continue;
310 pending = atomic_load_acq_int(
311 &oct->instr_queue[i]->instr_pending);
312 if (pending)
313 __lio_check_db_timeout(oct, i);
314 instr_cnt += pending;
315 }
316
317 if (instr_cnt == 0)
318 break;
319
320 lio_sleep_timeout(1);
321
322 } while (retry-- && instr_cnt);
323
324 return (instr_cnt);
325 }
326
327 static inline void
lio_ring_doorbell(struct octeon_device * oct,struct lio_instr_queue * iq)328 lio_ring_doorbell(struct octeon_device *oct, struct lio_instr_queue *iq)
329 {
330
331 if (atomic_load_acq_int(&oct->status) == LIO_DEV_RUNNING) {
332 lio_write_csr32(oct, iq->doorbell_reg, iq->fill_cnt);
333 /* make sure doorbell write goes through */
334 __compiler_membar();
335 iq->fill_cnt = 0;
336 iq->last_db_time = ticks;
337 return;
338 }
339 }
340
341 static inline void
__lio_copy_cmd_into_iq(struct lio_instr_queue * iq,uint8_t * cmd)342 __lio_copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd)
343 {
344 uint8_t *iqptr, cmdsize;
345
346 cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
347 iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
348
349 memcpy(iqptr, cmd, cmdsize);
350 }
351
352 static inline struct lio_iq_post_status
__lio_post_command2(struct lio_instr_queue * iq,uint8_t * cmd)353 __lio_post_command2(struct lio_instr_queue *iq, uint8_t *cmd)
354 {
355 struct lio_iq_post_status st;
356
357 st.status = LIO_IQ_SEND_OK;
358
359 /*
360 * This ensures that the read index does not wrap around to the same
361 * position if queue gets full before Octeon could fetch any instr.
362 */
363 if (atomic_load_acq_int(&iq->instr_pending) >=
364 (int32_t)(iq->max_count - 1)) {
365 st.status = LIO_IQ_SEND_FAILED;
366 st.index = -1;
367 return (st);
368 }
369
370 if (atomic_load_acq_int(&iq->instr_pending) >=
371 (int32_t)(iq->max_count - 2))
372 st.status = LIO_IQ_SEND_STOP;
373
374 __lio_copy_cmd_into_iq(iq, cmd);
375
376 /* "index" is returned, host_write_index is modified. */
377 st.index = iq->host_write_index;
378 iq->host_write_index = lio_incr_index(iq->host_write_index, 1,
379 iq->max_count);
380 iq->fill_cnt++;
381
382 /*
383 * Flush the command into memory. We need to be sure the data is in
384 * memory before indicating that the instruction is pending.
385 */
386 wmb();
387
388 atomic_add_int(&iq->instr_pending, 1);
389
390 return (st);
391 }
392
393 static inline void
__lio_add_to_request_list(struct lio_instr_queue * iq,int idx,void * buf,int reqtype)394 __lio_add_to_request_list(struct lio_instr_queue *iq, int idx, void *buf,
395 int reqtype)
396 {
397
398 iq->request_list[idx].buf = buf;
399 iq->request_list[idx].reqtype = reqtype;
400 }
401
402 /* Can only run in process context */
403 int
lio_process_iq_request_list(struct octeon_device * oct,struct lio_instr_queue * iq,uint32_t budget)404 lio_process_iq_request_list(struct octeon_device *oct,
405 struct lio_instr_queue *iq, uint32_t budget)
406 {
407 struct lio_soft_command *sc;
408 struct octeon_instr_irh *irh = NULL;
409 struct lio_mbuf_free_info *finfo;
410 void *buf;
411 uint32_t inst_count = 0;
412 uint32_t old = iq->flush_index;
413 int reqtype;
414
415 while (old != iq->octeon_read_index) {
416 reqtype = iq->request_list[old].reqtype;
417 buf = iq->request_list[old].buf;
418 finfo = buf;
419
420 if (reqtype == LIO_REQTYPE_NONE)
421 goto skip_this;
422
423 switch (reqtype) {
424 case LIO_REQTYPE_NORESP_NET:
425 lio_free_mbuf(iq, buf);
426 break;
427 case LIO_REQTYPE_NORESP_NET_SG:
428 lio_free_sgmbuf(iq, buf);
429 break;
430 case LIO_REQTYPE_RESP_NET:
431 case LIO_REQTYPE_SOFT_COMMAND:
432 sc = buf;
433 if (LIO_CN23XX_PF(oct))
434 irh = (struct octeon_instr_irh *)
435 &sc->cmd.cmd3.irh;
436 if (irh->rflag) {
437 /*
438 * We're expecting a response from Octeon.
439 * It's up to lio_process_ordered_list() to
440 * process sc. Add sc to the ordered soft
441 * command response list because we expect
442 * a response from Octeon.
443 */
444 mtx_lock(&oct->response_list
445 [LIO_ORDERED_SC_LIST].lock);
446 atomic_add_int(&oct->response_list
447 [LIO_ORDERED_SC_LIST].
448 pending_req_count, 1);
449 STAILQ_INSERT_TAIL(&oct->response_list
450 [LIO_ORDERED_SC_LIST].
451 head, &sc->node, entries);
452 mtx_unlock(&oct->response_list
453 [LIO_ORDERED_SC_LIST].lock);
454 } else {
455 if (sc->callback != NULL) {
456 /* This callback must not sleep */
457 sc->callback(oct, LIO_REQUEST_DONE,
458 sc->callback_arg);
459 }
460 }
461
462 break;
463 default:
464 lio_dev_err(oct, "%s Unknown reqtype: %d buf: %p at idx %d\n",
465 __func__, reqtype, buf, old);
466 }
467
468 iq->request_list[old].buf = NULL;
469 iq->request_list[old].reqtype = 0;
470
471 skip_this:
472 inst_count++;
473 old = lio_incr_index(old, 1, iq->max_count);
474
475 if ((budget) && (inst_count >= budget))
476 break;
477 }
478
479 iq->flush_index = old;
480
481 return (inst_count);
482 }
483
484 /* Can only be called from process context */
485 int
lio_flush_iq(struct octeon_device * oct,struct lio_instr_queue * iq,uint32_t budget)486 lio_flush_iq(struct octeon_device *oct, struct lio_instr_queue *iq,
487 uint32_t budget)
488 {
489 uint32_t inst_processed = 0;
490 uint32_t tot_inst_processed = 0;
491 int tx_done = 1;
492
493 if (!mtx_trylock(&iq->iq_flush_running_lock))
494 return (tx_done);
495
496 mtx_lock(&iq->lock);
497
498 iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);
499
500 do {
501 /* Process any outstanding IQ packets. */
502 if (iq->flush_index == iq->octeon_read_index)
503 break;
504
505 if (budget)
506 inst_processed =
507 lio_process_iq_request_list(oct, iq,
508 budget -
509 tot_inst_processed);
510 else
511 inst_processed =
512 lio_process_iq_request_list(oct, iq, 0);
513
514 if (inst_processed) {
515 atomic_subtract_int(&iq->instr_pending, inst_processed);
516 iq->stats.instr_processed += inst_processed;
517 }
518 tot_inst_processed += inst_processed;
519 inst_processed = 0;
520
521 } while (tot_inst_processed < budget);
522
523 if (budget && (tot_inst_processed >= budget))
524 tx_done = 0;
525
526 iq->last_db_time = ticks;
527
528 mtx_unlock(&iq->lock);
529
530 mtx_unlock(&iq->iq_flush_running_lock);
531
532 return (tx_done);
533 }
534
535 /*
536 * Process instruction queue after timeout.
537 * This routine gets called from a taskqueue or when removing the module.
538 */
539 static void
__lio_check_db_timeout(struct octeon_device * oct,uint64_t iq_no)540 __lio_check_db_timeout(struct octeon_device *oct, uint64_t iq_no)
541 {
542 struct lio_instr_queue *iq;
543 uint64_t next_time;
544
545 if (oct == NULL)
546 return;
547
548 iq = oct->instr_queue[iq_no];
549 if (iq == NULL)
550 return;
551
552 if (atomic_load_acq_int(&iq->instr_pending)) {
553 /* If ticks - last_db_time < db_timeout do nothing */
554 next_time = iq->last_db_time + lio_ms_to_ticks(iq->db_timeout);
555 if (!lio_check_timeout(ticks, next_time))
556 return;
557
558 iq->last_db_time = ticks;
559
560 /* Flush the instruction queue */
561 lio_flush_iq(oct, iq, 0);
562
563 lio_enable_irq(NULL, iq);
564 }
565
566 if (oct->props.ifp != NULL && iq->br != NULL) {
567 if (mtx_trylock(&iq->enq_lock)) {
568 if (!drbr_empty(oct->props.ifp, iq->br))
569 lio_mq_start_locked(oct->props.ifp, iq);
570
571 mtx_unlock(&iq->enq_lock);
572 }
573 }
574 }
575
576 /*
577 * Called by the Poll thread at regular intervals to check the instruction
578 * queue for commands to be posted and for commands that were fetched by Octeon.
579 */
580 static void
lio_check_db_timeout(void * arg,int pending)581 lio_check_db_timeout(void *arg, int pending)
582 {
583 struct lio_tq *db_tq = (struct lio_tq *)arg;
584 struct octeon_device *oct = db_tq->ctxptr;
585 uint64_t iq_no = db_tq->ctxul;
586 uint32_t delay = 10;
587
588 __lio_check_db_timeout(oct, iq_no);
589 taskqueue_enqueue_timeout(db_tq->tq, &db_tq->work,
590 lio_ms_to_ticks(delay));
591 }
592
593 int
lio_send_command(struct octeon_device * oct,uint32_t iq_no,uint32_t force_db,void * cmd,void * buf,uint32_t datasize,uint32_t reqtype)594 lio_send_command(struct octeon_device *oct, uint32_t iq_no,
595 uint32_t force_db, void *cmd, void *buf,
596 uint32_t datasize, uint32_t reqtype)
597 {
598 struct lio_iq_post_status st;
599 struct lio_instr_queue *iq = oct->instr_queue[iq_no];
600
601 /*
602 * Get the lock and prevent other tasks and tx interrupt handler
603 * from running.
604 */
605 mtx_lock(&iq->post_lock);
606
607 st = __lio_post_command2(iq, cmd);
608
609 if (st.status != LIO_IQ_SEND_FAILED) {
610 __lio_add_to_request_list(iq, st.index, buf, reqtype);
611 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);
612 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);
613
614 if (force_db || (st.status == LIO_IQ_SEND_STOP))
615 lio_ring_doorbell(oct, iq);
616 } else {
617 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);
618 }
619
620 mtx_unlock(&iq->post_lock);
621
622 /*
623 * This is only done here to expedite packets being flushed for
624 * cases where there are no IQ completion interrupts.
625 */
626
627 return (st.status);
628 }
629
630 void
lio_prepare_soft_command(struct octeon_device * oct,struct lio_soft_command * sc,uint8_t opcode,uint8_t subcode,uint32_t irh_ossp,uint64_t ossp0,uint64_t ossp1)631 lio_prepare_soft_command(struct octeon_device *oct, struct lio_soft_command *sc,
632 uint8_t opcode, uint8_t subcode, uint32_t irh_ossp,
633 uint64_t ossp0, uint64_t ossp1)
634 {
635 struct lio_config *lio_cfg;
636 struct octeon_instr_ih3 *ih3;
637 struct octeon_instr_pki_ih3 *pki_ih3;
638 struct octeon_instr_irh *irh;
639 struct octeon_instr_rdp *rdp;
640
641 KASSERT(opcode <= 15, ("%s, %d, opcode > 15", __func__, __LINE__));
642 KASSERT(subcode <= 127, ("%s, %d, opcode > 127", __func__, __LINE__));
643
644 lio_cfg = lio_get_conf(oct);
645
646 if (LIO_CN23XX_PF(oct)) {
647 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
648
649 ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;
650
651 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
652
653 pki_ih3->w = 1;
654 pki_ih3->raw = 1;
655 pki_ih3->utag = 1;
656 pki_ih3->uqpg = oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
657 pki_ih3->utt = 1;
658 pki_ih3->tag = LIO_CONTROL;
659 pki_ih3->tagtype = LIO_ATOMIC_TAG;
660 pki_ih3->qpg = oct->instr_queue[sc->iq_no]->txpciq.s.qpg;
661 pki_ih3->pm = 0x7;
662 pki_ih3->sl = 8;
663
664 if (sc->datasize)
665 ih3->dlengsz = sc->datasize;
666
667 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
668 irh->opcode = opcode;
669 irh->subcode = subcode;
670
671 /* opcode/subcode specific parameters (ossp) */
672 irh->ossp = irh_ossp;
673 sc->cmd.cmd3.ossp[0] = ossp0;
674 sc->cmd.cmd3.ossp[1] = ossp1;
675
676 if (sc->rdatasize) {
677 rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
678 rdp->pcie_port = oct->pcie_port;
679 rdp->rlen = sc->rdatasize;
680
681 irh->rflag = 1;
682 /* PKI IH3 */
683 /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */
684 ih3->fsz = LIO_SOFTCMDRESP_IH3;
685 } else {
686 irh->rflag = 0;
687 /* PKI IH3 */
688 /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
689 ih3->fsz = LIO_PCICMD_O3;
690 }
691 }
692 }
693
694 int
lio_send_soft_command(struct octeon_device * oct,struct lio_soft_command * sc)695 lio_send_soft_command(struct octeon_device *oct, struct lio_soft_command *sc)
696 {
697 struct octeon_instr_ih3 *ih3;
698 struct octeon_instr_irh *irh;
699 uint32_t len = 0;
700
701 if (LIO_CN23XX_PF(oct)) {
702 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
703 if (ih3->dlengsz) {
704 KASSERT(sc->dmadptr, ("%s, %d, sc->dmadptr is NULL",
705 __func__, __LINE__));
706 sc->cmd.cmd3.dptr = sc->dmadptr;
707 }
708
709 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
710 if (irh->rflag) {
711 KASSERT(sc->dmarptr, ("%s, %d, sc->dmarptr is NULL",
712 __func__, __LINE__));
713 KASSERT(sc->status_word, ("%s, %d, sc->status_word is NULL",
714 __func__, __LINE__));
715 *sc->status_word = COMPLETION_WORD_INIT;
716 sc->cmd.cmd3.rptr = sc->dmarptr;
717 }
718 len = (uint32_t)ih3->dlengsz;
719 }
720 if (sc->wait_time)
721 sc->timeout = ticks + lio_ms_to_ticks(sc->wait_time);
722
723 return (lio_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,
724 len, LIO_REQTYPE_SOFT_COMMAND));
725 }
726
727 int
lio_setup_sc_buffer_pool(struct octeon_device * oct)728 lio_setup_sc_buffer_pool(struct octeon_device *oct)
729 {
730 struct lio_soft_command *sc;
731 uint64_t dma_addr;
732 int i;
733
734 STAILQ_INIT(&oct->sc_buf_pool.head);
735 mtx_init(&oct->sc_buf_pool.lock, "sc_pool_lock", NULL, MTX_DEF);
736 atomic_store_rel_int(&oct->sc_buf_pool.alloc_buf_count, 0);
737
738 for (i = 0; i < LIO_MAX_SOFT_COMMAND_BUFFERS; i++) {
739 sc = (struct lio_soft_command *)
740 lio_dma_alloc(LIO_SOFT_COMMAND_BUFFER_SIZE, (vm_paddr_t *)&dma_addr);
741 if (sc == NULL) {
742 lio_free_sc_buffer_pool(oct);
743 return (1);
744 }
745
746 sc->dma_addr = dma_addr;
747 sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE;
748
749 STAILQ_INSERT_TAIL(&oct->sc_buf_pool.head, &sc->node, entries);
750 }
751
752 return (0);
753 }
754
755 int
lio_free_sc_buffer_pool(struct octeon_device * oct)756 lio_free_sc_buffer_pool(struct octeon_device *oct)
757 {
758 struct lio_stailq_node *tmp, *tmp2;
759 struct lio_soft_command *sc;
760
761 mtx_lock(&oct->sc_buf_pool.lock);
762
763 STAILQ_FOREACH_SAFE(tmp, &oct->sc_buf_pool.head, entries, tmp2) {
764 sc = LIO_STAILQ_FIRST_ENTRY(&oct->sc_buf_pool.head,
765 struct lio_soft_command, node);
766
767 STAILQ_REMOVE_HEAD(&oct->sc_buf_pool.head, entries);
768
769 lio_dma_free(sc->size, sc);
770 }
771
772 STAILQ_INIT(&oct->sc_buf_pool.head);
773
774 mtx_unlock(&oct->sc_buf_pool.lock);
775
776 return (0);
777 }
778
779 struct lio_soft_command *
lio_alloc_soft_command(struct octeon_device * oct,uint32_t datasize,uint32_t rdatasize,uint32_t ctxsize)780 lio_alloc_soft_command(struct octeon_device *oct, uint32_t datasize,
781 uint32_t rdatasize, uint32_t ctxsize)
782 {
783 struct lio_soft_command *sc = NULL;
784 struct lio_stailq_node *tmp;
785 uint64_t dma_addr;
786 uint32_t size;
787 uint32_t offset = sizeof(struct lio_soft_command);
788
789 KASSERT((offset + datasize + rdatasize + ctxsize) <=
790 LIO_SOFT_COMMAND_BUFFER_SIZE,
791 ("%s, %d, offset + datasize + rdatasize + ctxsize > LIO_SOFT_COMMAND_BUFFER_SIZE",
792 __func__, __LINE__));
793
794 mtx_lock(&oct->sc_buf_pool.lock);
795
796 if (STAILQ_EMPTY(&oct->sc_buf_pool.head)) {
797 mtx_unlock(&oct->sc_buf_pool.lock);
798 return (NULL);
799 }
800 tmp = STAILQ_LAST(&oct->sc_buf_pool.head, lio_stailq_node, entries);
801
802 STAILQ_REMOVE(&oct->sc_buf_pool.head, tmp, lio_stailq_node, entries);
803
804 atomic_add_int(&oct->sc_buf_pool.alloc_buf_count, 1);
805
806 mtx_unlock(&oct->sc_buf_pool.lock);
807
808 sc = (struct lio_soft_command *)tmp;
809
810 dma_addr = sc->dma_addr;
811 size = sc->size;
812
813 bzero(sc, sc->size);
814
815 sc->dma_addr = dma_addr;
816 sc->size = size;
817
818 if (ctxsize) {
819 sc->ctxptr = (uint8_t *)sc + offset;
820 sc->ctxsize = ctxsize;
821 }
822
823 /* Start data at 128 byte boundary */
824 offset = (offset + ctxsize + 127) & 0xffffff80;
825
826 if (datasize) {
827 sc->virtdptr = (uint8_t *)sc + offset;
828 sc->dmadptr = dma_addr + offset;
829 sc->datasize = datasize;
830 }
831 /* Start rdata at 128 byte boundary */
832 offset = (offset + datasize + 127) & 0xffffff80;
833
834 if (rdatasize) {
835 KASSERT(rdatasize >= 16, ("%s, %d, rdatasize < 16", __func__,
836 __LINE__));
837 sc->virtrptr = (uint8_t *)sc + offset;
838 sc->dmarptr = dma_addr + offset;
839 sc->rdatasize = rdatasize;
840 sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) +
841 rdatasize - 8);
842 }
843 return (sc);
844 }
845
846 void
lio_free_soft_command(struct octeon_device * oct,struct lio_soft_command * sc)847 lio_free_soft_command(struct octeon_device *oct,
848 struct lio_soft_command *sc)
849 {
850
851 mtx_lock(&oct->sc_buf_pool.lock);
852
853 STAILQ_INSERT_TAIL(&oct->sc_buf_pool.head, &sc->node, entries);
854
855 atomic_subtract_int(&oct->sc_buf_pool.alloc_buf_count, 1);
856
857 mtx_unlock(&oct->sc_buf_pool.lock);
858 }
859