1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4
5 #include <rte_cryptodev.h>
6 #include <rte_malloc.h>
7
8 #include "rte_cryptodev_scheduler_operations.h"
9 #include "scheduler_pmd_private.h"
10
11 #define PRIMARY_WORKER_IDX 0
12 #define SECONDARY_WORKER_IDX 1
13 #define NB_FAILOVER_WORKERS 2
14 #define WORKER_SWITCH_MASK (0x01)
15
16 struct fo_scheduler_qp_ctx {
17 struct scheduler_worker primary_worker;
18 struct scheduler_worker secondary_worker;
19
20 uint8_t deq_idx;
21 };
22
23 static __rte_always_inline uint16_t
failover_worker_enqueue(struct scheduler_worker * worker,struct rte_crypto_op ** ops,uint16_t nb_ops)24 failover_worker_enqueue(struct scheduler_worker *worker,
25 struct rte_crypto_op **ops, uint16_t nb_ops)
26 {
27 uint16_t i, processed_ops;
28
29 for (i = 0; i < nb_ops && i < 4; i++)
30 rte_prefetch0(ops[i]->sym->session);
31
32 processed_ops = rte_cryptodev_enqueue_burst(worker->dev_id,
33 worker->qp_id, ops, nb_ops);
34 worker->nb_inflight_cops += processed_ops;
35
36 return processed_ops;
37 }
38
39 static uint16_t
schedule_enqueue(void * qp,struct rte_crypto_op ** ops,uint16_t nb_ops)40 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
41 {
42 struct fo_scheduler_qp_ctx *qp_ctx =
43 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
44 uint16_t enqueued_ops;
45
46 if (unlikely(nb_ops == 0))
47 return 0;
48
49 enqueued_ops = failover_worker_enqueue(&qp_ctx->primary_worker,
50 ops, nb_ops);
51
52 if (enqueued_ops < nb_ops)
53 enqueued_ops += failover_worker_enqueue(
54 &qp_ctx->secondary_worker,
55 &ops[enqueued_ops],
56 nb_ops - enqueued_ops);
57
58 return enqueued_ops;
59 }
60
61
62 static uint16_t
schedule_enqueue_ordering(void * qp,struct rte_crypto_op ** ops,uint16_t nb_ops)63 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops,
64 uint16_t nb_ops)
65 {
66 struct rte_ring *order_ring =
67 ((struct scheduler_qp_ctx *)qp)->order_ring;
68 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
69 nb_ops);
70 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
71 nb_ops_to_enq);
72
73 scheduler_order_insert(order_ring, ops, nb_ops_enqd);
74
75 return nb_ops_enqd;
76 }
77
78 static uint16_t
schedule_dequeue(void * qp,struct rte_crypto_op ** ops,uint16_t nb_ops)79 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
80 {
81 struct fo_scheduler_qp_ctx *qp_ctx =
82 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
83 struct scheduler_worker *workers[NB_FAILOVER_WORKERS] = {
84 &qp_ctx->primary_worker, &qp_ctx->secondary_worker};
85 struct scheduler_worker *worker = workers[qp_ctx->deq_idx];
86 uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0;
87
88 if (worker->nb_inflight_cops) {
89 nb_deq_ops = rte_cryptodev_dequeue_burst(worker->dev_id,
90 worker->qp_id, ops, nb_ops);
91 worker->nb_inflight_cops -= nb_deq_ops;
92 }
93
94 qp_ctx->deq_idx = (~qp_ctx->deq_idx) & WORKER_SWITCH_MASK;
95
96 if (nb_deq_ops == nb_ops)
97 return nb_deq_ops;
98
99 worker = workers[qp_ctx->deq_idx];
100
101 if (worker->nb_inflight_cops) {
102 nb_deq_ops2 = rte_cryptodev_dequeue_burst(worker->dev_id,
103 worker->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops);
104 worker->nb_inflight_cops -= nb_deq_ops2;
105 }
106
107 return nb_deq_ops + nb_deq_ops2;
108 }
109
110 static uint16_t
schedule_dequeue_ordering(void * qp,struct rte_crypto_op ** ops,uint16_t nb_ops)111 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops,
112 uint16_t nb_ops)
113 {
114 struct rte_ring *order_ring =
115 ((struct scheduler_qp_ctx *)qp)->order_ring;
116
117 schedule_dequeue(qp, ops, nb_ops);
118
119 return scheduler_order_drain(order_ring, ops, nb_ops);
120 }
121
122 static int
worker_attach(__rte_unused struct rte_cryptodev * dev,__rte_unused uint8_t worker_id)123 worker_attach(__rte_unused struct rte_cryptodev *dev,
124 __rte_unused uint8_t worker_id)
125 {
126 return 0;
127 }
128
129 static int
worker_detach(__rte_unused struct rte_cryptodev * dev,__rte_unused uint8_t worker_id)130 worker_detach(__rte_unused struct rte_cryptodev *dev,
131 __rte_unused uint8_t worker_id)
132 {
133 return 0;
134 }
135
136 static int
scheduler_start(struct rte_cryptodev * dev)137 scheduler_start(struct rte_cryptodev *dev)
138 {
139 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
140 uint16_t i;
141
142 if (sched_ctx->nb_workers < 2) {
143 CR_SCHED_LOG(ERR, "Number of workers shall no less than 2");
144 return -ENOMEM;
145 }
146
147 if (sched_ctx->reordering_enabled) {
148 dev->enqueue_burst = schedule_enqueue_ordering;
149 dev->dequeue_burst = schedule_dequeue_ordering;
150 } else {
151 dev->enqueue_burst = schedule_enqueue;
152 dev->dequeue_burst = schedule_dequeue;
153 }
154
155 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
156 struct fo_scheduler_qp_ctx *qp_ctx =
157 ((struct scheduler_qp_ctx *)
158 dev->data->queue_pairs[i])->private_qp_ctx;
159
160 rte_memcpy(&qp_ctx->primary_worker,
161 &sched_ctx->workers[PRIMARY_WORKER_IDX],
162 sizeof(struct scheduler_worker));
163 rte_memcpy(&qp_ctx->secondary_worker,
164 &sched_ctx->workers[SECONDARY_WORKER_IDX],
165 sizeof(struct scheduler_worker));
166 }
167
168 return 0;
169 }
170
171 static int
scheduler_stop(__rte_unused struct rte_cryptodev * dev)172 scheduler_stop(__rte_unused struct rte_cryptodev *dev)
173 {
174 return 0;
175 }
176
177 static int
scheduler_config_qp(struct rte_cryptodev * dev,uint16_t qp_id)178 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
179 {
180 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
181 struct fo_scheduler_qp_ctx *fo_qp_ctx;
182
183 fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0,
184 rte_socket_id());
185 if (!fo_qp_ctx) {
186 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
187 return -ENOMEM;
188 }
189
190 qp_ctx->private_qp_ctx = (void *)fo_qp_ctx;
191
192 return 0;
193 }
194
195 static int
scheduler_create_private_ctx(__rte_unused struct rte_cryptodev * dev)196 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
197 {
198 return 0;
199 }
200
201 static struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
202 worker_attach,
203 worker_detach,
204 scheduler_start,
205 scheduler_stop,
206 scheduler_config_qp,
207 scheduler_create_private_ctx,
208 NULL, /* option_set */
209 NULL /*option_get */
210 };
211
212 static struct rte_cryptodev_scheduler fo_scheduler = {
213 .name = "failover-scheduler",
214 .description = "scheduler which enqueues to the primary worker, "
215 "and only then enqueues to the secondary worker "
216 "upon failing on enqueuing to primary",
217 .mode = CDEV_SCHED_MODE_FAILOVER,
218 .ops = &scheduler_fo_ops
219 };
220
221 struct rte_cryptodev_scheduler *crypto_scheduler_failover = &fo_scheduler;
222