1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_cryptodev.h> 34 #include <rte_malloc.h> 35 36 #include "rte_cryptodev_scheduler_operations.h" 37 #include "scheduler_pmd_private.h" 38 39 #define PRIMARY_SLAVE_IDX 0 40 #define SECONDARY_SLAVE_IDX 1 41 #define NB_FAILOVER_SLAVES 2 42 #define SLAVE_SWITCH_MASK (0x01) 43 44 struct fo_scheduler_qp_ctx { 45 struct scheduler_slave primary_slave; 46 struct scheduler_slave secondary_slave; 47 48 uint8_t deq_idx; 49 }; 50 51 static __rte_always_inline uint16_t 52 failover_slave_enqueue(struct scheduler_slave *slave, 53 struct rte_crypto_op **ops, uint16_t nb_ops) 54 { 55 uint16_t i, processed_ops; 56 57 for (i = 0; i < nb_ops && i < 4; i++) 58 rte_prefetch0(ops[i]->sym->session); 59 60 processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, 61 slave->qp_id, ops, nb_ops); 62 slave->nb_inflight_cops += processed_ops; 63 64 return processed_ops; 65 } 66 67 static uint16_t 68 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 69 { 70 struct fo_scheduler_qp_ctx *qp_ctx = 71 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 72 uint16_t enqueued_ops; 73 74 if (unlikely(nb_ops == 0)) 75 return 0; 76 77 enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave, 78 ops, nb_ops); 79 80 if (enqueued_ops < nb_ops) 81 enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave, 82 &ops[enqueued_ops], 83 nb_ops - enqueued_ops); 84 85 return enqueued_ops; 86 } 87 88 89 static uint16_t 90 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, 91 uint16_t nb_ops) 92 { 93 struct rte_ring *order_ring = 94 ((struct scheduler_qp_ctx *)qp)->order_ring; 95 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, 96 nb_ops); 97 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, 98 nb_ops_to_enq); 99 100 scheduler_order_insert(order_ring, ops, nb_ops_enqd); 101 102 return nb_ops_enqd; 103 } 104 105 static uint16_t 106 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) 107 { 108 struct fo_scheduler_qp_ctx *qp_ctx = 109 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; 110 struct scheduler_slave *slaves[NB_FAILOVER_SLAVES] = { 111 &qp_ctx->primary_slave, &qp_ctx->secondary_slave}; 112 struct scheduler_slave *slave = slaves[qp_ctx->deq_idx]; 113 uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0; 114 115 if (slave->nb_inflight_cops) { 116 nb_deq_ops = rte_cryptodev_dequeue_burst(slave->dev_id, 117 slave->qp_id, ops, nb_ops); 118 slave->nb_inflight_cops -= nb_deq_ops; 119 } 120 121 qp_ctx->deq_idx = (~qp_ctx->deq_idx) & SLAVE_SWITCH_MASK; 122 123 if (nb_deq_ops == nb_ops) 124 return nb_deq_ops; 125 126 slave = slaves[qp_ctx->deq_idx]; 127 128 if (slave->nb_inflight_cops) { 129 nb_deq_ops2 = rte_cryptodev_dequeue_burst(slave->dev_id, 130 slave->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops); 131 slave->nb_inflight_cops -= nb_deq_ops2; 132 } 133 134 return nb_deq_ops + nb_deq_ops2; 135 } 136 137 static uint16_t 138 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, 139 uint16_t nb_ops) 140 { 141 struct rte_ring *order_ring = 142 ((struct scheduler_qp_ctx *)qp)->order_ring; 143 144 schedule_dequeue(qp, ops, nb_ops); 145 146 return scheduler_order_drain(order_ring, ops, nb_ops); 147 } 148 149 static int 150 slave_attach(__rte_unused struct rte_cryptodev *dev, 151 __rte_unused uint8_t slave_id) 152 { 153 return 0; 154 } 155 156 static int 157 slave_detach(__rte_unused struct rte_cryptodev *dev, 158 __rte_unused uint8_t slave_id) 159 { 160 return 0; 161 } 162 163 static int 164 scheduler_start(struct rte_cryptodev *dev) 165 { 166 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 167 uint16_t i; 168 169 if (sched_ctx->nb_slaves < 2) { 170 CS_LOG_ERR("Number of slaves shall no less than 2"); 171 return -ENOMEM; 172 } 173 174 if (sched_ctx->reordering_enabled) { 175 dev->enqueue_burst = schedule_enqueue_ordering; 176 dev->dequeue_burst = schedule_dequeue_ordering; 177 } else { 178 dev->enqueue_burst = schedule_enqueue; 179 dev->dequeue_burst = schedule_dequeue; 180 } 181 182 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 183 struct fo_scheduler_qp_ctx *qp_ctx = 184 ((struct scheduler_qp_ctx *) 185 dev->data->queue_pairs[i])->private_qp_ctx; 186 187 rte_memcpy(&qp_ctx->primary_slave, 188 &sched_ctx->slaves[PRIMARY_SLAVE_IDX], 189 sizeof(struct scheduler_slave)); 190 rte_memcpy(&qp_ctx->secondary_slave, 191 &sched_ctx->slaves[SECONDARY_SLAVE_IDX], 192 sizeof(struct scheduler_slave)); 193 } 194 195 return 0; 196 } 197 198 static int 199 scheduler_stop(__rte_unused struct rte_cryptodev *dev) 200 { 201 return 0; 202 } 203 204 static int 205 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) 206 { 207 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; 208 struct fo_scheduler_qp_ctx *fo_qp_ctx; 209 210 fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0, 211 rte_socket_id()); 212 if (!fo_qp_ctx) { 213 CS_LOG_ERR("failed allocate memory for private queue pair"); 214 return -ENOMEM; 215 } 216 217 qp_ctx->private_qp_ctx = (void *)fo_qp_ctx; 218 219 return 0; 220 } 221 222 static int 223 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev) 224 { 225 return 0; 226 } 227 228 struct rte_cryptodev_scheduler_ops scheduler_fo_ops = { 229 slave_attach, 230 slave_detach, 231 scheduler_start, 232 scheduler_stop, 233 scheduler_config_qp, 234 scheduler_create_private_ctx, 235 NULL, /* option_set */ 236 NULL /*option_get */ 237 }; 238 239 struct rte_cryptodev_scheduler fo_scheduler = { 240 .name = "failover-scheduler", 241 .description = "scheduler which enqueues to the primary slave, " 242 "and only then enqueues to the secondary slave " 243 "upon failing on enqueuing to primary", 244 .mode = CDEV_SCHED_MODE_FAILOVER, 245 .ops = &scheduler_fo_ops 246 }; 247 248 struct rte_cryptodev_scheduler *failover_scheduler = &fo_scheduler; 249