1 /* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
2 /* $NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $ */
3
4 /*
5 * Copyright (c) 2019 Internet Initiative Japan, Inc.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 * POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 /*
31 * Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 *
37 * * Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * * Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in
41 * the documentation and/or other materials provided with the
42 * distribution.
43 * * Neither the name of Intel Corporation nor the names of its
44 * contributors may be used to endorse or promote products derived
45 * from this software without specific prior written permission.
46 *
47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58 */
59
60 #include <sys/cdefs.h>
61 __FBSDID("$FreeBSD$");
62 #if 0
63 __KERNEL_RCSID(0, "$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $");
64 #endif
65
66 #include <sys/param.h>
67 #include <sys/systm.h>
68 #include <sys/bus.h>
69 #include <sys/cpu.h>
70 #include <sys/firmware.h>
71 #include <sys/kernel.h>
72 #include <sys/mbuf.h>
73 #include <sys/md5.h>
74 #include <sys/module.h>
75 #include <sys/mutex.h>
76 #include <sys/smp.h>
77 #include <sys/sysctl.h>
78 #include <sys/rman.h>
79
80 #include <machine/bus.h>
81
82 #include <opencrypto/cryptodev.h>
83 #include <opencrypto/xform.h>
84
85 #include "cryptodev_if.h"
86
87 #include <dev/pci/pcireg.h>
88 #include <dev/pci/pcivar.h>
89
90 #include "qatreg.h"
91 #include "qatvar.h"
92 #include "qat_aevar.h"
93
94 extern struct qat_hw qat_hw_c2xxx;
95 extern struct qat_hw qat_hw_c3xxx;
96 extern struct qat_hw qat_hw_c62x;
97 extern struct qat_hw qat_hw_d15xx;
98 extern struct qat_hw qat_hw_dh895xcc;
99
100 #define PCI_VENDOR_INTEL 0x8086
101 #define PCI_PRODUCT_INTEL_C2000_IQIA_PHYS 0x1f18
102 #define PCI_PRODUCT_INTEL_C3K_QAT 0x19e2
103 #define PCI_PRODUCT_INTEL_C3K_QAT_VF 0x19e3
104 #define PCI_PRODUCT_INTEL_C620_QAT 0x37c8
105 #define PCI_PRODUCT_INTEL_C620_QAT_VF 0x37c9
106 #define PCI_PRODUCT_INTEL_XEOND_QAT 0x6f54
107 #define PCI_PRODUCT_INTEL_XEOND_QAT_VF 0x6f55
108 #define PCI_PRODUCT_INTEL_DH895XCC_QAT 0x0435
109 #define PCI_PRODUCT_INTEL_DH895XCC_QAT_VF 0x0443
110
111 static const struct qat_product {
112 uint16_t qatp_vendor;
113 uint16_t qatp_product;
114 const char *qatp_name;
115 enum qat_chip_type qatp_chip;
116 const struct qat_hw *qatp_hw;
117 } qat_products[] = {
118 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C2000_IQIA_PHYS,
119 "Intel C2000 QuickAssist PF",
120 QAT_CHIP_C2XXX, &qat_hw_c2xxx },
121 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C3K_QAT,
122 "Intel C3000 QuickAssist PF",
123 QAT_CHIP_C3XXX, &qat_hw_c3xxx },
124 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C620_QAT,
125 "Intel C620/Xeon D-2100 QuickAssist PF",
126 QAT_CHIP_C62X, &qat_hw_c62x },
127 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XEOND_QAT,
128 "Intel Xeon D-1500 QuickAssist PF",
129 QAT_CHIP_D15XX, &qat_hw_d15xx },
130 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH895XCC_QAT,
131 "Intel 8950 QuickAssist PCIe Adapter PF",
132 QAT_CHIP_DH895XCC, &qat_hw_dh895xcc },
133 { 0, 0, NULL, 0, NULL },
134 };
135
136 /* Hash Algorithm specific structure */
137
138 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
139 static const uint8_t sha1_initial_state[QAT_HASH_SHA1_STATE_SIZE] = {
140 0x67, 0x45, 0x23, 0x01,
141 0xef, 0xcd, 0xab, 0x89,
142 0x98, 0xba, 0xdc, 0xfe,
143 0x10, 0x32, 0x54, 0x76,
144 0xc3, 0xd2, 0xe1, 0xf0
145 };
146
147 /* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */
148 static const uint8_t sha256_initial_state[QAT_HASH_SHA256_STATE_SIZE] = {
149 0x6a, 0x09, 0xe6, 0x67,
150 0xbb, 0x67, 0xae, 0x85,
151 0x3c, 0x6e, 0xf3, 0x72,
152 0xa5, 0x4f, 0xf5, 0x3a,
153 0x51, 0x0e, 0x52, 0x7f,
154 0x9b, 0x05, 0x68, 0x8c,
155 0x1f, 0x83, 0xd9, 0xab,
156 0x5b, 0xe0, 0xcd, 0x19
157 };
158
159 /* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
160 static const uint8_t sha384_initial_state[QAT_HASH_SHA384_STATE_SIZE] = {
161 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8,
162 0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07,
163 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17,
164 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39,
165 0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31,
166 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11,
167 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7,
168 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4
169 };
170
171 /* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
172 static const uint8_t sha512_initial_state[QAT_HASH_SHA512_STATE_SIZE] = {
173 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08,
174 0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b,
175 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b,
176 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1,
177 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1,
178 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f,
179 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b,
180 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79
181 };
182
183 static const struct qat_sym_hash_alg_info sha1_info = {
184 .qshai_digest_len = QAT_HASH_SHA1_DIGEST_SIZE,
185 .qshai_block_len = QAT_HASH_SHA1_BLOCK_SIZE,
186 .qshai_state_size = QAT_HASH_SHA1_STATE_SIZE,
187 .qshai_init_state = sha1_initial_state,
188 .qshai_sah = &auth_hash_hmac_sha1,
189 .qshai_state_offset = 0,
190 .qshai_state_word = 4,
191 };
192
193 static const struct qat_sym_hash_alg_info sha256_info = {
194 .qshai_digest_len = QAT_HASH_SHA256_DIGEST_SIZE,
195 .qshai_block_len = QAT_HASH_SHA256_BLOCK_SIZE,
196 .qshai_state_size = QAT_HASH_SHA256_STATE_SIZE,
197 .qshai_init_state = sha256_initial_state,
198 .qshai_sah = &auth_hash_hmac_sha2_256,
199 .qshai_state_offset = offsetof(SHA256_CTX, state),
200 .qshai_state_word = 4,
201 };
202
203 static const struct qat_sym_hash_alg_info sha384_info = {
204 .qshai_digest_len = QAT_HASH_SHA384_DIGEST_SIZE,
205 .qshai_block_len = QAT_HASH_SHA384_BLOCK_SIZE,
206 .qshai_state_size = QAT_HASH_SHA384_STATE_SIZE,
207 .qshai_init_state = sha384_initial_state,
208 .qshai_sah = &auth_hash_hmac_sha2_384,
209 .qshai_state_offset = offsetof(SHA384_CTX, state),
210 .qshai_state_word = 8,
211 };
212
213 static const struct qat_sym_hash_alg_info sha512_info = {
214 .qshai_digest_len = QAT_HASH_SHA512_DIGEST_SIZE,
215 .qshai_block_len = QAT_HASH_SHA512_BLOCK_SIZE,
216 .qshai_state_size = QAT_HASH_SHA512_STATE_SIZE,
217 .qshai_init_state = sha512_initial_state,
218 .qshai_sah = &auth_hash_hmac_sha2_512,
219 .qshai_state_offset = offsetof(SHA512_CTX, state),
220 .qshai_state_word = 8,
221 };
222
223 static const struct qat_sym_hash_alg_info aes_gcm_info = {
224 .qshai_digest_len = QAT_HASH_AES_GCM_DIGEST_SIZE,
225 .qshai_block_len = QAT_HASH_AES_GCM_BLOCK_SIZE,
226 .qshai_state_size = QAT_HASH_AES_GCM_STATE_SIZE,
227 .qshai_sah = &auth_hash_nist_gmac_aes_128,
228 };
229
230 /* Hash QAT specific structures */
231
232 static const struct qat_sym_hash_qat_info sha1_config = {
233 .qshqi_algo_enc = HW_AUTH_ALGO_SHA1,
234 .qshqi_auth_counter = QAT_HASH_SHA1_BLOCK_SIZE,
235 .qshqi_state1_len = HW_SHA1_STATE1_SZ,
236 .qshqi_state2_len = HW_SHA1_STATE2_SZ,
237 };
238
239 static const struct qat_sym_hash_qat_info sha256_config = {
240 .qshqi_algo_enc = HW_AUTH_ALGO_SHA256,
241 .qshqi_auth_counter = QAT_HASH_SHA256_BLOCK_SIZE,
242 .qshqi_state1_len = HW_SHA256_STATE1_SZ,
243 .qshqi_state2_len = HW_SHA256_STATE2_SZ
244 };
245
246 static const struct qat_sym_hash_qat_info sha384_config = {
247 .qshqi_algo_enc = HW_AUTH_ALGO_SHA384,
248 .qshqi_auth_counter = QAT_HASH_SHA384_BLOCK_SIZE,
249 .qshqi_state1_len = HW_SHA384_STATE1_SZ,
250 .qshqi_state2_len = HW_SHA384_STATE2_SZ
251 };
252
253 static const struct qat_sym_hash_qat_info sha512_config = {
254 .qshqi_algo_enc = HW_AUTH_ALGO_SHA512,
255 .qshqi_auth_counter = QAT_HASH_SHA512_BLOCK_SIZE,
256 .qshqi_state1_len = HW_SHA512_STATE1_SZ,
257 .qshqi_state2_len = HW_SHA512_STATE2_SZ
258 };
259
260 static const struct qat_sym_hash_qat_info aes_gcm_config = {
261 .qshqi_algo_enc = HW_AUTH_ALGO_GALOIS_128,
262 .qshqi_auth_counter = QAT_HASH_AES_GCM_BLOCK_SIZE,
263 .qshqi_state1_len = HW_GALOIS_128_STATE1_SZ,
264 .qshqi_state2_len =
265 HW_GALOIS_H_SZ + HW_GALOIS_LEN_A_SZ + HW_GALOIS_E_CTR0_SZ,
266 };
267
268 static const struct qat_sym_hash_def qat_sym_hash_defs[] = {
269 [QAT_SYM_HASH_SHA1] = { &sha1_info, &sha1_config },
270 [QAT_SYM_HASH_SHA256] = { &sha256_info, &sha256_config },
271 [QAT_SYM_HASH_SHA384] = { &sha384_info, &sha384_config },
272 [QAT_SYM_HASH_SHA512] = { &sha512_info, &sha512_config },
273 [QAT_SYM_HASH_AES_GCM] = { &aes_gcm_info, &aes_gcm_config },
274 };
275
276 static const struct qat_product *qat_lookup(device_t);
277 static int qat_probe(device_t);
278 static int qat_attach(device_t);
279 static int qat_init(device_t);
280 static int qat_start(device_t);
281 static int qat_detach(device_t);
282
283 static int qat_newsession(device_t dev, crypto_session_t cses,
284 const struct crypto_session_params *csp);
285 static void qat_freesession(device_t dev, crypto_session_t cses);
286
287 static int qat_setup_msix_intr(struct qat_softc *);
288
289 static void qat_etr_init(struct qat_softc *);
290 static void qat_etr_deinit(struct qat_softc *);
291 static void qat_etr_bank_init(struct qat_softc *, int);
292 static void qat_etr_bank_deinit(struct qat_softc *sc, int);
293
294 static void qat_etr_ap_bank_init(struct qat_softc *);
295 static void qat_etr_ap_bank_set_ring_mask(uint32_t *, uint32_t, int);
296 static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *, uint32_t *,
297 uint32_t, int);
298 static void qat_etr_ap_bank_setup_ring(struct qat_softc *,
299 struct qat_ring *);
300 static int qat_etr_verify_ring_size(uint32_t, uint32_t);
301
302 static int qat_etr_ring_intr(struct qat_softc *, struct qat_bank *,
303 struct qat_ring *);
304 static void qat_etr_bank_intr(void *);
305
306 static void qat_arb_update(struct qat_softc *, struct qat_bank *);
307
308 static struct qat_sym_cookie *qat_crypto_alloc_sym_cookie(
309 struct qat_crypto_bank *);
310 static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *,
311 struct qat_sym_cookie *);
312 static int qat_crypto_setup_ring(struct qat_softc *,
313 struct qat_crypto_bank *);
314 static int qat_crypto_bank_init(struct qat_softc *,
315 struct qat_crypto_bank *);
316 static int qat_crypto_init(struct qat_softc *);
317 static void qat_crypto_deinit(struct qat_softc *);
318 static int qat_crypto_start(struct qat_softc *);
319 static void qat_crypto_stop(struct qat_softc *);
320 static int qat_crypto_sym_rxintr(struct qat_softc *, void *, void *);
321
322 static MALLOC_DEFINE(M_QAT, "qat", "Intel QAT driver");
323
324 static const struct qat_product *
qat_lookup(device_t dev)325 qat_lookup(device_t dev)
326 {
327 const struct qat_product *qatp;
328
329 for (qatp = qat_products; qatp->qatp_name != NULL; qatp++) {
330 if (pci_get_vendor(dev) == qatp->qatp_vendor &&
331 pci_get_device(dev) == qatp->qatp_product)
332 return qatp;
333 }
334 return NULL;
335 }
336
337 static int
qat_probe(device_t dev)338 qat_probe(device_t dev)
339 {
340 const struct qat_product *prod;
341
342 prod = qat_lookup(dev);
343 if (prod != NULL) {
344 device_set_desc(dev, prod->qatp_name);
345 return BUS_PROBE_DEFAULT;
346 }
347 return ENXIO;
348 }
349
350 static int
qat_attach(device_t dev)351 qat_attach(device_t dev)
352 {
353 struct qat_softc *sc = device_get_softc(dev);
354 const struct qat_product *qatp;
355 int bar, count, error, i;
356
357 sc->sc_dev = dev;
358 sc->sc_rev = pci_get_revid(dev);
359 sc->sc_crypto.qcy_cid = -1;
360
361 qatp = qat_lookup(dev);
362 memcpy(&sc->sc_hw, qatp->qatp_hw, sizeof(struct qat_hw));
363
364 /* Determine active accelerators and engines */
365 sc->sc_accel_mask = sc->sc_hw.qhw_get_accel_mask(sc);
366 sc->sc_ae_mask = sc->sc_hw.qhw_get_ae_mask(sc);
367
368 sc->sc_accel_num = 0;
369 for (i = 0; i < sc->sc_hw.qhw_num_accel; i++) {
370 if (sc->sc_accel_mask & (1 << i))
371 sc->sc_accel_num++;
372 }
373 sc->sc_ae_num = 0;
374 for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) {
375 if (sc->sc_ae_mask & (1 << i))
376 sc->sc_ae_num++;
377 }
378
379 if (!sc->sc_accel_mask || (sc->sc_ae_mask & 0x01) == 0) {
380 device_printf(sc->sc_dev, "couldn't find acceleration");
381 goto fail;
382 }
383
384 MPASS(sc->sc_accel_num <= MAX_NUM_ACCEL);
385 MPASS(sc->sc_ae_num <= MAX_NUM_AE);
386
387 /* Determine SKU and capabilities */
388 sc->sc_sku = sc->sc_hw.qhw_get_sku(sc);
389 sc->sc_accel_cap = sc->sc_hw.qhw_get_accel_cap(sc);
390 sc->sc_fw_uof_name = sc->sc_hw.qhw_get_fw_uof_name(sc);
391
392 i = 0;
393 if (sc->sc_hw.qhw_sram_bar_id != NO_PCI_REG) {
394 MPASS(sc->sc_hw.qhw_sram_bar_id == 0);
395 uint32_t fusectl = pci_read_config(dev, FUSECTL_REG, 4);
396 /* Skip SRAM BAR */
397 i = (fusectl & FUSECTL_MASK) ? 1 : 0;
398 }
399 for (bar = 0; bar < PCIR_MAX_BAR_0; bar++) {
400 uint32_t val = pci_read_config(dev, PCIR_BAR(bar), 4);
401 if (val == 0 || !PCI_BAR_MEM(val))
402 continue;
403
404 sc->sc_rid[i] = PCIR_BAR(bar);
405 sc->sc_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
406 &sc->sc_rid[i], RF_ACTIVE);
407 if (sc->sc_res[i] == NULL) {
408 device_printf(dev, "couldn't map BAR %d\n", bar);
409 goto fail;
410 }
411
412 sc->sc_csrt[i] = rman_get_bustag(sc->sc_res[i]);
413 sc->sc_csrh[i] = rman_get_bushandle(sc->sc_res[i]);
414
415 i++;
416 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64)
417 bar++;
418 }
419
420 pci_enable_busmaster(dev);
421
422 count = sc->sc_hw.qhw_num_banks + 1;
423 if (pci_msix_count(dev) < count) {
424 device_printf(dev, "insufficient MSI-X vectors (%d vs. %d)\n",
425 pci_msix_count(dev), count);
426 goto fail;
427 }
428 error = pci_alloc_msix(dev, &count);
429 if (error != 0) {
430 device_printf(dev, "failed to allocate MSI-X vectors\n");
431 goto fail;
432 }
433
434 error = qat_init(dev);
435 if (error == 0)
436 return 0;
437
438 fail:
439 qat_detach(dev);
440 return ENXIO;
441 }
442
443 static int
qat_init(device_t dev)444 qat_init(device_t dev)
445 {
446 struct qat_softc *sc = device_get_softc(dev);
447 int error;
448
449 qat_etr_init(sc);
450
451 if (sc->sc_hw.qhw_init_admin_comms != NULL &&
452 (error = sc->sc_hw.qhw_init_admin_comms(sc)) != 0) {
453 device_printf(sc->sc_dev,
454 "Could not initialize admin comms: %d\n", error);
455 return error;
456 }
457
458 if (sc->sc_hw.qhw_init_arb != NULL &&
459 (error = sc->sc_hw.qhw_init_arb(sc)) != 0) {
460 device_printf(sc->sc_dev,
461 "Could not initialize hw arbiter: %d\n", error);
462 return error;
463 }
464
465 error = qat_ae_init(sc);
466 if (error) {
467 device_printf(sc->sc_dev,
468 "Could not initialize Acceleration Engine: %d\n", error);
469 return error;
470 }
471
472 error = qat_aefw_load(sc);
473 if (error) {
474 device_printf(sc->sc_dev,
475 "Could not load firmware: %d\n", error);
476 return error;
477 }
478
479 error = qat_setup_msix_intr(sc);
480 if (error) {
481 device_printf(sc->sc_dev,
482 "Could not setup interrupts: %d\n", error);
483 return error;
484 }
485
486 sc->sc_hw.qhw_enable_intr(sc);
487
488 error = qat_crypto_init(sc);
489 if (error) {
490 device_printf(sc->sc_dev,
491 "Could not initialize service: %d\n", error);
492 return error;
493 }
494
495 if (sc->sc_hw.qhw_enable_error_correction != NULL)
496 sc->sc_hw.qhw_enable_error_correction(sc);
497
498 if (sc->sc_hw.qhw_set_ssm_wdtimer != NULL &&
499 (error = sc->sc_hw.qhw_set_ssm_wdtimer(sc)) != 0) {
500 device_printf(sc->sc_dev,
501 "Could not initialize watchdog timer: %d\n", error);
502 return error;
503 }
504
505 error = qat_start(dev);
506 if (error) {
507 device_printf(sc->sc_dev,
508 "Could not start: %d\n", error);
509 return error;
510 }
511
512 return 0;
513 }
514
515 static int
qat_start(device_t dev)516 qat_start(device_t dev)
517 {
518 struct qat_softc *sc = device_get_softc(dev);
519 int error;
520
521 error = qat_ae_start(sc);
522 if (error)
523 return error;
524
525 if (sc->sc_hw.qhw_send_admin_init != NULL &&
526 (error = sc->sc_hw.qhw_send_admin_init(sc)) != 0) {
527 return error;
528 }
529
530 error = qat_crypto_start(sc);
531 if (error)
532 return error;
533
534 return 0;
535 }
536
537 static int
qat_detach(device_t dev)538 qat_detach(device_t dev)
539 {
540 struct qat_softc *sc;
541 int bar, i;
542
543 sc = device_get_softc(dev);
544
545 qat_crypto_stop(sc);
546 qat_crypto_deinit(sc);
547 qat_aefw_unload(sc);
548
549 if (sc->sc_etr_banks != NULL) {
550 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) {
551 struct qat_bank *qb = &sc->sc_etr_banks[i];
552
553 if (qb->qb_ih_cookie != NULL)
554 (void)bus_teardown_intr(dev, qb->qb_ih,
555 qb->qb_ih_cookie);
556 if (qb->qb_ih != NULL)
557 (void)bus_release_resource(dev, SYS_RES_IRQ,
558 i + 1, qb->qb_ih);
559 }
560 }
561 if (sc->sc_ih_cookie != NULL) {
562 (void)bus_teardown_intr(dev, sc->sc_ih, sc->sc_ih_cookie);
563 sc->sc_ih_cookie = NULL;
564 }
565 if (sc->sc_ih != NULL) {
566 (void)bus_release_resource(dev, SYS_RES_IRQ,
567 sc->sc_hw.qhw_num_banks + 1, sc->sc_ih);
568 sc->sc_ih = NULL;
569 }
570 pci_release_msi(dev);
571
572 qat_etr_deinit(sc);
573
574 for (bar = 0; bar < MAX_BARS; bar++) {
575 if (sc->sc_res[bar] != NULL) {
576 (void)bus_release_resource(dev, SYS_RES_MEMORY,
577 sc->sc_rid[bar], sc->sc_res[bar]);
578 sc->sc_res[bar] = NULL;
579 }
580 }
581
582 return 0;
583 }
584
585 void *
qat_alloc_mem(size_t size)586 qat_alloc_mem(size_t size)
587 {
588 return (malloc(size, M_QAT, M_WAITOK | M_ZERO));
589 }
590
591 void
qat_free_mem(void * ptr)592 qat_free_mem(void *ptr)
593 {
594 free(ptr, M_QAT);
595 }
596
597 static void
qat_alloc_dmamem_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)598 qat_alloc_dmamem_cb(void *arg, bus_dma_segment_t *segs, int nseg,
599 int error)
600 {
601 struct qat_dmamem *qdm;
602
603 if (error != 0)
604 return;
605
606 KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg));
607 qdm = arg;
608 qdm->qdm_dma_seg = segs[0];
609 }
610
611 int
qat_alloc_dmamem(struct qat_softc * sc,struct qat_dmamem * qdm,int nseg,bus_size_t size,bus_size_t alignment)612 qat_alloc_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm,
613 int nseg, bus_size_t size, bus_size_t alignment)
614 {
615 int error;
616
617 KASSERT(qdm->qdm_dma_vaddr == NULL,
618 ("%s: DMA memory descriptor in use", __func__));
619
620 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),
621 alignment, 0, /* alignment, boundary */
622 BUS_SPACE_MAXADDR, /* lowaddr */
623 BUS_SPACE_MAXADDR, /* highaddr */
624 NULL, NULL, /* filter, filterarg */
625 size, /* maxsize */
626 nseg, /* nsegments */
627 size, /* maxsegsize */
628 BUS_DMA_COHERENT, /* flags */
629 NULL, NULL, /* lockfunc, lockarg */
630 &qdm->qdm_dma_tag);
631 if (error != 0)
632 return error;
633
634 error = bus_dmamem_alloc(qdm->qdm_dma_tag, &qdm->qdm_dma_vaddr,
635 BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
636 &qdm->qdm_dma_map);
637 if (error != 0) {
638 device_printf(sc->sc_dev,
639 "couldn't allocate dmamem, error = %d\n", error);
640 goto fail_0;
641 }
642
643 error = bus_dmamap_load(qdm->qdm_dma_tag, qdm->qdm_dma_map,
644 qdm->qdm_dma_vaddr, size, qat_alloc_dmamem_cb, qdm,
645 BUS_DMA_NOWAIT);
646 if (error) {
647 device_printf(sc->sc_dev,
648 "couldn't load dmamem map, error = %d\n", error);
649 goto fail_1;
650 }
651
652 return 0;
653 fail_1:
654 bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map);
655 fail_0:
656 bus_dma_tag_destroy(qdm->qdm_dma_tag);
657 return error;
658 }
659
660 void
qat_free_dmamem(struct qat_softc * sc,struct qat_dmamem * qdm)661 qat_free_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm)
662 {
663 if (qdm->qdm_dma_tag != NULL) {
664 bus_dmamap_unload(qdm->qdm_dma_tag, qdm->qdm_dma_map);
665 bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr,
666 qdm->qdm_dma_map);
667 bus_dma_tag_destroy(qdm->qdm_dma_tag);
668 explicit_bzero(qdm, sizeof(*qdm));
669 }
670 }
671
672 static int
qat_setup_msix_intr(struct qat_softc * sc)673 qat_setup_msix_intr(struct qat_softc *sc)
674 {
675 device_t dev;
676 int error, i, rid;
677
678 dev = sc->sc_dev;
679
680 for (i = 1; i <= sc->sc_hw.qhw_num_banks; i++) {
681 struct qat_bank *qb = &sc->sc_etr_banks[i - 1];
682
683 rid = i;
684 qb->qb_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
685 RF_ACTIVE);
686 if (qb->qb_ih == NULL) {
687 device_printf(dev,
688 "failed to allocate bank intr resource\n");
689 return ENXIO;
690 }
691 error = bus_setup_intr(dev, qb->qb_ih,
692 INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_etr_bank_intr, qb,
693 &qb->qb_ih_cookie);
694 if (error != 0) {
695 device_printf(dev, "failed to set up bank intr\n");
696 return error;
697 }
698 error = bus_bind_intr(dev, qb->qb_ih, (i - 1) % mp_ncpus);
699 if (error != 0)
700 device_printf(dev, "failed to bind intr %d\n", i);
701 }
702
703 rid = i;
704 sc->sc_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
705 RF_ACTIVE);
706 if (sc->sc_ih == NULL)
707 return ENXIO;
708 error = bus_setup_intr(dev, sc->sc_ih, INTR_TYPE_NET | INTR_MPSAFE,
709 NULL, qat_ae_cluster_intr, sc, &sc->sc_ih_cookie);
710
711 return error;
712 }
713
714 static void
qat_etr_init(struct qat_softc * sc)715 qat_etr_init(struct qat_softc *sc)
716 {
717 int i;
718
719 sc->sc_etr_banks = qat_alloc_mem(
720 sizeof(struct qat_bank) * sc->sc_hw.qhw_num_banks);
721
722 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++)
723 qat_etr_bank_init(sc, i);
724
725 if (sc->sc_hw.qhw_num_ap_banks) {
726 sc->sc_etr_ap_banks = qat_alloc_mem(
727 sizeof(struct qat_ap_bank) * sc->sc_hw.qhw_num_ap_banks);
728 qat_etr_ap_bank_init(sc);
729 }
730 }
731
732 static void
qat_etr_deinit(struct qat_softc * sc)733 qat_etr_deinit(struct qat_softc *sc)
734 {
735 int i;
736
737 if (sc->sc_etr_banks != NULL) {
738 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++)
739 qat_etr_bank_deinit(sc, i);
740 qat_free_mem(sc->sc_etr_banks);
741 sc->sc_etr_banks = NULL;
742 }
743 if (sc->sc_etr_ap_banks != NULL) {
744 qat_free_mem(sc->sc_etr_ap_banks);
745 sc->sc_etr_ap_banks = NULL;
746 }
747 }
748
749 static void
qat_etr_bank_init(struct qat_softc * sc,int bank)750 qat_etr_bank_init(struct qat_softc *sc, int bank)
751 {
752 struct qat_bank *qb = &sc->sc_etr_banks[bank];
753 int i, tx_rx_gap = sc->sc_hw.qhw_tx_rx_gap;
754
755 MPASS(bank < sc->sc_hw.qhw_num_banks);
756
757 mtx_init(&qb->qb_bank_mtx, "qb bank", NULL, MTX_DEF);
758
759 qb->qb_sc = sc;
760 qb->qb_bank = bank;
761 qb->qb_coalescing_time = COALESCING_TIME_INTERVAL_DEFAULT;
762
763 /* Clean CSRs for all rings within the bank */
764 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) {
765 struct qat_ring *qr = &qb->qb_et_rings[i];
766
767 qat_etr_bank_ring_write_4(sc, bank, i,
768 ETR_RING_CONFIG, 0);
769 qat_etr_bank_ring_base_write_8(sc, bank, i, 0);
770
771 if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) {
772 qr->qr_inflight = qat_alloc_mem(sizeof(uint32_t));
773 } else if (sc->sc_hw.qhw_tx_rings_mask &
774 (1 << (i - tx_rx_gap))) {
775 /* Share inflight counter with rx and tx */
776 qr->qr_inflight =
777 qb->qb_et_rings[i - tx_rx_gap].qr_inflight;
778 }
779 }
780
781 if (sc->sc_hw.qhw_init_etr_intr != NULL) {
782 sc->sc_hw.qhw_init_etr_intr(sc, bank);
783 } else {
784 /* common code in qat 1.7 */
785 qat_etr_bank_write_4(sc, bank, ETR_INT_REG,
786 ETR_INT_REG_CLEAR_MASK);
787 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank /
788 ETR_RINGS_PER_INT_SRCSEL; i++) {
789 qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL +
790 (i * ETR_INT_SRCSEL_NEXT_OFFSET),
791 ETR_INT_SRCSEL_MASK);
792 }
793 }
794 }
795
796 static void
qat_etr_bank_deinit(struct qat_softc * sc,int bank)797 qat_etr_bank_deinit(struct qat_softc *sc, int bank)
798 {
799 struct qat_bank *qb;
800 struct qat_ring *qr;
801 int i;
802
803 qb = &sc->sc_etr_banks[bank];
804 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) {
805 if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) {
806 qr = &qb->qb_et_rings[i];
807 qat_free_mem(qr->qr_inflight);
808 }
809 }
810 }
811
812 static void
qat_etr_ap_bank_init(struct qat_softc * sc)813 qat_etr_ap_bank_init(struct qat_softc *sc)
814 {
815 int ap_bank;
816
817 for (ap_bank = 0; ap_bank < sc->sc_hw.qhw_num_ap_banks; ap_bank++) {
818 struct qat_ap_bank *qab = &sc->sc_etr_ap_banks[ap_bank];
819
820 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_MASK,
821 ETR_AP_NF_MASK_INIT);
822 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 0);
823 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_MASK,
824 ETR_AP_NE_MASK_INIT);
825 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 0);
826
827 memset(qab, 0, sizeof(*qab));
828 }
829 }
830
831 static void
qat_etr_ap_bank_set_ring_mask(uint32_t * ap_mask,uint32_t ring,int set_mask)832 qat_etr_ap_bank_set_ring_mask(uint32_t *ap_mask, uint32_t ring, int set_mask)
833 {
834 if (set_mask)
835 *ap_mask |= (1 << ETR_RING_NUMBER_IN_AP_BANK(ring));
836 else
837 *ap_mask &= ~(1 << ETR_RING_NUMBER_IN_AP_BANK(ring));
838 }
839
840 static void
qat_etr_ap_bank_set_ring_dest(struct qat_softc * sc,uint32_t * ap_dest,uint32_t ring,int set_dest)841 qat_etr_ap_bank_set_ring_dest(struct qat_softc *sc, uint32_t *ap_dest,
842 uint32_t ring, int set_dest)
843 {
844 uint32_t ae_mask;
845 uint8_t mailbox, ae, nae;
846 uint8_t *dest = (uint8_t *)ap_dest;
847
848 mailbox = ETR_RING_AP_MAILBOX_NUMBER(ring);
849
850 nae = 0;
851 ae_mask = sc->sc_ae_mask;
852 for (ae = 0; ae < sc->sc_hw.qhw_num_engines; ae++) {
853 if ((ae_mask & (1 << ae)) == 0)
854 continue;
855
856 if (set_dest) {
857 dest[nae] = __SHIFTIN(ae, ETR_AP_DEST_AE) |
858 __SHIFTIN(mailbox, ETR_AP_DEST_MAILBOX) |
859 ETR_AP_DEST_ENABLE;
860 } else {
861 dest[nae] = 0;
862 }
863 nae++;
864 if (nae == ETR_MAX_AE_PER_MAILBOX)
865 break;
866 }
867 }
868
869 static void
qat_etr_ap_bank_setup_ring(struct qat_softc * sc,struct qat_ring * qr)870 qat_etr_ap_bank_setup_ring(struct qat_softc *sc, struct qat_ring *qr)
871 {
872 struct qat_ap_bank *qab;
873 int ap_bank;
874
875 if (sc->sc_hw.qhw_num_ap_banks == 0)
876 return;
877
878 ap_bank = ETR_RING_AP_BANK_NUMBER(qr->qr_ring);
879 MPASS(ap_bank < sc->sc_hw.qhw_num_ap_banks);
880 qab = &sc->sc_etr_ap_banks[ap_bank];
881
882 if (qr->qr_cb == NULL) {
883 qat_etr_ap_bank_set_ring_mask(&qab->qab_ne_mask, qr->qr_ring, 1);
884 if (!qab->qab_ne_dest) {
885 qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_ne_dest,
886 qr->qr_ring, 1);
887 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST,
888 qab->qab_ne_dest);
889 }
890 } else {
891 qat_etr_ap_bank_set_ring_mask(&qab->qab_nf_mask, qr->qr_ring, 1);
892 if (!qab->qab_nf_dest) {
893 qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_nf_dest,
894 qr->qr_ring, 1);
895 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST,
896 qab->qab_nf_dest);
897 }
898 }
899 }
900
901 static int
qat_etr_verify_ring_size(uint32_t msg_size,uint32_t num_msgs)902 qat_etr_verify_ring_size(uint32_t msg_size, uint32_t num_msgs)
903 {
904 int i = QAT_MIN_RING_SIZE;
905
906 for (; i <= QAT_MAX_RING_SIZE; i++)
907 if ((msg_size * num_msgs) == QAT_SIZE_TO_RING_SIZE_IN_BYTES(i))
908 return i;
909
910 return QAT_DEFAULT_RING_SIZE;
911 }
912
913 int
qat_etr_setup_ring(struct qat_softc * sc,int bank,uint32_t ring,uint32_t num_msgs,uint32_t msg_size,qat_cb_t cb,void * cb_arg,const char * name,struct qat_ring ** rqr)914 qat_etr_setup_ring(struct qat_softc *sc, int bank, uint32_t ring,
915 uint32_t num_msgs, uint32_t msg_size, qat_cb_t cb, void *cb_arg,
916 const char *name, struct qat_ring **rqr)
917 {
918 struct qat_bank *qb;
919 struct qat_ring *qr = NULL;
920 int error;
921 uint32_t ring_size_bytes, ring_config;
922 uint64_t ring_base;
923 uint32_t wm_nf = ETR_RING_CONFIG_NEAR_WM_512;
924 uint32_t wm_ne = ETR_RING_CONFIG_NEAR_WM_0;
925
926 MPASS(bank < sc->sc_hw.qhw_num_banks);
927
928 /* Allocate a ring from specified bank */
929 qb = &sc->sc_etr_banks[bank];
930
931 if (ring >= sc->sc_hw.qhw_num_rings_per_bank)
932 return EINVAL;
933 if (qb->qb_allocated_rings & (1 << ring))
934 return ENOENT;
935 qr = &qb->qb_et_rings[ring];
936 qb->qb_allocated_rings |= 1 << ring;
937
938 /* Initialize allocated ring */
939 qr->qr_ring = ring;
940 qr->qr_bank = bank;
941 qr->qr_name = name;
942 qr->qr_ring_id = qr->qr_bank * sc->sc_hw.qhw_num_rings_per_bank + ring;
943 qr->qr_ring_mask = (1 << ring);
944 qr->qr_cb = cb;
945 qr->qr_cb_arg = cb_arg;
946
947 /* Setup the shadow variables */
948 qr->qr_head = 0;
949 qr->qr_tail = 0;
950 qr->qr_msg_size = QAT_BYTES_TO_MSG_SIZE(msg_size);
951 qr->qr_ring_size = qat_etr_verify_ring_size(msg_size, num_msgs);
952
953 /*
954 * To make sure that ring is alligned to ring size allocate
955 * at least 4k and then tell the user it is smaller.
956 */
957 ring_size_bytes = QAT_SIZE_TO_RING_SIZE_IN_BYTES(qr->qr_ring_size);
958 ring_size_bytes = QAT_RING_SIZE_BYTES_MIN(ring_size_bytes);
959 error = qat_alloc_dmamem(sc, &qr->qr_dma, 1, ring_size_bytes,
960 ring_size_bytes);
961 if (error)
962 return error;
963
964 qr->qr_ring_vaddr = qr->qr_dma.qdm_dma_vaddr;
965 qr->qr_ring_paddr = qr->qr_dma.qdm_dma_seg.ds_addr;
966
967 memset(qr->qr_ring_vaddr, QAT_RING_PATTERN,
968 qr->qr_dma.qdm_dma_seg.ds_len);
969
970 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
971 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
972
973 if (cb == NULL) {
974 ring_config = ETR_RING_CONFIG_BUILD(qr->qr_ring_size);
975 } else {
976 ring_config =
977 ETR_RING_CONFIG_BUILD_RESP(qr->qr_ring_size, wm_nf, wm_ne);
978 }
979 qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_CONFIG, ring_config);
980
981 ring_base = ETR_RING_BASE_BUILD(qr->qr_ring_paddr, qr->qr_ring_size);
982 qat_etr_bank_ring_base_write_8(sc, bank, ring, ring_base);
983
984 if (sc->sc_hw.qhw_init_arb != NULL)
985 qat_arb_update(sc, qb);
986
987 mtx_init(&qr->qr_ring_mtx, "qr ring", NULL, MTX_DEF);
988
989 qat_etr_ap_bank_setup_ring(sc, qr);
990
991 if (cb != NULL) {
992 uint32_t intr_mask;
993
994 qb->qb_intr_mask |= qr->qr_ring_mask;
995 intr_mask = qb->qb_intr_mask;
996
997 qat_etr_bank_write_4(sc, bank, ETR_INT_COL_EN, intr_mask);
998 qat_etr_bank_write_4(sc, bank, ETR_INT_COL_CTL,
999 ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time);
1000 }
1001
1002 *rqr = qr;
1003
1004 return 0;
1005 }
1006
1007 static inline u_int
qat_modulo(u_int data,u_int shift)1008 qat_modulo(u_int data, u_int shift)
1009 {
1010 u_int div = data >> shift;
1011 u_int mult = div << shift;
1012 return data - mult;
1013 }
1014
1015 int
qat_etr_put_msg(struct qat_softc * sc,struct qat_ring * qr,uint32_t * msg)1016 qat_etr_put_msg(struct qat_softc *sc, struct qat_ring *qr, uint32_t *msg)
1017 {
1018 uint32_t inflight;
1019 uint32_t *addr;
1020
1021 mtx_lock(&qr->qr_ring_mtx);
1022
1023 inflight = atomic_fetchadd_32(qr->qr_inflight, 1) + 1;
1024 if (inflight > QAT_MAX_INFLIGHTS(qr->qr_ring_size, qr->qr_msg_size)) {
1025 atomic_subtract_32(qr->qr_inflight, 1);
1026 qr->qr_need_wakeup = true;
1027 mtx_unlock(&qr->qr_ring_mtx);
1028 counter_u64_add(sc->sc_ring_full_restarts, 1);
1029 return ERESTART;
1030 }
1031
1032 addr = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_tail);
1033
1034 memcpy(addr, msg, QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size));
1035
1036 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
1037 BUS_DMASYNC_PREWRITE);
1038
1039 qr->qr_tail = qat_modulo(qr->qr_tail +
1040 QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size),
1041 QAT_RING_SIZE_MODULO(qr->qr_ring_size));
1042
1043 qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring,
1044 ETR_RING_TAIL_OFFSET, qr->qr_tail);
1045
1046 mtx_unlock(&qr->qr_ring_mtx);
1047
1048 return 0;
1049 }
1050
1051 static int
qat_etr_ring_intr(struct qat_softc * sc,struct qat_bank * qb,struct qat_ring * qr)1052 qat_etr_ring_intr(struct qat_softc *sc, struct qat_bank *qb,
1053 struct qat_ring *qr)
1054 {
1055 uint32_t *msg, nmsg = 0;
1056 int handled = 0;
1057 bool blocked = false;
1058
1059 mtx_lock(&qr->qr_ring_mtx);
1060
1061 msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head);
1062
1063 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
1064 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1065
1066 while (atomic_load_32(msg) != ETR_RING_EMPTY_ENTRY_SIG) {
1067 atomic_subtract_32(qr->qr_inflight, 1);
1068
1069 if (qr->qr_cb != NULL) {
1070 mtx_unlock(&qr->qr_ring_mtx);
1071 handled |= qr->qr_cb(sc, qr->qr_cb_arg, msg);
1072 mtx_lock(&qr->qr_ring_mtx);
1073 }
1074
1075 atomic_store_32(msg, ETR_RING_EMPTY_ENTRY_SIG);
1076
1077 qr->qr_head = qat_modulo(qr->qr_head +
1078 QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size),
1079 QAT_RING_SIZE_MODULO(qr->qr_ring_size));
1080 nmsg++;
1081
1082 msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head);
1083 }
1084
1085 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
1086 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1087
1088 if (nmsg > 0) {
1089 qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring,
1090 ETR_RING_HEAD_OFFSET, qr->qr_head);
1091 if (qr->qr_need_wakeup) {
1092 blocked = true;
1093 qr->qr_need_wakeup = false;
1094 }
1095 }
1096
1097 mtx_unlock(&qr->qr_ring_mtx);
1098
1099 if (blocked)
1100 crypto_unblock(sc->sc_crypto.qcy_cid, CRYPTO_SYMQ);
1101
1102 return handled;
1103 }
1104
1105 static void
qat_etr_bank_intr(void * arg)1106 qat_etr_bank_intr(void *arg)
1107 {
1108 struct qat_bank *qb = arg;
1109 struct qat_softc *sc = qb->qb_sc;
1110 uint32_t estat;
1111 int i;
1112
1113 mtx_lock(&qb->qb_bank_mtx);
1114
1115 qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 0);
1116
1117 /* Now handle all the responses */
1118 estat = ~qat_etr_bank_read_4(sc, qb->qb_bank, ETR_E_STAT);
1119 estat &= qb->qb_intr_mask;
1120
1121 qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL,
1122 ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time);
1123
1124 mtx_unlock(&qb->qb_bank_mtx);
1125
1126 while ((i = ffs(estat)) != 0) {
1127 struct qat_ring *qr = &qb->qb_et_rings[--i];
1128 estat &= ~(1 << i);
1129 (void)qat_etr_ring_intr(sc, qb, qr);
1130 }
1131 }
1132
1133 void
qat_arb_update(struct qat_softc * sc,struct qat_bank * qb)1134 qat_arb_update(struct qat_softc *sc, struct qat_bank *qb)
1135 {
1136
1137 qat_arb_ringsrvarben_write_4(sc, qb->qb_bank,
1138 qb->qb_allocated_rings & 0xff);
1139 }
1140
1141 static struct qat_sym_cookie *
qat_crypto_alloc_sym_cookie(struct qat_crypto_bank * qcb)1142 qat_crypto_alloc_sym_cookie(struct qat_crypto_bank *qcb)
1143 {
1144 struct qat_sym_cookie *qsc;
1145
1146 mtx_lock(&qcb->qcb_bank_mtx);
1147
1148 if (qcb->qcb_symck_free_count == 0) {
1149 mtx_unlock(&qcb->qcb_bank_mtx);
1150 return NULL;
1151 }
1152
1153 qsc = qcb->qcb_symck_free[--qcb->qcb_symck_free_count];
1154
1155 mtx_unlock(&qcb->qcb_bank_mtx);
1156
1157 return qsc;
1158 }
1159
1160 static void
qat_crypto_free_sym_cookie(struct qat_crypto_bank * qcb,struct qat_sym_cookie * qsc)1161 qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb,
1162 struct qat_sym_cookie *qsc)
1163 {
1164 explicit_bzero(qsc->qsc_iv_buf, EALG_MAX_BLOCK_LEN);
1165 explicit_bzero(qsc->qsc_auth_res, QAT_SYM_HASH_BUFFER_LEN);
1166
1167 mtx_lock(&qcb->qcb_bank_mtx);
1168 qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc;
1169 mtx_unlock(&qcb->qcb_bank_mtx);
1170 }
1171
1172 void
qat_memcpy_htobe64(void * dst,const void * src,size_t len)1173 qat_memcpy_htobe64(void *dst, const void *src, size_t len)
1174 {
1175 uint64_t *dst0 = dst;
1176 const uint64_t *src0 = src;
1177 size_t i;
1178
1179 MPASS(len % sizeof(*dst0) == 0);
1180
1181 for (i = 0; i < len / sizeof(*dst0); i++)
1182 *(dst0 + i) = htobe64(*(src0 + i));
1183 }
1184
1185 void
qat_memcpy_htobe32(void * dst,const void * src,size_t len)1186 qat_memcpy_htobe32(void *dst, const void *src, size_t len)
1187 {
1188 uint32_t *dst0 = dst;
1189 const uint32_t *src0 = src;
1190 size_t i;
1191
1192 MPASS(len % sizeof(*dst0) == 0);
1193
1194 for (i = 0; i < len / sizeof(*dst0); i++)
1195 *(dst0 + i) = htobe32(*(src0 + i));
1196 }
1197
1198 void
qat_memcpy_htobe(void * dst,const void * src,size_t len,uint32_t wordbyte)1199 qat_memcpy_htobe(void *dst, const void *src, size_t len, uint32_t wordbyte)
1200 {
1201 switch (wordbyte) {
1202 case 4:
1203 qat_memcpy_htobe32(dst, src, len);
1204 break;
1205 case 8:
1206 qat_memcpy_htobe64(dst, src, len);
1207 break;
1208 default:
1209 panic("invalid word size %u", wordbyte);
1210 }
1211 }
1212
1213 void
qat_crypto_gmac_precompute(const struct qat_crypto_desc * desc,const uint8_t * key,int klen,const struct qat_sym_hash_def * hash_def,uint8_t * state)1214 qat_crypto_gmac_precompute(const struct qat_crypto_desc *desc,
1215 const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def,
1216 uint8_t *state)
1217 {
1218 uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)];
1219 char zeros[AES_BLOCK_LEN];
1220 int rounds;
1221
1222 memset(zeros, 0, sizeof(zeros));
1223 rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY);
1224 rijndaelEncrypt(ks, rounds, zeros, state);
1225 explicit_bzero(ks, sizeof(ks));
1226 }
1227
1228 void
qat_crypto_hmac_precompute(const struct qat_crypto_desc * desc,const uint8_t * key,int klen,const struct qat_sym_hash_def * hash_def,uint8_t * state1,uint8_t * state2)1229 qat_crypto_hmac_precompute(const struct qat_crypto_desc *desc,
1230 const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def,
1231 uint8_t *state1, uint8_t *state2)
1232 {
1233 union authctx ctx;
1234 const struct auth_hash *sah = hash_def->qshd_alg->qshai_sah;
1235 uint32_t state_offset = hash_def->qshd_alg->qshai_state_offset;
1236 uint32_t state_size = hash_def->qshd_alg->qshai_state_size;
1237 uint32_t state_word = hash_def->qshd_alg->qshai_state_word;
1238
1239 hmac_init_ipad(sah, key, klen, &ctx);
1240 qat_memcpy_htobe(state1, (uint8_t *)&ctx + state_offset, state_size,
1241 state_word);
1242 hmac_init_opad(sah, key, klen, &ctx);
1243 qat_memcpy_htobe(state2, (uint8_t *)&ctx + state_offset, state_size,
1244 state_word);
1245 explicit_bzero(&ctx, sizeof(ctx));
1246 }
1247
1248 static enum hw_cipher_algo
qat_aes_cipher_algo(int klen)1249 qat_aes_cipher_algo(int klen)
1250 {
1251 switch (klen) {
1252 case HW_AES_128_KEY_SZ:
1253 return HW_CIPHER_ALGO_AES128;
1254 case HW_AES_192_KEY_SZ:
1255 return HW_CIPHER_ALGO_AES192;
1256 case HW_AES_256_KEY_SZ:
1257 return HW_CIPHER_ALGO_AES256;
1258 default:
1259 panic("invalid key length %d", klen);
1260 }
1261 }
1262
1263 uint16_t
qat_crypto_load_cipher_session(const struct qat_crypto_desc * desc,const struct qat_session * qs)1264 qat_crypto_load_cipher_session(const struct qat_crypto_desc *desc,
1265 const struct qat_session *qs)
1266 {
1267 enum hw_cipher_algo algo;
1268 enum hw_cipher_dir dir;
1269 enum hw_cipher_convert key_convert;
1270 enum hw_cipher_mode mode;
1271
1272 dir = desc->qcd_cipher_dir;
1273 key_convert = HW_CIPHER_NO_CONVERT;
1274 mode = qs->qs_cipher_mode;
1275 switch (mode) {
1276 case HW_CIPHER_CBC_MODE:
1277 case HW_CIPHER_XTS_MODE:
1278 algo = qs->qs_cipher_algo;
1279
1280 /*
1281 * AES decrypt key needs to be reversed.
1282 * Instead of reversing the key at session registration,
1283 * it is instead reversed on-the-fly by setting the KEY_CONVERT
1284 * bit here.
1285 */
1286 if (desc->qcd_cipher_dir == HW_CIPHER_DECRYPT)
1287 key_convert = HW_CIPHER_KEY_CONVERT;
1288 break;
1289 case HW_CIPHER_CTR_MODE:
1290 algo = qs->qs_cipher_algo;
1291 dir = HW_CIPHER_ENCRYPT;
1292 break;
1293 default:
1294 panic("unhandled cipher mode %d", mode);
1295 break;
1296 }
1297
1298 return HW_CIPHER_CONFIG_BUILD(mode, algo, key_convert, dir);
1299 }
1300
1301 uint16_t
qat_crypto_load_auth_session(const struct qat_crypto_desc * desc,const struct qat_session * qs,const struct qat_sym_hash_def ** hash_def)1302 qat_crypto_load_auth_session(const struct qat_crypto_desc *desc,
1303 const struct qat_session *qs, const struct qat_sym_hash_def **hash_def)
1304 {
1305 enum qat_sym_hash_algorithm algo;
1306
1307 switch (qs->qs_auth_algo) {
1308 case HW_AUTH_ALGO_SHA1:
1309 algo = QAT_SYM_HASH_SHA1;
1310 break;
1311 case HW_AUTH_ALGO_SHA256:
1312 algo = QAT_SYM_HASH_SHA256;
1313 break;
1314 case HW_AUTH_ALGO_SHA384:
1315 algo = QAT_SYM_HASH_SHA384;
1316 break;
1317 case HW_AUTH_ALGO_SHA512:
1318 algo = QAT_SYM_HASH_SHA512;
1319 break;
1320 case HW_AUTH_ALGO_GALOIS_128:
1321 algo = QAT_SYM_HASH_AES_GCM;
1322 break;
1323 default:
1324 panic("unhandled auth algorithm %d", qs->qs_auth_algo);
1325 break;
1326 }
1327 *hash_def = &qat_sym_hash_defs[algo];
1328
1329 return HW_AUTH_CONFIG_BUILD(qs->qs_auth_mode,
1330 (*hash_def)->qshd_qat->qshqi_algo_enc,
1331 (*hash_def)->qshd_alg->qshai_digest_len);
1332 }
1333
1334 struct qat_crypto_load_cb_arg {
1335 struct qat_session *qs;
1336 struct qat_sym_cookie *qsc;
1337 struct cryptop *crp;
1338 int error;
1339 };
1340
1341 static int
qat_crypto_populate_buf_list(struct buffer_list_desc * buffers,bus_dma_segment_t * segs,int niseg,int noseg,int skip)1342 qat_crypto_populate_buf_list(struct buffer_list_desc *buffers,
1343 bus_dma_segment_t *segs, int niseg, int noseg, int skip)
1344 {
1345 struct flat_buffer_desc *flatbuf;
1346 bus_addr_t addr;
1347 bus_size_t len;
1348 int iseg, oseg;
1349
1350 for (iseg = 0, oseg = noseg; iseg < niseg && oseg < QAT_MAXSEG;
1351 iseg++) {
1352 addr = segs[iseg].ds_addr;
1353 len = segs[iseg].ds_len;
1354
1355 if (skip > 0) {
1356 if (skip < len) {
1357 addr += skip;
1358 len -= skip;
1359 skip = 0;
1360 } else {
1361 skip -= len;
1362 continue;
1363 }
1364 }
1365
1366 flatbuf = &buffers->flat_bufs[oseg++];
1367 flatbuf->data_len_in_bytes = (uint32_t)len;
1368 flatbuf->phy_buffer = (uint64_t)addr;
1369 }
1370 buffers->num_buffers = oseg;
1371 return iseg < niseg ? E2BIG : 0;
1372 }
1373
1374 static void
qat_crypto_load_aadbuf_cb(void * _arg,bus_dma_segment_t * segs,int nseg,int error)1375 qat_crypto_load_aadbuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
1376 int error)
1377 {
1378 struct qat_crypto_load_cb_arg *arg;
1379 struct qat_sym_cookie *qsc;
1380
1381 arg = _arg;
1382 if (error != 0) {
1383 arg->error = error;
1384 return;
1385 }
1386
1387 qsc = arg->qsc;
1388 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs,
1389 nseg, 0, 0);
1390 }
1391
1392 static void
qat_crypto_load_buf_cb(void * _arg,bus_dma_segment_t * segs,int nseg,int error)1393 qat_crypto_load_buf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
1394 int error)
1395 {
1396 struct cryptop *crp;
1397 struct qat_crypto_load_cb_arg *arg;
1398 struct qat_session *qs;
1399 struct qat_sym_cookie *qsc;
1400 int noseg, skip;
1401
1402 arg = _arg;
1403 if (error != 0) {
1404 arg->error = error;
1405 return;
1406 }
1407
1408 crp = arg->crp;
1409 qs = arg->qs;
1410 qsc = arg->qsc;
1411
1412 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
1413 /* AAD was handled in qat_crypto_load(). */
1414 skip = crp->crp_payload_start;
1415 noseg = 0;
1416 } else if (crp->crp_aad == NULL && crp->crp_aad_length > 0) {
1417 skip = crp->crp_aad_start;
1418 noseg = 0;
1419 } else {
1420 skip = crp->crp_payload_start;
1421 noseg = crp->crp_aad == NULL ?
1422 0 : qsc->qsc_buf_list.num_buffers;
1423 }
1424 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs,
1425 nseg, noseg, skip);
1426 }
1427
1428 static void
qat_crypto_load_obuf_cb(void * _arg,bus_dma_segment_t * segs,int nseg,int error)1429 qat_crypto_load_obuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
1430 int error)
1431 {
1432 struct buffer_list_desc *ibufs, *obufs;
1433 struct flat_buffer_desc *ibuf, *obuf;
1434 struct cryptop *crp;
1435 struct qat_crypto_load_cb_arg *arg;
1436 struct qat_session *qs;
1437 struct qat_sym_cookie *qsc;
1438 int buflen, osegs, tocopy;
1439
1440 arg = _arg;
1441 if (error != 0) {
1442 arg->error = error;
1443 return;
1444 }
1445
1446 crp = arg->crp;
1447 qs = arg->qs;
1448 qsc = arg->qsc;
1449
1450 /*
1451 * The payload must start at the same offset in the output SG list as in
1452 * the input SG list. Copy over SG entries from the input corresponding
1453 * to the AAD buffer.
1454 */
1455 osegs = 0;
1456 if (qs->qs_auth_algo != HW_AUTH_ALGO_GALOIS_128 &&
1457 crp->crp_aad_length > 0) {
1458 tocopy = crp->crp_aad == NULL ?
1459 crp->crp_payload_start - crp->crp_aad_start :
1460 crp->crp_aad_length;
1461
1462 ibufs = &qsc->qsc_buf_list;
1463 obufs = &qsc->qsc_obuf_list;
1464 for (; osegs < ibufs->num_buffers && tocopy > 0; osegs++) {
1465 ibuf = &ibufs->flat_bufs[osegs];
1466 obuf = &obufs->flat_bufs[osegs];
1467
1468 obuf->phy_buffer = ibuf->phy_buffer;
1469 buflen = imin(ibuf->data_len_in_bytes, tocopy);
1470 obuf->data_len_in_bytes = buflen;
1471 tocopy -= buflen;
1472 }
1473 }
1474
1475 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_obuf_list, segs,
1476 nseg, osegs, crp->crp_payload_output_start);
1477 }
1478
1479 static int
qat_crypto_load(struct qat_session * qs,struct qat_sym_cookie * qsc,struct qat_crypto_desc const * desc,struct cryptop * crp)1480 qat_crypto_load(struct qat_session *qs, struct qat_sym_cookie *qsc,
1481 struct qat_crypto_desc const *desc, struct cryptop *crp)
1482 {
1483 struct qat_crypto_load_cb_arg arg;
1484 int error;
1485
1486 crypto_read_iv(crp, qsc->qsc_iv_buf);
1487
1488 arg.crp = crp;
1489 arg.qs = qs;
1490 arg.qsc = qsc;
1491 arg.error = 0;
1492
1493 error = 0;
1494 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128 &&
1495 crp->crp_aad_length > 0) {
1496 /*
1497 * The firmware expects AAD to be in a contiguous buffer and
1498 * padded to a multiple of 16 bytes. To satisfy these
1499 * constraints we bounce the AAD into a per-request buffer.
1500 * There is a small limit on the AAD size so this is not too
1501 * onerous.
1502 */
1503 memset(qsc->qsc_gcm_aad, 0, QAT_GCM_AAD_SIZE_MAX);
1504 if (crp->crp_aad == NULL) {
1505 crypto_copydata(crp, crp->crp_aad_start,
1506 crp->crp_aad_length, qsc->qsc_gcm_aad);
1507 } else {
1508 memcpy(qsc->qsc_gcm_aad, crp->crp_aad,
1509 crp->crp_aad_length);
1510 }
1511 } else if (crp->crp_aad != NULL) {
1512 error = bus_dmamap_load(
1513 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag,
1514 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap,
1515 crp->crp_aad, crp->crp_aad_length,
1516 qat_crypto_load_aadbuf_cb, &arg, BUS_DMA_NOWAIT);
1517 if (error == 0)
1518 error = arg.error;
1519 }
1520 if (error == 0) {
1521 error = bus_dmamap_load_crp_buffer(
1522 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag,
1523 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap,
1524 &crp->crp_buf, qat_crypto_load_buf_cb, &arg,
1525 BUS_DMA_NOWAIT);
1526 if (error == 0)
1527 error = arg.error;
1528 }
1529 if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
1530 error = bus_dmamap_load_crp_buffer(
1531 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag,
1532 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap,
1533 &crp->crp_obuf, qat_crypto_load_obuf_cb, &arg,
1534 BUS_DMA_NOWAIT);
1535 if (error == 0)
1536 error = arg.error;
1537 }
1538 return error;
1539 }
1540
1541 static inline struct qat_crypto_bank *
qat_crypto_select_bank(struct qat_crypto * qcy)1542 qat_crypto_select_bank(struct qat_crypto *qcy)
1543 {
1544 u_int cpuid = PCPU_GET(cpuid);
1545
1546 return &qcy->qcy_banks[cpuid % qcy->qcy_num_banks];
1547 }
1548
1549 static int
qat_crypto_setup_ring(struct qat_softc * sc,struct qat_crypto_bank * qcb)1550 qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb)
1551 {
1552 char *name;
1553 int bank, curname, error, i, j;
1554
1555 bank = qcb->qcb_bank;
1556 curname = 0;
1557
1558 name = qcb->qcb_ring_names[curname++];
1559 snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank);
1560 error = qat_etr_setup_ring(sc, qcb->qcb_bank,
1561 sc->sc_hw.qhw_ring_sym_tx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_req_size,
1562 NULL, NULL, name, &qcb->qcb_sym_tx);
1563 if (error)
1564 return error;
1565
1566 name = qcb->qcb_ring_names[curname++];
1567 snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_rx", bank);
1568 error = qat_etr_setup_ring(sc, qcb->qcb_bank,
1569 sc->sc_hw.qhw_ring_sym_rx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_resp_size,
1570 qat_crypto_sym_rxintr, qcb, name, &qcb->qcb_sym_rx);
1571 if (error)
1572 return error;
1573
1574 for (i = 0; i < QAT_NSYMCOOKIE; i++) {
1575 struct qat_dmamem *qdm = &qcb->qcb_symck_dmamems[i];
1576 struct qat_sym_cookie *qsc;
1577
1578 error = qat_alloc_dmamem(sc, qdm, 1,
1579 sizeof(struct qat_sym_cookie), QAT_OPTIMAL_ALIGN);
1580 if (error)
1581 return error;
1582
1583 qsc = qdm->qdm_dma_vaddr;
1584 qsc->qsc_self_dmamap = qdm->qdm_dma_map;
1585 qsc->qsc_self_dma_tag = qdm->qdm_dma_tag;
1586 qsc->qsc_bulk_req_params_buf_paddr =
1587 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1588 qsc_bulk_cookie.qsbc_req_params_buf);
1589 qsc->qsc_buffer_list_desc_paddr =
1590 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1591 qsc_buf_list);
1592 qsc->qsc_obuffer_list_desc_paddr =
1593 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1594 qsc_obuf_list);
1595 qsc->qsc_obuffer_list_desc_paddr =
1596 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1597 qsc_obuf_list);
1598 qsc->qsc_iv_buf_paddr =
1599 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1600 qsc_iv_buf);
1601 qsc->qsc_auth_res_paddr =
1602 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1603 qsc_auth_res);
1604 qsc->qsc_gcm_aad_paddr =
1605 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1606 qsc_gcm_aad);
1607 qsc->qsc_content_desc_paddr =
1608 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1609 qsc_content_desc);
1610 qcb->qcb_symck_free[i] = qsc;
1611 qcb->qcb_symck_free_count++;
1612
1613 for (j = 0; j < QAT_SYM_DMA_COUNT; j++) {
1614 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),
1615 1, 0, /* alignment, boundary */
1616 BUS_SPACE_MAXADDR, /* lowaddr */
1617 BUS_SPACE_MAXADDR, /* highaddr */
1618 NULL, NULL, /* filter, filterarg */
1619 QAT_MAXLEN, /* maxsize */
1620 QAT_MAXSEG, /* nsegments */
1621 QAT_MAXLEN, /* maxsegsize */
1622 BUS_DMA_COHERENT, /* flags */
1623 NULL, NULL, /* lockfunc, lockarg */
1624 &qsc->qsc_dma[j].qsd_dma_tag);
1625 if (error != 0)
1626 return error;
1627 error = bus_dmamap_create(qsc->qsc_dma[j].qsd_dma_tag,
1628 BUS_DMA_COHERENT, &qsc->qsc_dma[j].qsd_dmamap);
1629 if (error != 0)
1630 return error;
1631 }
1632 }
1633
1634 return 0;
1635 }
1636
1637 static int
qat_crypto_bank_init(struct qat_softc * sc,struct qat_crypto_bank * qcb)1638 qat_crypto_bank_init(struct qat_softc *sc, struct qat_crypto_bank *qcb)
1639 {
1640 mtx_init(&qcb->qcb_bank_mtx, "qcb bank", NULL, MTX_DEF);
1641
1642 return qat_crypto_setup_ring(sc, qcb);
1643 }
1644
1645 static void
qat_crypto_bank_deinit(struct qat_softc * sc,struct qat_crypto_bank * qcb)1646 qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb)
1647 {
1648 struct qat_dmamem *qdm;
1649 struct qat_sym_cookie *qsc;
1650 int i, j;
1651
1652 for (i = 0; i < QAT_NSYMCOOKIE; i++) {
1653 qdm = &qcb->qcb_symck_dmamems[i];
1654 qsc = qcb->qcb_symck_free[i];
1655 for (j = 0; j < QAT_SYM_DMA_COUNT; j++) {
1656 bus_dmamap_destroy(qsc->qsc_dma[j].qsd_dma_tag,
1657 qsc->qsc_dma[j].qsd_dmamap);
1658 bus_dma_tag_destroy(qsc->qsc_dma[j].qsd_dma_tag);
1659 }
1660 qat_free_dmamem(sc, qdm);
1661 }
1662 qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma);
1663 qat_free_dmamem(sc, &qcb->qcb_sym_rx->qr_dma);
1664
1665 mtx_destroy(&qcb->qcb_bank_mtx);
1666 }
1667
1668 static int
qat_crypto_init(struct qat_softc * sc)1669 qat_crypto_init(struct qat_softc *sc)
1670 {
1671 struct qat_crypto *qcy = &sc->sc_crypto;
1672 struct sysctl_ctx_list *ctx;
1673 struct sysctl_oid *oid;
1674 struct sysctl_oid_list *children;
1675 int bank, error, num_banks;
1676
1677 qcy->qcy_sc = sc;
1678
1679 if (sc->sc_hw.qhw_init_arb != NULL)
1680 num_banks = imin(mp_ncpus, sc->sc_hw.qhw_num_banks);
1681 else
1682 num_banks = sc->sc_ae_num;
1683
1684 qcy->qcy_num_banks = num_banks;
1685
1686 qcy->qcy_banks =
1687 qat_alloc_mem(sizeof(struct qat_crypto_bank) * num_banks);
1688
1689 for (bank = 0; bank < num_banks; bank++) {
1690 struct qat_crypto_bank *qcb = &qcy->qcy_banks[bank];
1691 qcb->qcb_bank = bank;
1692 error = qat_crypto_bank_init(sc, qcb);
1693 if (error)
1694 return error;
1695 }
1696
1697 mtx_init(&qcy->qcy_crypto_mtx, "qcy crypto", NULL, MTX_DEF);
1698
1699 ctx = device_get_sysctl_ctx(sc->sc_dev);
1700 oid = device_get_sysctl_tree(sc->sc_dev);
1701 children = SYSCTL_CHILDREN(oid);
1702 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats",
1703 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics");
1704 children = SYSCTL_CHILDREN(oid);
1705
1706 sc->sc_gcm_aad_restarts = counter_u64_alloc(M_WAITOK);
1707 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_restarts",
1708 CTLFLAG_RD, &sc->sc_gcm_aad_restarts,
1709 "GCM requests deferred due to AAD size change");
1710 sc->sc_gcm_aad_updates = counter_u64_alloc(M_WAITOK);
1711 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_updates",
1712 CTLFLAG_RD, &sc->sc_gcm_aad_updates,
1713 "GCM requests that required session state update");
1714 sc->sc_ring_full_restarts = counter_u64_alloc(M_WAITOK);
1715 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ring_full",
1716 CTLFLAG_RD, &sc->sc_ring_full_restarts,
1717 "Requests deferred due to in-flight max reached");
1718 sc->sc_sym_alloc_failures = counter_u64_alloc(M_WAITOK);
1719 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sym_alloc_failures",
1720 CTLFLAG_RD, &sc->sc_sym_alloc_failures,
1721 "Request allocation failures");
1722
1723 return 0;
1724 }
1725
1726 static void
qat_crypto_deinit(struct qat_softc * sc)1727 qat_crypto_deinit(struct qat_softc *sc)
1728 {
1729 struct qat_crypto *qcy = &sc->sc_crypto;
1730 struct qat_crypto_bank *qcb;
1731 int bank;
1732
1733 counter_u64_free(sc->sc_sym_alloc_failures);
1734 counter_u64_free(sc->sc_ring_full_restarts);
1735 counter_u64_free(sc->sc_gcm_aad_updates);
1736 counter_u64_free(sc->sc_gcm_aad_restarts);
1737
1738 if (qcy->qcy_banks != NULL) {
1739 for (bank = 0; bank < qcy->qcy_num_banks; bank++) {
1740 qcb = &qcy->qcy_banks[bank];
1741 qat_crypto_bank_deinit(sc, qcb);
1742 }
1743 qat_free_mem(qcy->qcy_banks);
1744 mtx_destroy(&qcy->qcy_crypto_mtx);
1745 }
1746 }
1747
1748 static int
qat_crypto_start(struct qat_softc * sc)1749 qat_crypto_start(struct qat_softc *sc)
1750 {
1751 struct qat_crypto *qcy;
1752
1753 qcy = &sc->sc_crypto;
1754 qcy->qcy_cid = crypto_get_driverid(sc->sc_dev,
1755 sizeof(struct qat_session), CRYPTOCAP_F_HARDWARE);
1756 if (qcy->qcy_cid < 0) {
1757 device_printf(sc->sc_dev,
1758 "could not get opencrypto driver id\n");
1759 return ENOENT;
1760 }
1761
1762 return 0;
1763 }
1764
1765 static void
qat_crypto_stop(struct qat_softc * sc)1766 qat_crypto_stop(struct qat_softc *sc)
1767 {
1768 struct qat_crypto *qcy;
1769
1770 qcy = &sc->sc_crypto;
1771 if (qcy->qcy_cid >= 0)
1772 (void)crypto_unregister_all(qcy->qcy_cid);
1773 }
1774
1775 static void
qat_crypto_sym_dma_unload(struct qat_sym_cookie * qsc,enum qat_sym_dma i)1776 qat_crypto_sym_dma_unload(struct qat_sym_cookie *qsc, enum qat_sym_dma i)
1777 {
1778 bus_dmamap_sync(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap,
1779 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1780 bus_dmamap_unload(qsc->qsc_dma[i].qsd_dma_tag,
1781 qsc->qsc_dma[i].qsd_dmamap);
1782 }
1783
1784 static int
qat_crypto_sym_rxintr(struct qat_softc * sc,void * arg,void * msg)1785 qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg)
1786 {
1787 char icv[QAT_SYM_HASH_BUFFER_LEN];
1788 struct qat_crypto_bank *qcb = arg;
1789 struct qat_crypto *qcy;
1790 struct qat_session *qs;
1791 struct qat_sym_cookie *qsc;
1792 struct qat_sym_bulk_cookie *qsbc;
1793 struct cryptop *crp;
1794 int error;
1795 uint16_t auth_sz;
1796 bool blocked;
1797
1798 qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset);
1799
1800 qsbc = &qsc->qsc_bulk_cookie;
1801 qcy = qsbc->qsbc_crypto;
1802 qs = qsbc->qsbc_session;
1803 crp = qsbc->qsbc_cb_tag;
1804
1805 bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap,
1806 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1807
1808 if (crp->crp_aad != NULL)
1809 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_AADBUF);
1810 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_BUF);
1811 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
1812 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_OBUF);
1813
1814 error = 0;
1815 if ((auth_sz = qs->qs_auth_mlen) != 0) {
1816 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
1817 crypto_copydata(crp, crp->crp_digest_start,
1818 auth_sz, icv);
1819 if (timingsafe_bcmp(icv, qsc->qsc_auth_res,
1820 auth_sz) != 0) {
1821 error = EBADMSG;
1822 }
1823 } else {
1824 crypto_copyback(crp, crp->crp_digest_start,
1825 auth_sz, qsc->qsc_auth_res);
1826 }
1827 }
1828
1829 qat_crypto_free_sym_cookie(qcb, qsc);
1830
1831 blocked = false;
1832 mtx_lock(&qs->qs_session_mtx);
1833 MPASS(qs->qs_status & QAT_SESSION_STATUS_ACTIVE);
1834 qs->qs_inflight--;
1835 if (__predict_false(qs->qs_need_wakeup && qs->qs_inflight == 0)) {
1836 blocked = true;
1837 qs->qs_need_wakeup = false;
1838 }
1839 mtx_unlock(&qs->qs_session_mtx);
1840
1841 crp->crp_etype = error;
1842 crypto_done(crp);
1843
1844 if (blocked)
1845 crypto_unblock(qcy->qcy_cid, CRYPTO_SYMQ);
1846
1847 return 1;
1848 }
1849
1850 static int
qat_probesession(device_t dev,const struct crypto_session_params * csp)1851 qat_probesession(device_t dev, const struct crypto_session_params *csp)
1852 {
1853 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
1854 0)
1855 return EINVAL;
1856
1857 if (csp->csp_cipher_alg == CRYPTO_AES_XTS &&
1858 qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) {
1859 /*
1860 * AES-XTS is not supported by the NanoQAT.
1861 */
1862 return EINVAL;
1863 }
1864
1865 switch (csp->csp_mode) {
1866 case CSP_MODE_CIPHER:
1867 switch (csp->csp_cipher_alg) {
1868 case CRYPTO_AES_CBC:
1869 case CRYPTO_AES_ICM:
1870 if (csp->csp_ivlen != AES_BLOCK_LEN)
1871 return EINVAL;
1872 break;
1873 case CRYPTO_AES_XTS:
1874 if (csp->csp_ivlen != AES_XTS_IV_LEN)
1875 return EINVAL;
1876 break;
1877 default:
1878 return EINVAL;
1879 }
1880 break;
1881 case CSP_MODE_DIGEST:
1882 switch (csp->csp_auth_alg) {
1883 case CRYPTO_SHA1:
1884 case CRYPTO_SHA1_HMAC:
1885 case CRYPTO_SHA2_256:
1886 case CRYPTO_SHA2_256_HMAC:
1887 case CRYPTO_SHA2_384:
1888 case CRYPTO_SHA2_384_HMAC:
1889 case CRYPTO_SHA2_512:
1890 case CRYPTO_SHA2_512_HMAC:
1891 break;
1892 case CRYPTO_AES_NIST_GMAC:
1893 if (csp->csp_ivlen != AES_GCM_IV_LEN)
1894 return EINVAL;
1895 break;
1896 default:
1897 return EINVAL;
1898 }
1899 break;
1900 case CSP_MODE_AEAD:
1901 switch (csp->csp_cipher_alg) {
1902 case CRYPTO_AES_NIST_GCM_16:
1903 if (csp->csp_ivlen != AES_GCM_IV_LEN)
1904 return EINVAL;
1905 break;
1906 default:
1907 return EINVAL;
1908 }
1909 break;
1910 case CSP_MODE_ETA:
1911 switch (csp->csp_auth_alg) {
1912 case CRYPTO_SHA1_HMAC:
1913 case CRYPTO_SHA2_256_HMAC:
1914 case CRYPTO_SHA2_384_HMAC:
1915 case CRYPTO_SHA2_512_HMAC:
1916 switch (csp->csp_cipher_alg) {
1917 case CRYPTO_AES_CBC:
1918 case CRYPTO_AES_ICM:
1919 if (csp->csp_ivlen != AES_BLOCK_LEN)
1920 return EINVAL;
1921 break;
1922 case CRYPTO_AES_XTS:
1923 if (csp->csp_ivlen != AES_XTS_IV_LEN)
1924 return EINVAL;
1925 break;
1926 default:
1927 return EINVAL;
1928 }
1929 break;
1930 default:
1931 return EINVAL;
1932 }
1933 break;
1934 default:
1935 return EINVAL;
1936 }
1937
1938 return CRYPTODEV_PROBE_HARDWARE;
1939 }
1940
1941 static int
qat_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)1942 qat_newsession(device_t dev, crypto_session_t cses,
1943 const struct crypto_session_params *csp)
1944 {
1945 struct qat_crypto *qcy;
1946 struct qat_dmamem *qdm;
1947 struct qat_session *qs;
1948 struct qat_softc *sc;
1949 struct qat_crypto_desc *ddesc, *edesc;
1950 int error, slices;
1951
1952 sc = device_get_softc(dev);
1953 qs = crypto_get_driver_session(cses);
1954 qcy = &sc->sc_crypto;
1955
1956 qdm = &qs->qs_desc_mem;
1957 error = qat_alloc_dmamem(sc, qdm, QAT_MAXSEG,
1958 sizeof(struct qat_crypto_desc) * 2, QAT_OPTIMAL_ALIGN);
1959 if (error != 0)
1960 return error;
1961
1962 mtx_init(&qs->qs_session_mtx, "qs session", NULL, MTX_DEF);
1963 qs->qs_aad_length = -1;
1964
1965 qs->qs_dec_desc = ddesc = qdm->qdm_dma_vaddr;
1966 qs->qs_enc_desc = edesc = ddesc + 1;
1967
1968 ddesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr;
1969 ddesc->qcd_hash_state_paddr = ddesc->qcd_desc_paddr +
1970 offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf);
1971 edesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr +
1972 sizeof(struct qat_crypto_desc);
1973 edesc->qcd_hash_state_paddr = edesc->qcd_desc_paddr +
1974 offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf);
1975
1976 qs->qs_status = QAT_SESSION_STATUS_ACTIVE;
1977 qs->qs_inflight = 0;
1978
1979 qs->qs_cipher_key = csp->csp_cipher_key;
1980 qs->qs_cipher_klen = csp->csp_cipher_klen;
1981 qs->qs_auth_key = csp->csp_auth_key;
1982 qs->qs_auth_klen = csp->csp_auth_klen;
1983
1984 switch (csp->csp_cipher_alg) {
1985 case CRYPTO_AES_CBC:
1986 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
1987 qs->qs_cipher_mode = HW_CIPHER_CBC_MODE;
1988 break;
1989 case CRYPTO_AES_ICM:
1990 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
1991 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
1992 break;
1993 case CRYPTO_AES_XTS:
1994 qs->qs_cipher_algo =
1995 qat_aes_cipher_algo(csp->csp_cipher_klen / 2);
1996 qs->qs_cipher_mode = HW_CIPHER_XTS_MODE;
1997 break;
1998 case CRYPTO_AES_NIST_GCM_16:
1999 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
2000 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
2001 qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128;
2002 qs->qs_auth_mode = HW_AUTH_MODE1;
2003 break;
2004 case 0:
2005 break;
2006 default:
2007 panic("%s: unhandled cipher algorithm %d", __func__,
2008 csp->csp_cipher_alg);
2009 }
2010
2011 switch (csp->csp_auth_alg) {
2012 case CRYPTO_SHA1_HMAC:
2013 qs->qs_auth_algo = HW_AUTH_ALGO_SHA1;
2014 qs->qs_auth_mode = HW_AUTH_MODE1;
2015 break;
2016 case CRYPTO_SHA1:
2017 qs->qs_auth_algo = HW_AUTH_ALGO_SHA1;
2018 qs->qs_auth_mode = HW_AUTH_MODE0;
2019 break;
2020 case CRYPTO_SHA2_256_HMAC:
2021 qs->qs_auth_algo = HW_AUTH_ALGO_SHA256;
2022 qs->qs_auth_mode = HW_AUTH_MODE1;
2023 break;
2024 case CRYPTO_SHA2_256:
2025 qs->qs_auth_algo = HW_AUTH_ALGO_SHA256;
2026 qs->qs_auth_mode = HW_AUTH_MODE0;
2027 break;
2028 case CRYPTO_SHA2_384_HMAC:
2029 qs->qs_auth_algo = HW_AUTH_ALGO_SHA384;
2030 qs->qs_auth_mode = HW_AUTH_MODE1;
2031 break;
2032 case CRYPTO_SHA2_384:
2033 qs->qs_auth_algo = HW_AUTH_ALGO_SHA384;
2034 qs->qs_auth_mode = HW_AUTH_MODE0;
2035 break;
2036 case CRYPTO_SHA2_512_HMAC:
2037 qs->qs_auth_algo = HW_AUTH_ALGO_SHA512;
2038 qs->qs_auth_mode = HW_AUTH_MODE1;
2039 break;
2040 case CRYPTO_SHA2_512:
2041 qs->qs_auth_algo = HW_AUTH_ALGO_SHA512;
2042 qs->qs_auth_mode = HW_AUTH_MODE0;
2043 break;
2044 case CRYPTO_AES_NIST_GMAC:
2045 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_auth_klen);
2046 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
2047 qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128;
2048 qs->qs_auth_mode = HW_AUTH_MODE1;
2049
2050 qs->qs_cipher_key = qs->qs_auth_key;
2051 qs->qs_cipher_klen = qs->qs_auth_klen;
2052 break;
2053 case 0:
2054 break;
2055 default:
2056 panic("%s: unhandled auth algorithm %d", __func__,
2057 csp->csp_auth_alg);
2058 }
2059
2060 slices = 0;
2061 switch (csp->csp_mode) {
2062 case CSP_MODE_AEAD:
2063 case CSP_MODE_ETA:
2064 /* auth then decrypt */
2065 ddesc->qcd_slices[0] = FW_SLICE_AUTH;
2066 ddesc->qcd_slices[1] = FW_SLICE_CIPHER;
2067 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
2068 ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER;
2069 /* encrypt then auth */
2070 edesc->qcd_slices[0] = FW_SLICE_CIPHER;
2071 edesc->qcd_slices[1] = FW_SLICE_AUTH;
2072 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
2073 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH;
2074 slices = 2;
2075 break;
2076 case CSP_MODE_CIPHER:
2077 /* decrypt */
2078 ddesc->qcd_slices[0] = FW_SLICE_CIPHER;
2079 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
2080 ddesc->qcd_cmd_id = FW_LA_CMD_CIPHER;
2081 /* encrypt */
2082 edesc->qcd_slices[0] = FW_SLICE_CIPHER;
2083 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
2084 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER;
2085 slices = 1;
2086 break;
2087 case CSP_MODE_DIGEST:
2088 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
2089 /* auth then decrypt */
2090 ddesc->qcd_slices[0] = FW_SLICE_AUTH;
2091 ddesc->qcd_slices[1] = FW_SLICE_CIPHER;
2092 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
2093 ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER;
2094 /* encrypt then auth */
2095 edesc->qcd_slices[0] = FW_SLICE_CIPHER;
2096 edesc->qcd_slices[1] = FW_SLICE_AUTH;
2097 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
2098 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH;
2099 slices = 2;
2100 } else {
2101 ddesc->qcd_slices[0] = FW_SLICE_AUTH;
2102 ddesc->qcd_cmd_id = FW_LA_CMD_AUTH;
2103 edesc->qcd_slices[0] = FW_SLICE_AUTH;
2104 edesc->qcd_cmd_id = FW_LA_CMD_AUTH;
2105 slices = 1;
2106 }
2107 break;
2108 default:
2109 panic("%s: unhandled crypto algorithm %d, %d", __func__,
2110 csp->csp_cipher_alg, csp->csp_auth_alg);
2111 }
2112 ddesc->qcd_slices[slices] = FW_SLICE_DRAM_WR;
2113 edesc->qcd_slices[slices] = FW_SLICE_DRAM_WR;
2114
2115 qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, ddesc);
2116 qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, edesc);
2117
2118 if (csp->csp_auth_mlen != 0)
2119 qs->qs_auth_mlen = csp->csp_auth_mlen;
2120 else
2121 qs->qs_auth_mlen = edesc->qcd_auth_sz;
2122
2123 /* Compute the GMAC by specifying a null cipher payload. */
2124 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC)
2125 ddesc->qcd_cmd_id = edesc->qcd_cmd_id = FW_LA_CMD_AUTH;
2126
2127 return 0;
2128 }
2129
2130 static void
qat_crypto_clear_desc(struct qat_crypto_desc * desc)2131 qat_crypto_clear_desc(struct qat_crypto_desc *desc)
2132 {
2133 explicit_bzero(desc->qcd_content_desc, sizeof(desc->qcd_content_desc));
2134 explicit_bzero(desc->qcd_hash_state_prefix_buf,
2135 sizeof(desc->qcd_hash_state_prefix_buf));
2136 explicit_bzero(desc->qcd_req_cache, sizeof(desc->qcd_req_cache));
2137 }
2138
2139 static void
qat_freesession(device_t dev,crypto_session_t cses)2140 qat_freesession(device_t dev, crypto_session_t cses)
2141 {
2142 struct qat_session *qs;
2143
2144 qs = crypto_get_driver_session(cses);
2145 KASSERT(qs->qs_inflight == 0,
2146 ("%s: session %p has requests in flight", __func__, qs));
2147
2148 qat_crypto_clear_desc(qs->qs_enc_desc);
2149 qat_crypto_clear_desc(qs->qs_dec_desc);
2150 qat_free_dmamem(device_get_softc(dev), &qs->qs_desc_mem);
2151 mtx_destroy(&qs->qs_session_mtx);
2152 }
2153
2154 static int
qat_process(device_t dev,struct cryptop * crp,int hint)2155 qat_process(device_t dev, struct cryptop *crp, int hint)
2156 {
2157 struct qat_crypto *qcy;
2158 struct qat_crypto_bank *qcb;
2159 struct qat_crypto_desc const *desc;
2160 struct qat_session *qs;
2161 struct qat_softc *sc;
2162 struct qat_sym_cookie *qsc;
2163 struct qat_sym_bulk_cookie *qsbc;
2164 int error;
2165
2166 sc = device_get_softc(dev);
2167 qcy = &sc->sc_crypto;
2168 qs = crypto_get_driver_session(crp->crp_session);
2169 qsc = NULL;
2170
2171 if (__predict_false(crypto_buffer_len(&crp->crp_buf) > QAT_MAXLEN)) {
2172 error = E2BIG;
2173 goto fail1;
2174 }
2175
2176 mtx_lock(&qs->qs_session_mtx);
2177 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
2178 if (crp->crp_aad_length > QAT_GCM_AAD_SIZE_MAX) {
2179 error = E2BIG;
2180 mtx_unlock(&qs->qs_session_mtx);
2181 goto fail1;
2182 }
2183
2184 /*
2185 * The firmware interface for GCM annoyingly requires the AAD
2186 * size to be stored in the session's content descriptor, which
2187 * is not really meant to be updated after session
2188 * initialization. For IPSec the AAD size is fixed so this is
2189 * not much of a problem in practice, but we have to catch AAD
2190 * size updates here so that the device code can safely update
2191 * the session's recorded AAD size.
2192 */
2193 if (__predict_false(crp->crp_aad_length != qs->qs_aad_length)) {
2194 if (qs->qs_inflight == 0) {
2195 if (qs->qs_aad_length != -1) {
2196 counter_u64_add(sc->sc_gcm_aad_updates,
2197 1);
2198 }
2199 qs->qs_aad_length = crp->crp_aad_length;
2200 } else {
2201 qs->qs_need_wakeup = true;
2202 mtx_unlock(&qs->qs_session_mtx);
2203 counter_u64_add(sc->sc_gcm_aad_restarts, 1);
2204 error = ERESTART;
2205 goto fail1;
2206 }
2207 }
2208 }
2209 qs->qs_inflight++;
2210 mtx_unlock(&qs->qs_session_mtx);
2211
2212 qcb = qat_crypto_select_bank(qcy);
2213
2214 qsc = qat_crypto_alloc_sym_cookie(qcb);
2215 if (qsc == NULL) {
2216 counter_u64_add(sc->sc_sym_alloc_failures, 1);
2217 error = ENOBUFS;
2218 goto fail2;
2219 }
2220
2221 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
2222 desc = qs->qs_enc_desc;
2223 else
2224 desc = qs->qs_dec_desc;
2225
2226 error = qat_crypto_load(qs, qsc, desc, crp);
2227 if (error != 0)
2228 goto fail2;
2229
2230 qsbc = &qsc->qsc_bulk_cookie;
2231 qsbc->qsbc_crypto = qcy;
2232 qsbc->qsbc_session = qs;
2233 qsbc->qsbc_cb_tag = crp;
2234
2235 sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp);
2236
2237 if (crp->crp_aad != NULL) {
2238 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag,
2239 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap,
2240 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2241 }
2242 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag,
2243 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap,
2244 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2245 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
2246 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag,
2247 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap,
2248 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2249 }
2250 bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap,
2251 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2252
2253 error = qat_etr_put_msg(sc, qcb->qcb_sym_tx,
2254 (uint32_t *)qsbc->qsbc_msg);
2255 if (error)
2256 goto fail2;
2257
2258 return 0;
2259
2260 fail2:
2261 if (qsc)
2262 qat_crypto_free_sym_cookie(qcb, qsc);
2263 mtx_lock(&qs->qs_session_mtx);
2264 qs->qs_inflight--;
2265 mtx_unlock(&qs->qs_session_mtx);
2266 fail1:
2267 crp->crp_etype = error;
2268 crypto_done(crp);
2269 return 0;
2270 }
2271
2272 static device_method_t qat_methods[] = {
2273 /* Device interface */
2274 DEVMETHOD(device_probe, qat_probe),
2275 DEVMETHOD(device_attach, qat_attach),
2276 DEVMETHOD(device_detach, qat_detach),
2277
2278 /* Cryptodev interface */
2279 DEVMETHOD(cryptodev_probesession, qat_probesession),
2280 DEVMETHOD(cryptodev_newsession, qat_newsession),
2281 DEVMETHOD(cryptodev_freesession, qat_freesession),
2282 DEVMETHOD(cryptodev_process, qat_process),
2283
2284 DEVMETHOD_END
2285 };
2286
2287 static devclass_t qat_devclass;
2288
2289 static driver_t qat_driver = {
2290 .name = "qat",
2291 .methods = qat_methods,
2292 .size = sizeof(struct qat_softc),
2293 };
2294
2295 DRIVER_MODULE(qat, pci, qat_driver, qat_devclass, 0, 0);
2296 MODULE_VERSION(qat, 1);
2297 MODULE_DEPEND(qat, crypto, 1, 1, 1);
2298 MODULE_DEPEND(qat, pci, 1, 1, 1);
2299