1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2016 Intel Corporation
3 */
4
5 #ifndef _AESNI_MB_PMD_PRIVATE_H_
6 #define _AESNI_MB_PMD_PRIVATE_H_
7
8 #include <intel-ipsec-mb.h>
9
10 #if defined(RTE_LIB_SECURITY) && (IMB_VERSION_NUM) >= IMB_VERSION(0, 54, 0)
11 #define AESNI_MB_DOCSIS_SEC_ENABLED 1
12 #include <rte_security.h>
13 #include <rte_security_driver.h>
14 #endif
15
16 enum aesni_mb_vector_mode {
17 RTE_AESNI_MB_NOT_SUPPORTED = 0,
18 RTE_AESNI_MB_SSE,
19 RTE_AESNI_MB_AVX,
20 RTE_AESNI_MB_AVX2,
21 RTE_AESNI_MB_AVX512
22 };
23
24 #define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb
25 /**< AES-NI Multi buffer PMD device name */
26
27 /** AESNI_MB PMD LOGTYPE DRIVER */
28 extern int aesni_mb_logtype_driver;
29
30 #define AESNI_MB_LOG(level, fmt, ...) \
31 rte_log(RTE_LOG_ ## level, aesni_mb_logtype_driver, \
32 "%s() line %u: " fmt "\n", __func__, __LINE__, \
33 ## __VA_ARGS__)
34
35
36 #define HMAC_IPAD_VALUE (0x36)
37 #define HMAC_OPAD_VALUE (0x5C)
38
39 /* Maximum length for digest */
40 #define DIGEST_LENGTH_MAX 64
41 static const unsigned auth_blocksize[] = {
42 [NULL_HASH] = 0,
43 [MD5] = 64,
44 [SHA1] = 64,
45 [SHA_224] = 64,
46 [SHA_256] = 64,
47 [SHA_384] = 128,
48 [SHA_512] = 128,
49 [AES_XCBC] = 16,
50 [AES_CCM] = 16,
51 [AES_CMAC] = 16,
52 [AES_GMAC] = 16,
53 [PLAIN_SHA1] = 64,
54 [PLAIN_SHA_224] = 64,
55 [PLAIN_SHA_256] = 64,
56 [PLAIN_SHA_384] = 128,
57 [PLAIN_SHA_512] = 128,
58 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
59 [IMB_AUTH_ZUC_EIA3_BITLEN] = 16,
60 [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 16,
61 [IMB_AUTH_KASUMI_UIA1] = 16
62 #endif
63 };
64
65 /**
66 * Get the blocksize in bytes for a specified authentication algorithm
67 *
68 * @Note: this function will not return a valid value for a non-valid
69 * authentication algorithm
70 */
71 static inline unsigned
get_auth_algo_blocksize(JOB_HASH_ALG algo)72 get_auth_algo_blocksize(JOB_HASH_ALG algo)
73 {
74 return auth_blocksize[algo];
75 }
76
77 static const unsigned auth_truncated_digest_byte_lengths[] = {
78 [MD5] = 12,
79 [SHA1] = 12,
80 [SHA_224] = 14,
81 [SHA_256] = 16,
82 [SHA_384] = 24,
83 [SHA_512] = 32,
84 [AES_XCBC] = 12,
85 [AES_CMAC] = 12,
86 [AES_CCM] = 8,
87 [NULL_HASH] = 0,
88 [AES_GMAC] = 12,
89 [PLAIN_SHA1] = 20,
90 [PLAIN_SHA_224] = 28,
91 [PLAIN_SHA_256] = 32,
92 [PLAIN_SHA_384] = 48,
93 [PLAIN_SHA_512] = 64,
94 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
95 [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
96 [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
97 [IMB_AUTH_KASUMI_UIA1] = 4
98 #endif
99 };
100
101 /**
102 * Get the IPsec specified truncated length in bytes of the HMAC digest for a
103 * specified authentication algorithm
104 *
105 * @Note: this function will not return a valid value for a non-valid
106 * authentication algorithm
107 */
108 static inline unsigned
get_truncated_digest_byte_length(JOB_HASH_ALG algo)109 get_truncated_digest_byte_length(JOB_HASH_ALG algo)
110 {
111 return auth_truncated_digest_byte_lengths[algo];
112 }
113
114 static const unsigned auth_digest_byte_lengths[] = {
115 [MD5] = 16,
116 [SHA1] = 20,
117 [SHA_224] = 28,
118 [SHA_256] = 32,
119 [SHA_384] = 48,
120 [SHA_512] = 64,
121 [AES_XCBC] = 16,
122 [AES_CMAC] = 16,
123 [AES_CCM] = 16,
124 [AES_GMAC] = 16,
125 [NULL_HASH] = 0,
126 [PLAIN_SHA1] = 20,
127 [PLAIN_SHA_224] = 28,
128 [PLAIN_SHA_256] = 32,
129 [PLAIN_SHA_384] = 48,
130 [PLAIN_SHA_512] = 64,
131 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
132 [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
133 [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
134 [IMB_AUTH_KASUMI_UIA1] = 4
135 #endif
136 /**< Vector mode dependent pointer table of the multi-buffer APIs */
137
138 };
139
140 /**
141 * Get the full digest size in bytes for a specified authentication algorithm
142 * (if available in the Multi-buffer library)
143 *
144 * @Note: this function will not return a valid value for a non-valid
145 * authentication algorithm
146 */
147 static inline unsigned
get_digest_byte_length(JOB_HASH_ALG algo)148 get_digest_byte_length(JOB_HASH_ALG algo)
149 {
150 return auth_digest_byte_lengths[algo];
151 }
152
153 enum aesni_mb_operation {
154 AESNI_MB_OP_HASH_CIPHER,
155 AESNI_MB_OP_CIPHER_HASH,
156 AESNI_MB_OP_HASH_ONLY,
157 AESNI_MB_OP_CIPHER_ONLY,
158 AESNI_MB_OP_AEAD_HASH_CIPHER,
159 AESNI_MB_OP_AEAD_CIPHER_HASH,
160 AESNI_MB_OP_NOT_SUPPORTED
161 };
162
163 /** private data structure for each virtual AESNI device */
164 struct aesni_mb_private {
165 enum aesni_mb_vector_mode vector_mode;
166 /**< CPU vector instruction set mode */
167 unsigned max_nb_queue_pairs;
168 /**< Max number of queue pairs supported by device */
169 MB_MGR *mb_mgr;
170 /**< Multi-buffer instance */
171 };
172
173 /** AESNI Multi buffer queue pair */
174 struct aesni_mb_qp {
175 uint16_t id;
176 /**< Queue Pair Identifier */
177 char name[RTE_CRYPTODEV_NAME_MAX_LEN];
178 /**< Unique Queue Pair Name */
179 MB_MGR *mb_mgr;
180 /**< Multi-buffer instance */
181 struct rte_ring *ingress_queue;
182 /**< Ring for placing operations ready for processing */
183 struct rte_mempool *sess_mp;
184 /**< Session Mempool */
185 struct rte_mempool *sess_mp_priv;
186 /**< Session Private Data Mempool */
187 struct rte_cryptodev_stats stats;
188 /**< Queue pair statistics */
189 uint8_t digest_idx;
190 /**< Index of the next slot to be used in temp_digests,
191 * to store the digest for a given operation
192 */
193 uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];
194 /**< Buffers used to store the digest generated
195 * by the driver when verifying a digest provided
196 * by the user (using authentication verify operation)
197 */
198 } __rte_cache_aligned;
199
200 /** AES-NI multi-buffer private session structure */
201 struct aesni_mb_session {
202 JOB_CHAIN_ORDER chain_order;
203 struct {
204 uint16_t length;
205 uint16_t offset;
206 } iv;
207 struct {
208 uint16_t length;
209 uint16_t offset;
210 } auth_iv;
211 /**< IV parameters */
212
213 /** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;
214 /**< Vector mode dependent pointer table of the multi-buffer APIs */
215
216 struct {
217 /** Cipher direction - encrypt / decrypt */
218 JOB_CIPHER_DIRECTION direction;
219 /** Cipher mode - CBC / Counter */
220 JOB_CIPHER_MODE mode;
221
222 uint64_t key_length_in_bytes;
223
224 union {
225 struct {
226 uint32_t encode[60] __rte_aligned(16);
227 /**< encode key */
228 uint32_t decode[60] __rte_aligned(16);
229 /**< decode key */
230 } expanded_aes_keys;
231 /**< Expanded AES keys - Allocating space to
232 * contain the maximum expanded key size which
233 * is 240 bytes for 256 bit AES, calculate by:
234 * ((key size (bytes)) *
235 * ((number of rounds) + 1))
236 */
237 struct {
238 const void *ks_ptr[3];
239 uint64_t key[3][16];
240 } exp_3des_keys;
241 /**< Expanded 3DES keys */
242
243 struct gcm_key_data gcm_key;
244 /**< Expanded GCM key */
245 uint8_t zuc_cipher_key[16];
246 /**< ZUC cipher key */
247 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
248 snow3g_key_schedule_t pKeySched_snow3g_cipher;
249 /**< SNOW3G scheduled cipher key */
250 kasumi_key_sched_t pKeySched_kasumi_cipher;
251 /**< KASUMI scheduled cipher key */
252 #endif
253 };
254 } cipher;
255
256 /** Authentication Parameters */
257 struct {
258 JOB_HASH_ALG algo; /**< Authentication Algorithm */
259 enum rte_crypto_auth_operation operation;
260 /**< auth operation generate or verify */
261 union {
262 struct {
263 uint8_t inner[128] __rte_aligned(16);
264 /**< inner pad */
265 uint8_t outer[128] __rte_aligned(16);
266 /**< outer pad */
267 } pads;
268 /**< HMAC Authentication pads -
269 * allocating space for the maximum pad
270 * size supported which is 128 bytes for
271 * SHA512
272 */
273
274 struct {
275 uint32_t k1_expanded[44] __rte_aligned(16);
276 /**< k1 (expanded key). */
277 uint8_t k2[16] __rte_aligned(16);
278 /**< k2. */
279 uint8_t k3[16] __rte_aligned(16);
280 /**< k3. */
281 } xcbc;
282
283 struct {
284 uint32_t expkey[60] __rte_aligned(16);
285 /**< k1 (expanded key). */
286 uint32_t skey1[4] __rte_aligned(16);
287 /**< k2. */
288 uint32_t skey2[4] __rte_aligned(16);
289 /**< k3. */
290 } cmac;
291 /**< Expanded XCBC authentication keys */
292 uint8_t zuc_auth_key[16];
293 /**< ZUC authentication key */
294 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
295 snow3g_key_schedule_t pKeySched_snow3g_auth;
296 /**< SNOW3G scheduled authentication key */
297 kasumi_key_sched_t pKeySched_kasumi_auth;
298 /**< KASUMI scheduled authentication key */
299 #endif
300 };
301 /** Generated digest size by the Multi-buffer library */
302 uint16_t gen_digest_len;
303 /** Requested digest size from Cryptodev */
304 uint16_t req_digest_len;
305
306 } auth;
307 struct {
308 /** AAD data length */
309 uint16_t aad_len;
310 } aead;
311 } __rte_cache_aligned;
312
313 extern int
314 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
315 struct aesni_mb_session *sess,
316 const struct rte_crypto_sym_xform *xform);
317
318 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
319 extern int
320 aesni_mb_set_docsis_sec_session_parameters(
321 __rte_unused struct rte_cryptodev *dev,
322 struct rte_security_session_conf *conf,
323 void *sess);
324 #endif
325
326 /** device specific operations function pointer structures */
327 extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;
328 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
329 extern struct rte_security_ops *rte_aesni_mb_pmd_sec_ops;
330 #endif
331
332 extern uint32_t
333 aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
334 struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
335 struct rte_crypto_sym_vec *vec);
336
337 #endif /* _AESNI_MB_PMD_PRIVATE_H_ */
338