1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
3 */
4
5 #ifndef _CCP_DEV_H_
6 #define _CCP_DEV_H_
7
8 #include <limits.h>
9 #include <stdbool.h>
10 #include <stdint.h>
11 #include <string.h>
12
13 #include <rte_bus_pci.h>
14 #include <rte_atomic.h>
15 #include <rte_byteorder.h>
16 #include <rte_io.h>
17 #include <rte_pci.h>
18 #include <rte_spinlock.h>
19 #include <rte_crypto_sym.h>
20 #include <cryptodev_pmd.h>
21
22 /**< CCP specific */
23 #define MAX_HW_QUEUES 5
24 #define CCP_MAX_TRNG_RETRIES 10
25 #define CCP_ALIGN(x, y) ((((x) + (y - 1)) / y) * y)
26
27 /**< CCP Register Mappings */
28 #define Q_MASK_REG 0x000
29 #define TRNG_OUT_REG 0x00c
30
31 /* CCP Version 5 Specifics */
32 #define CMD_QUEUE_MASK_OFFSET 0x00
33 #define CMD_QUEUE_PRIO_OFFSET 0x04
34 #define CMD_REQID_CONFIG_OFFSET 0x08
35 #define CMD_CMD_TIMEOUT_OFFSET 0x10
36 #define LSB_PUBLIC_MASK_LO_OFFSET 0x18
37 #define LSB_PUBLIC_MASK_HI_OFFSET 0x1C
38 #define LSB_PRIVATE_MASK_LO_OFFSET 0x20
39 #define LSB_PRIVATE_MASK_HI_OFFSET 0x24
40
41 #define CMD_Q_CONTROL_BASE 0x0000
42 #define CMD_Q_TAIL_LO_BASE 0x0004
43 #define CMD_Q_HEAD_LO_BASE 0x0008
44 #define CMD_Q_INT_ENABLE_BASE 0x000C
45 #define CMD_Q_INTERRUPT_STATUS_BASE 0x0010
46
47 #define CMD_Q_STATUS_BASE 0x0100
48 #define CMD_Q_INT_STATUS_BASE 0x0104
49
50 #define CMD_CONFIG_0_OFFSET 0x6000
51 #define CMD_TRNG_CTL_OFFSET 0x6008
52 #define CMD_AES_MASK_OFFSET 0x6010
53 #define CMD_CLK_GATE_CTL_OFFSET 0x603C
54
55 /* Address offset between two virtual queue registers */
56 #define CMD_Q_STATUS_INCR 0x1000
57
58 /* Bit masks */
59 #define CMD_Q_RUN 0x1
60 #define CMD_Q_SIZE 0x1F
61 #define CMD_Q_SHIFT 3
62 #define COMMANDS_PER_QUEUE 8192
63
64 #define QUEUE_SIZE_VAL ((ffs(COMMANDS_PER_QUEUE) - 2) & \
65 CMD_Q_SIZE)
66 #define Q_DESC_SIZE sizeof(struct ccp_desc)
67 #define Q_SIZE(n) (COMMANDS_PER_QUEUE*(n))
68
69 #define INT_COMPLETION 0x1
70 #define INT_ERROR 0x2
71 #define INT_QUEUE_STOPPED 0x4
72 #define ALL_INTERRUPTS (INT_COMPLETION| \
73 INT_ERROR| \
74 INT_QUEUE_STOPPED)
75
76 #define LSB_REGION_WIDTH 5
77 #define MAX_LSB_CNT 8
78
79 #define LSB_SIZE 16
80 #define LSB_ITEM_SIZE 32
81 #define SLSB_MAP_SIZE (MAX_LSB_CNT * LSB_SIZE)
82 #define LSB_ENTRY_NUMBER(LSB_ADDR) (LSB_ADDR / LSB_ITEM_SIZE)
83
84 /* General CCP Defines */
85
86 #define CCP_SB_BYTES 32
87 /* Word 0 */
88 #define CCP_CMD_DW0(p) ((p)->dw0)
89 #define CCP_CMD_SOC(p) (CCP_CMD_DW0(p).soc)
90 #define CCP_CMD_IOC(p) (CCP_CMD_DW0(p).ioc)
91 #define CCP_CMD_INIT(p) (CCP_CMD_DW0(p).init)
92 #define CCP_CMD_EOM(p) (CCP_CMD_DW0(p).eom)
93 #define CCP_CMD_FUNCTION(p) (CCP_CMD_DW0(p).function)
94 #define CCP_CMD_ENGINE(p) (CCP_CMD_DW0(p).engine)
95 #define CCP_CMD_PROT(p) (CCP_CMD_DW0(p).prot)
96
97 /* Word 1 */
98 #define CCP_CMD_DW1(p) ((p)->length)
99 #define CCP_CMD_LEN(p) (CCP_CMD_DW1(p))
100
101 /* Word 2 */
102 #define CCP_CMD_DW2(p) ((p)->src_lo)
103 #define CCP_CMD_SRC_LO(p) (CCP_CMD_DW2(p))
104
105 /* Word 3 */
106 #define CCP_CMD_DW3(p) ((p)->dw3)
107 #define CCP_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
108 #define CCP_CMD_SRC_HI(p) ((p)->dw3.src_hi)
109 #define CCP_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
110 #define CCP_CMD_FIX_SRC(p) ((p)->dw3.fixed)
111
112 /* Words 4/5 */
113 #define CCP_CMD_DW4(p) ((p)->dw4)
114 #define CCP_CMD_DST_LO(p) (CCP_CMD_DW4(p).dst_lo)
115 #define CCP_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
116 #define CCP_CMD_DST_HI(p) (CCP_CMD_DW5(p))
117 #define CCP_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
118 #define CCP_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
119 #define CCP_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
120 #define CCP_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
121
122 /* Word 6/7 */
123 #define CCP_CMD_DW6(p) ((p)->key_lo)
124 #define CCP_CMD_KEY_LO(p) (CCP_CMD_DW6(p))
125 #define CCP_CMD_DW7(p) ((p)->dw7)
126 #define CCP_CMD_KEY_HI(p) ((p)->dw7.key_hi)
127 #define CCP_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
128
129 /* bitmap */
130 enum {
131 BITS_PER_WORD = sizeof(unsigned long) * CHAR_BIT
132 };
133
134 #define WORD_OFFSET(b) ((b) / BITS_PER_WORD)
135 #define BIT_OFFSET(b) ((b) % BITS_PER_WORD)
136
137 #define CCP_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
138 #define CCP_BITMAP_SIZE(nr) \
139 CCP_DIV_ROUND_UP(nr, CHAR_BIT * sizeof(unsigned long))
140
141 #define CCP_BITMAP_FIRST_WORD_MASK(start) \
142 (~0UL << ((start) & (BITS_PER_WORD - 1)))
143 #define CCP_BITMAP_LAST_WORD_MASK(nbits) \
144 (~0UL >> (-(nbits) & (BITS_PER_WORD - 1)))
145
146 #define __ccp_round_mask(x, y) ((typeof(x))((y)-1))
147 #define ccp_round_down(x, y) ((x) & ~__ccp_round_mask(x, y))
148
149 /** CCP registers Write/Read */
150
ccp_pci_reg_write(void * base,int offset,uint32_t value)151 static inline void ccp_pci_reg_write(void *base, int offset,
152 uint32_t value)
153 {
154 volatile void *reg_addr = ((uint8_t *)base + offset);
155
156 rte_write32((rte_cpu_to_le_32(value)), reg_addr);
157 }
158
ccp_pci_reg_read(void * base,int offset)159 static inline uint32_t ccp_pci_reg_read(void *base, int offset)
160 {
161 volatile void *reg_addr = ((uint8_t *)base + offset);
162
163 return rte_le_to_cpu_32(rte_read32(reg_addr));
164 }
165
166 #define CCP_READ_REG(hw_addr, reg_offset) \
167 ccp_pci_reg_read(hw_addr, reg_offset)
168
169 #define CCP_WRITE_REG(hw_addr, reg_offset, value) \
170 ccp_pci_reg_write(hw_addr, reg_offset, value)
171
172 TAILQ_HEAD(ccp_list, ccp_device);
173
174 extern struct ccp_list ccp_list;
175
176 /**
177 * CCP device version
178 */
179 enum ccp_device_version {
180 CCP_VERSION_5A = 0,
181 CCP_VERSION_5B,
182 };
183
184 /**
185 * A structure describing a CCP command queue.
186 */
187 struct ccp_queue {
188 struct ccp_device *dev;
189 char memz_name[RTE_MEMZONE_NAMESIZE];
190
191 rte_atomic64_t free_slots;
192 /**< available free slots updated from enq/deq calls */
193
194 /* Queue identifier */
195 uint64_t id; /**< queue id */
196 uint64_t qidx; /**< queue index */
197 uint64_t qsize; /**< queue size */
198
199 /* Queue address */
200 struct ccp_desc *qbase_desc;
201 void *qbase_addr;
202 phys_addr_t qbase_phys_addr;
203 /**< queue-page registers addr */
204 void *reg_base;
205
206 uint32_t qcontrol;
207 /**< queue ctrl reg */
208
209 int lsb;
210 /**< lsb region assigned to queue */
211 unsigned long lsbmask;
212 /**< lsb regions queue can access */
213 unsigned long lsbmap[CCP_BITMAP_SIZE(LSB_SIZE)];
214 /**< all lsb resources which queue is using */
215 uint32_t sb_key;
216 /**< lsb assigned for queue */
217 uint32_t sb_iv;
218 /**< lsb assigned for iv */
219 uint32_t sb_sha;
220 /**< lsb assigned for sha ctx */
221 uint32_t sb_hmac;
222 /**< lsb assigned for hmac ctx */
223 } __rte_cache_aligned;
224
225 /**
226 * A structure describing a CCP device.
227 */
228 struct ccp_device {
229 TAILQ_ENTRY(ccp_device) next;
230 int id;
231 /**< ccp dev id on platform */
232 struct ccp_queue cmd_q[MAX_HW_QUEUES];
233 /**< ccp queue */
234 int cmd_q_count;
235 /**< no. of ccp Queues */
236 struct rte_pci_device pci;
237 /**< ccp pci identifier */
238 unsigned long lsbmap[CCP_BITMAP_SIZE(SLSB_MAP_SIZE)];
239 /**< shared lsb mask of ccp */
240 rte_spinlock_t lsb_lock;
241 /**< protection for shared lsb region allocation */
242 int qidx;
243 /**< current queue index */
244 int hwrng_retries;
245 /**< retry counter for CCP TRNG */
246 } __rte_cache_aligned;
247
248 /**< CCP H/W engine related */
249 /**
250 * ccp_engine - CCP operation identifiers
251 *
252 * @CCP_ENGINE_AES: AES operation
253 * @CCP_ENGINE_XTS_AES: 128-bit XTS AES operation
254 * @CCP_ENGINE_3DES: DES/3DES operation
255 * @CCP_ENGINE_SHA: SHA operation
256 * @CCP_ENGINE_RSA: RSA operation
257 * @CCP_ENGINE_PASSTHRU: pass-through operation
258 * @CCP_ENGINE_ZLIB_DECOMPRESS: unused
259 * @CCP_ENGINE_ECC: ECC operation
260 */
261 enum ccp_engine {
262 CCP_ENGINE_AES = 0,
263 CCP_ENGINE_XTS_AES_128,
264 CCP_ENGINE_3DES,
265 CCP_ENGINE_SHA,
266 CCP_ENGINE_RSA,
267 CCP_ENGINE_PASSTHRU,
268 CCP_ENGINE_ZLIB_DECOMPRESS,
269 CCP_ENGINE_ECC,
270 CCP_ENGINE__LAST,
271 };
272
273 /* Passthru engine */
274 /**
275 * ccp_passthru_bitwise - type of bitwise passthru operation
276 *
277 * @CCP_PASSTHRU_BITWISE_NOOP: no bitwise operation performed
278 * @CCP_PASSTHRU_BITWISE_AND: perform bitwise AND of src with mask
279 * @CCP_PASSTHRU_BITWISE_OR: perform bitwise OR of src with mask
280 * @CCP_PASSTHRU_BITWISE_XOR: perform bitwise XOR of src with mask
281 * @CCP_PASSTHRU_BITWISE_MASK: overwrite with mask
282 */
283 enum ccp_passthru_bitwise {
284 CCP_PASSTHRU_BITWISE_NOOP = 0,
285 CCP_PASSTHRU_BITWISE_AND,
286 CCP_PASSTHRU_BITWISE_OR,
287 CCP_PASSTHRU_BITWISE_XOR,
288 CCP_PASSTHRU_BITWISE_MASK,
289 CCP_PASSTHRU_BITWISE__LAST,
290 };
291
292 /**
293 * ccp_passthru_byteswap - type of byteswap passthru operation
294 *
295 * @CCP_PASSTHRU_BYTESWAP_NOOP: no byte swapping performed
296 * @CCP_PASSTHRU_BYTESWAP_32BIT: swap bytes within 32-bit words
297 * @CCP_PASSTHRU_BYTESWAP_256BIT: swap bytes within 256-bit words
298 */
299 enum ccp_passthru_byteswap {
300 CCP_PASSTHRU_BYTESWAP_NOOP = 0,
301 CCP_PASSTHRU_BYTESWAP_32BIT,
302 CCP_PASSTHRU_BYTESWAP_256BIT,
303 CCP_PASSTHRU_BYTESWAP__LAST,
304 };
305
306 /**
307 * CCP passthru
308 */
309 struct ccp_passthru {
310 phys_addr_t src_addr;
311 phys_addr_t dest_addr;
312 enum ccp_passthru_bitwise bit_mod;
313 enum ccp_passthru_byteswap byte_swap;
314 int len;
315 int dir;
316 };
317
318 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
319 union ccp_function {
320 struct {
321 uint16_t size:7;
322 uint16_t encrypt:1;
323 uint16_t mode:5;
324 uint16_t type:2;
325 } aes;
326 struct {
327 uint16_t size:7;
328 uint16_t encrypt:1;
329 uint16_t mode:5;
330 uint16_t type:2;
331 } des;
332 struct {
333 uint16_t size:7;
334 uint16_t encrypt:1;
335 uint16_t rsvd:5;
336 uint16_t type:2;
337 } aes_xts;
338 struct {
339 uint16_t rsvd1:10;
340 uint16_t type:4;
341 uint16_t rsvd2:1;
342 } sha;
343 struct {
344 uint16_t mode:3;
345 uint16_t size:12;
346 } rsa;
347 struct {
348 uint16_t byteswap:2;
349 uint16_t bitwise:3;
350 uint16_t reflect:2;
351 uint16_t rsvd:8;
352 } pt;
353 struct {
354 uint16_t rsvd:13;
355 } zlib;
356 struct {
357 uint16_t size:10;
358 uint16_t type:2;
359 uint16_t mode:3;
360 } ecc;
361 uint16_t raw;
362 };
363
364
365 /**
366 * descriptor for version 5 CPP commands
367 * 8 32-bit words:
368 * word 0: function; engine; control bits
369 * word 1: length of source data
370 * word 2: low 32 bits of source pointer
371 * word 3: upper 16 bits of source pointer; source memory type
372 * word 4: low 32 bits of destination pointer
373 * word 5: upper 16 bits of destination pointer; destination memory
374 * type
375 * word 6: low 32 bits of key pointer
376 * word 7: upper 16 bits of key pointer; key memory type
377 */
378 struct dword0 {
379 uint32_t soc:1;
380 uint32_t ioc:1;
381 uint32_t rsvd1:1;
382 uint32_t init:1;
383 uint32_t eom:1;
384 uint32_t function:15;
385 uint32_t engine:4;
386 uint32_t prot:1;
387 uint32_t rsvd2:7;
388 };
389
390 struct dword3 {
391 uint32_t src_hi:16;
392 uint32_t src_mem:2;
393 uint32_t lsb_cxt_id:8;
394 uint32_t rsvd1:5;
395 uint32_t fixed:1;
396 };
397
398 union dword4 {
399 uint32_t dst_lo; /* NON-SHA */
400 uint32_t sha_len_lo; /* SHA */
401 };
402
403 union dword5 {
404 struct {
405 uint32_t dst_hi:16;
406 uint32_t dst_mem:2;
407 uint32_t rsvd1:13;
408 uint32_t fixed:1;
409 }
410 fields;
411 uint32_t sha_len_hi;
412 };
413
414 struct dword7 {
415 uint32_t key_hi:16;
416 uint32_t key_mem:2;
417 uint32_t rsvd1:14;
418 };
419
420 struct ccp_desc {
421 struct dword0 dw0;
422 uint32_t length;
423 uint32_t src_lo;
424 struct dword3 dw3;
425 union dword4 dw4;
426 union dword5 dw5;
427 uint32_t key_lo;
428 struct dword7 dw7;
429 };
430
431 /**
432 * ccp memory type
433 */
434 enum ccp_memtype {
435 CCP_MEMTYPE_SYSTEM = 0,
436 CCP_MEMTYPE_SB,
437 CCP_MEMTYPE_LOCAL,
438 CCP_MEMTYPE_LAST,
439 };
440
441 /**
442 * cmd id to follow order
443 */
444 enum ccp_cmd_order {
445 CCP_CMD_CIPHER = 0,
446 CCP_CMD_AUTH,
447 CCP_CMD_CIPHER_HASH,
448 CCP_CMD_HASH_CIPHER,
449 CCP_CMD_COMBINED,
450 CCP_CMD_NOT_SUPPORTED,
451 };
452
453 static inline uint32_t
low32_value(unsigned long addr)454 low32_value(unsigned long addr)
455 {
456 return ((uint64_t)addr) & 0x0ffffffff;
457 }
458
459 static inline uint32_t
high32_value(unsigned long addr)460 high32_value(unsigned long addr)
461 {
462 return ((uint64_t)addr >> 32) & 0x00000ffff;
463 }
464
465 /*
466 * Start CCP device
467 */
468 int ccp_dev_start(struct rte_cryptodev *dev);
469
470 /**
471 * Detect ccp platform and initialize all ccp devices
472 *
473 * @param ccp_id rte_pci_id list for supported CCP devices
474 * @return no. of successfully initialized CCP devices
475 */
476 int ccp_probe_devices(struct rte_pci_device *pci_dev,
477 const struct rte_pci_id *ccp_id);
478
479 /**
480 * allocate a ccp command queue
481 *
482 * @dev rte crypto device
483 * @param slot_req number of required
484 * @return allotted CCP queue on success otherwise NULL
485 */
486 struct ccp_queue *ccp_allot_queue(struct rte_cryptodev *dev, int slot_req);
487
488 /**
489 * read hwrng value
490 *
491 * @param trng_value data pointer to write RNG value
492 * @return 0 on success otherwise -1
493 */
494 int ccp_read_hwrng(uint32_t *trng_value);
495
496 #endif /* _CCP_DEV_H_ */
497