1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
3 */
4
5 #ifndef _OTX2_IO_ARM64_H_
6 #define _OTX2_IO_ARM64_H_
7
8 #define otx2_load_pair(val0, val1, addr) ({ \
9 asm volatile( \
10 "ldp %x[x0], %x[x1], [%x[p1]]" \
11 :[x0]"=r"(val0), [x1]"=r"(val1) \
12 :[p1]"r"(addr) \
13 ); })
14
15 #define otx2_store_pair(val0, val1, addr) ({ \
16 asm volatile( \
17 "stp %x[x0], %x[x1], [%x[p1],#0]!" \
18 ::[x0]"r"(val0), [x1]"r"(val1), [p1]"r"(addr) \
19 ); })
20
21 #define otx2_prefetch_store_keep(ptr) ({\
22 asm volatile("prfm pstl1keep, [%x0]\n" : : "r" (ptr)); })
23
24 static __rte_always_inline uint64_t
otx2_atomic64_add_nosync(int64_t incr,int64_t * ptr)25 otx2_atomic64_add_nosync(int64_t incr, int64_t *ptr)
26 {
27 uint64_t result;
28
29 /* Atomic add with no ordering */
30 asm volatile (
31 ".cpu generic+lse\n"
32 "ldadd %x[i], %x[r], [%[b]]"
33 : [r] "=r" (result), "+m" (*ptr)
34 : [i] "r" (incr), [b] "r" (ptr)
35 : "memory");
36 return result;
37 }
38
39 static __rte_always_inline uint64_t
otx2_atomic64_add_sync(int64_t incr,int64_t * ptr)40 otx2_atomic64_add_sync(int64_t incr, int64_t *ptr)
41 {
42 uint64_t result;
43
44 /* Atomic add with ordering */
45 asm volatile (
46 ".cpu generic+lse\n"
47 "ldadda %x[i], %x[r], [%[b]]"
48 : [r] "=r" (result), "+m" (*ptr)
49 : [i] "r" (incr), [b] "r" (ptr)
50 : "memory");
51 return result;
52 }
53
54 static __rte_always_inline uint64_t
otx2_lmt_submit(rte_iova_t io_address)55 otx2_lmt_submit(rte_iova_t io_address)
56 {
57 uint64_t result;
58
59 asm volatile (
60 ".cpu generic+lse\n"
61 "ldeor xzr,%x[rf],[%[rs]]" :
62 [rf] "=r"(result): [rs] "r"(io_address));
63 return result;
64 }
65
66 static __rte_always_inline uint64_t
otx2_lmt_submit_release(rte_iova_t io_address)67 otx2_lmt_submit_release(rte_iova_t io_address)
68 {
69 uint64_t result;
70
71 asm volatile (
72 ".cpu generic+lse\n"
73 "ldeorl xzr,%x[rf],[%[rs]]" :
74 [rf] "=r"(result) : [rs] "r"(io_address));
75 return result;
76 }
77
78 static __rte_always_inline void
otx2_lmt_mov(void * out,const void * in,const uint32_t lmtext)79 otx2_lmt_mov(void *out, const void *in, const uint32_t lmtext)
80 {
81 volatile const __uint128_t *src128 = (const __uint128_t *)in;
82 volatile __uint128_t *dst128 = (__uint128_t *)out;
83 dst128[0] = src128[0];
84 dst128[1] = src128[1];
85 /* lmtext receives following value:
86 * 1: NIX_SUBDC_EXT needed i.e. tx vlan case
87 * 2: NIX_SUBDC_EXT + NIX_SUBDC_MEM i.e. tstamp case
88 */
89 if (lmtext) {
90 dst128[2] = src128[2];
91 if (lmtext > 1)
92 dst128[3] = src128[3];
93 }
94 }
95
96 static __rte_always_inline void
otx2_lmt_mov_seg(void * out,const void * in,const uint16_t segdw)97 otx2_lmt_mov_seg(void *out, const void *in, const uint16_t segdw)
98 {
99 volatile const __uint128_t *src128 = (const __uint128_t *)in;
100 volatile __uint128_t *dst128 = (__uint128_t *)out;
101 uint8_t i;
102
103 for (i = 0; i < segdw; i++)
104 dst128[i] = src128[i];
105 }
106
107 #endif /* _OTX2_IO_ARM64_H_ */
108