1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
3 */
4
5 #ifndef _ROC_IO_H_
6 #define _ROC_IO_H_
7
8 #define ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id) \
9 do { \
10 /* 32 Lines per core */ \
11 lmt_id = plt_lcore_id() << ROC_LMT_LINES_PER_CORE_LOG2; \
12 /* Each line is of 128B */ \
13 (lmt_addr) += ((uint64_t)lmt_id << ROC_LMT_LINE_SIZE_LOG2); \
14 } while (0)
15
16 #define ROC_LMT_CPT_BASE_ID_GET(lmt_addr, lmt_id) \
17 do { \
18 /* 16 Lines per core */ \
19 lmt_id = ROC_LMT_CPT_BASE_ID_OFF; \
20 lmt_id += (plt_lcore_id() << ROC_LMT_CPT_LINES_PER_CORE_LOG2); \
21 /* Each line is of 128B */ \
22 (lmt_addr) += ((uint64_t)lmt_id << ROC_LMT_LINE_SIZE_LOG2); \
23 } while (0)
24
25 #define roc_load_pair(val0, val1, addr) \
26 ({ \
27 asm volatile("ldp %x[x0], %x[x1], [%x[p1]]" \
28 : [x0] "=r"(val0), [x1] "=r"(val1) \
29 : [p1] "r"(addr)); \
30 })
31
32 #define roc_store_pair(val0, val1, addr) \
33 ({ \
34 asm volatile( \
35 "stp %x[x0], %x[x1], [%x[p1], #0]!" ::[x0] "r"(val0), \
36 [x1] "r"(val1), [p1] "r"(addr)); \
37 })
38
39 #define roc_prefetch_store_keep(ptr) \
40 ({ asm volatile("prfm pstl1keep, [%x0]\n" : : "r"(ptr)); })
41
42 #if defined(__clang__)
43 static __plt_always_inline void
roc_atomic128_cas_noreturn(uint64_t swap0,uint64_t swap1,int64_t * ptr)44 roc_atomic128_cas_noreturn(uint64_t swap0, uint64_t swap1, int64_t *ptr)
45 {
46 register uint64_t x0 __asm("x0") = swap0;
47 register uint64_t x1 __asm("x1") = swap1;
48
49 asm volatile(PLT_CPU_FEATURE_PREAMBLE
50 "casp %[x0], %[x1], %[x0], %[x1], [%[ptr]]\n"
51 : [x0] "+r"(x0), [x1] "+r"(x1)
52 : [ptr] "r"(ptr)
53 : "memory");
54 }
55 #else
56 static __plt_always_inline void
roc_atomic128_cas_noreturn(uint64_t swap0,uint64_t swap1,uint64_t ptr)57 roc_atomic128_cas_noreturn(uint64_t swap0, uint64_t swap1, uint64_t ptr)
58 {
59 __uint128_t wdata = swap0 | ((__uint128_t)swap1 << 64);
60
61 asm volatile(PLT_CPU_FEATURE_PREAMBLE
62 "casp %[wdata], %H[wdata], %[wdata], %H[wdata], [%[ptr]]\n"
63 : [wdata] "+r"(wdata)
64 : [ptr] "r"(ptr)
65 : "memory");
66 }
67 #endif
68
69 static __plt_always_inline uint64_t
roc_atomic64_cas(uint64_t compare,uint64_t swap,int64_t * ptr)70 roc_atomic64_cas(uint64_t compare, uint64_t swap, int64_t *ptr)
71 {
72 asm volatile(PLT_CPU_FEATURE_PREAMBLE
73 "cas %[compare], %[swap], [%[ptr]]\n"
74 : [compare] "+r"(compare)
75 : [swap] "r"(swap), [ptr] "r"(ptr)
76 : "memory");
77
78 return compare;
79 }
80
81 static __plt_always_inline uint64_t
roc_atomic64_casl(uint64_t compare,uint64_t swap,int64_t * ptr)82 roc_atomic64_casl(uint64_t compare, uint64_t swap, int64_t *ptr)
83 {
84 asm volatile(PLT_CPU_FEATURE_PREAMBLE
85 "casl %[compare], %[swap], [%[ptr]]\n"
86 : [compare] "+r"(compare)
87 : [swap] "r"(swap), [ptr] "r"(ptr)
88 : "memory");
89
90 return compare;
91 }
92
93 static __plt_always_inline uint64_t
roc_atomic64_add_nosync(int64_t incr,int64_t * ptr)94 roc_atomic64_add_nosync(int64_t incr, int64_t *ptr)
95 {
96 uint64_t result;
97
98 /* Atomic add with no ordering */
99 asm volatile(PLT_CPU_FEATURE_PREAMBLE "ldadd %x[i], %x[r], [%[b]]"
100 : [r] "=r"(result), "+m"(*ptr)
101 : [i] "r"(incr), [b] "r"(ptr)
102 : "memory");
103 return result;
104 }
105
106 static __plt_always_inline uint64_t
roc_atomic64_add_sync(int64_t incr,int64_t * ptr)107 roc_atomic64_add_sync(int64_t incr, int64_t *ptr)
108 {
109 uint64_t result;
110
111 /* Atomic add with ordering */
112 asm volatile(PLT_CPU_FEATURE_PREAMBLE "ldadda %x[i], %x[r], [%[b]]"
113 : [r] "=r"(result), "+m"(*ptr)
114 : [i] "r"(incr), [b] "r"(ptr)
115 : "memory");
116 return result;
117 }
118
119 static __plt_always_inline uint64_t
roc_lmt_submit_ldeor(plt_iova_t io_address)120 roc_lmt_submit_ldeor(plt_iova_t io_address)
121 {
122 uint64_t result;
123
124 asm volatile(PLT_CPU_FEATURE_PREAMBLE "ldeor xzr, %x[rf], [%[rs]]"
125 : [rf] "=r"(result)
126 : [rs] "r"(io_address));
127 return result;
128 }
129
130 static __plt_always_inline uint64_t
roc_lmt_submit_ldeorl(plt_iova_t io_address)131 roc_lmt_submit_ldeorl(plt_iova_t io_address)
132 {
133 uint64_t result;
134
135 asm volatile(PLT_CPU_FEATURE_PREAMBLE "ldeorl xzr,%x[rf],[%[rs]]"
136 : [rf] "=r"(result)
137 : [rs] "r"(io_address));
138 return result;
139 }
140
141 static __plt_always_inline void
roc_lmt_submit_steor(uint64_t data,plt_iova_t io_address)142 roc_lmt_submit_steor(uint64_t data, plt_iova_t io_address)
143 {
144 asm volatile(PLT_CPU_FEATURE_PREAMBLE
145 "steor %x[d], [%[rs]]" ::[d] "r"(data),
146 [rs] "r"(io_address));
147 }
148
149 static __plt_always_inline void
roc_lmt_submit_steorl(uint64_t data,plt_iova_t io_address)150 roc_lmt_submit_steorl(uint64_t data, plt_iova_t io_address)
151 {
152 asm volatile(PLT_CPU_FEATURE_PREAMBLE
153 "steorl %x[d], [%[rs]]" ::[d] "r"(data),
154 [rs] "r"(io_address));
155 }
156
157 static __plt_always_inline void
roc_lmt_mov(void * out,const void * in,const uint32_t lmtext)158 roc_lmt_mov(void *out, const void *in, const uint32_t lmtext)
159 {
160 volatile const __uint128_t *src128 = (const __uint128_t *)in;
161 volatile __uint128_t *dst128 = (__uint128_t *)out;
162
163 dst128[0] = src128[0];
164 dst128[1] = src128[1];
165 /* lmtext receives following value:
166 * 1: NIX_SUBDC_EXT needed i.e. tx vlan case
167 */
168 if (lmtext)
169 dst128[2] = src128[2];
170 }
171
172 static __plt_always_inline void
roc_lmt_mov64(void * out,const void * in)173 roc_lmt_mov64(void *out, const void *in)
174 {
175 volatile const __uint128_t *src128 = (const __uint128_t *)in;
176 volatile __uint128_t *dst128 = (__uint128_t *)out;
177
178 dst128[0] = src128[0];
179 dst128[1] = src128[1];
180 dst128[2] = src128[2];
181 dst128[3] = src128[3];
182 }
183
184 static __plt_always_inline void
roc_lmt_mov_nv(void * out,const void * in,const uint32_t lmtext)185 roc_lmt_mov_nv(void *out, const void *in, const uint32_t lmtext)
186 {
187 const __uint128_t *src128 = (const __uint128_t *)in;
188 __uint128_t *dst128 = (__uint128_t *)out;
189
190 dst128[0] = src128[0];
191 dst128[1] = src128[1];
192 /* lmtext receives following value:
193 * 1: NIX_SUBDC_EXT needed i.e. tx vlan case
194 */
195 if (lmtext)
196 dst128[2] = src128[2];
197 }
198
199 static __plt_always_inline void
roc_lmt_mov_seg(void * out,const void * in,const uint16_t segdw)200 roc_lmt_mov_seg(void *out, const void *in, const uint16_t segdw)
201 {
202 volatile const __uint128_t *src128 = (const __uint128_t *)in;
203 volatile __uint128_t *dst128 = (__uint128_t *)out;
204 uint8_t i;
205
206 for (i = 0; i < segdw; i++)
207 dst128[i] = src128[i];
208 }
209
210 static __plt_always_inline void
roc_lmt_mov_one(void * out,const void * in)211 roc_lmt_mov_one(void *out, const void *in)
212 {
213 volatile const __uint128_t *src128 = (const __uint128_t *)in;
214 volatile __uint128_t *dst128 = (__uint128_t *)out;
215
216 *dst128 = *src128;
217 }
218
219 /* Non volatile version of roc_lmt_mov_seg() */
220 static __plt_always_inline void
roc_lmt_mov_seg_nv(void * out,const void * in,const uint16_t segdw)221 roc_lmt_mov_seg_nv(void *out, const void *in, const uint16_t segdw)
222 {
223 const __uint128_t *src128 = (const __uint128_t *)in;
224 __uint128_t *dst128 = (__uint128_t *)out;
225 uint8_t i;
226
227 for (i = 0; i < segdw; i++)
228 dst128[i] = src128[i];
229 }
230
231 static __plt_always_inline void
roc_atf_ret(void)232 roc_atf_ret(void)
233 {
234 /* This will allow wfi in EL0 to cause async exception to EL3
235 * which will optionally perform necessary actions.
236 */
237 __asm("wfi");
238 }
239
240 #endif /* _ROC_IO_H_ */
241