1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
3 */
4
5 #include <rte_dev.h>
6 #include <rte_pci.h>
7
8 #include "qat_device.h"
9 #include "qat_qp.h"
10 #include "adf_transport_access_macros_gen4vf.h"
11 #include "adf_pf2vf_msg.h"
12 #include "qat_pf2vf.h"
13
14 #include <stdint.h>
15
16 /* QAT GEN 4 specific macros */
17 #define QAT_GEN4_BUNDLE_NUM 4
18 #define QAT_GEN4_QPS_PER_BUNDLE_NUM 1
19
20 struct qat_dev_gen4_extra {
21 struct qat_qp_hw_data qp_gen4_data[QAT_GEN4_BUNDLE_NUM]
22 [QAT_GEN4_QPS_PER_BUNDLE_NUM];
23 };
24
25 static struct qat_pf2vf_dev qat_pf2vf_gen4 = {
26 .pf2vf_offset = ADF_4XXXIOV_PF2VM_OFFSET,
27 .vf2pf_offset = ADF_4XXXIOV_VM2PF_OFFSET,
28 .pf2vf_type_shift = ADF_PFVF_2X_MSGTYPE_SHIFT,
29 .pf2vf_type_mask = ADF_PFVF_2X_MSGTYPE_MASK,
30 .pf2vf_data_shift = ADF_PFVF_2X_MSGDATA_SHIFT,
31 .pf2vf_data_mask = ADF_PFVF_2X_MSGDATA_MASK,
32 };
33
34 static int
qat_query_svc_gen4(struct qat_pci_device * qat_dev,uint8_t * val)35 qat_query_svc_gen4(struct qat_pci_device *qat_dev, uint8_t *val)
36 {
37 struct qat_pf2vf_msg pf2vf_msg;
38
39 pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_GET_SMALL_BLOCK_REQ;
40 pf2vf_msg.block_hdr = ADF_VF2PF_BLOCK_MSG_GET_RING_TO_SVC_REQ;
41 pf2vf_msg.msg_data = 2;
42 return qat_pf2vf_exch_msg(qat_dev, pf2vf_msg, 2, val);
43 }
44
45 static int
qat_select_valid_queue_gen4(struct qat_pci_device * qat_dev,int qp_id,enum qat_service_type service_type)46 qat_select_valid_queue_gen4(struct qat_pci_device *qat_dev, int qp_id,
47 enum qat_service_type service_type)
48 {
49 int i = 0, valid_qps = 0;
50 struct qat_dev_gen4_extra *dev_extra = qat_dev->dev_private;
51
52 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
53 if (dev_extra->qp_gen4_data[i][0].service_type ==
54 service_type) {
55 if (valid_qps == qp_id)
56 return i;
57 ++valid_qps;
58 }
59 }
60 return -1;
61 }
62
63 static const struct qat_qp_hw_data *
qat_qp_get_hw_data_gen4(struct qat_pci_device * qat_dev,enum qat_service_type service_type,uint16_t qp_id)64 qat_qp_get_hw_data_gen4(struct qat_pci_device *qat_dev,
65 enum qat_service_type service_type, uint16_t qp_id)
66 {
67 struct qat_dev_gen4_extra *dev_extra = qat_dev->dev_private;
68 int ring_pair = qat_select_valid_queue_gen4(qat_dev, qp_id,
69 service_type);
70
71 if (ring_pair < 0)
72 return NULL;
73
74 return &dev_extra->qp_gen4_data[ring_pair][0];
75 }
76
77 static int
qat_qp_rings_per_service_gen4(struct qat_pci_device * qat_dev,enum qat_service_type service)78 qat_qp_rings_per_service_gen4(struct qat_pci_device *qat_dev,
79 enum qat_service_type service)
80 {
81 int i = 0, count = 0, max_ops_per_srv = 0;
82 struct qat_dev_gen4_extra *dev_extra = qat_dev->dev_private;
83
84 max_ops_per_srv = QAT_GEN4_BUNDLE_NUM;
85 for (i = 0, count = 0; i < max_ops_per_srv; i++)
86 if (dev_extra->qp_gen4_data[i][0].service_type == service)
87 count++;
88 return count;
89 }
90
91 static enum qat_service_type
gen4_pick_service(uint8_t hw_service)92 gen4_pick_service(uint8_t hw_service)
93 {
94 switch (hw_service) {
95 case QAT_SVC_SYM:
96 return QAT_SERVICE_SYMMETRIC;
97 case QAT_SVC_COMPRESSION:
98 return QAT_SERVICE_COMPRESSION;
99 case QAT_SVC_ASYM:
100 return QAT_SERVICE_ASYMMETRIC;
101 default:
102 return QAT_SERVICE_INVALID;
103 }
104 }
105
106 static int
qat_dev_read_config_gen4(struct qat_pci_device * qat_dev)107 qat_dev_read_config_gen4(struct qat_pci_device *qat_dev)
108 {
109 int i = 0;
110 uint16_t svc = 0;
111 struct qat_dev_gen4_extra *dev_extra = qat_dev->dev_private;
112 struct qat_qp_hw_data *hw_data;
113 enum qat_service_type service_type;
114 uint8_t hw_service;
115
116 if (qat_query_svc_gen4(qat_dev, (uint8_t *)&svc))
117 return -EFAULT;
118 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
119 hw_service = (svc >> (3 * i)) & 0x7;
120 service_type = gen4_pick_service(hw_service);
121 if (service_type == QAT_SERVICE_INVALID) {
122 QAT_LOG(ERR,
123 "Unrecognized service on bundle %d",
124 i);
125 return -ENOTSUP;
126 }
127 hw_data = &dev_extra->qp_gen4_data[i][0];
128 memset(hw_data, 0, sizeof(*hw_data));
129 hw_data->service_type = service_type;
130 if (service_type == QAT_SERVICE_ASYMMETRIC) {
131 hw_data->tx_msg_size = 64;
132 hw_data->rx_msg_size = 32;
133 } else if (service_type == QAT_SERVICE_SYMMETRIC ||
134 service_type ==
135 QAT_SERVICE_COMPRESSION) {
136 hw_data->tx_msg_size = 128;
137 hw_data->rx_msg_size = 32;
138 }
139 hw_data->tx_ring_num = 0;
140 hw_data->rx_ring_num = 1;
141 hw_data->hw_bundle_num = i;
142 }
143 return 0;
144 }
145
146 static void
qat_qp_build_ring_base_gen4(void * io_addr,struct qat_queue * queue)147 qat_qp_build_ring_base_gen4(void *io_addr,
148 struct qat_queue *queue)
149 {
150 uint64_t queue_base;
151
152 queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr,
153 queue->queue_size);
154 WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number,
155 queue->hw_queue_number, queue_base);
156 }
157
158 static void
qat_qp_adf_arb_enable_gen4(const struct qat_queue * txq,void * base_addr,rte_spinlock_t * lock)159 qat_qp_adf_arb_enable_gen4(const struct qat_queue *txq,
160 void *base_addr, rte_spinlock_t *lock)
161 {
162 uint32_t arb_csr_offset = 0, value;
163
164 rte_spinlock_lock(lock);
165 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
166 (ADF_RING_BUNDLE_SIZE_GEN4 *
167 txq->hw_bundle_number);
168 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
169 arb_csr_offset);
170 value |= (0x01 << txq->hw_queue_number);
171 ADF_CSR_WR(base_addr, arb_csr_offset, value);
172 rte_spinlock_unlock(lock);
173 }
174
175 static void
qat_qp_adf_arb_disable_gen4(const struct qat_queue * txq,void * base_addr,rte_spinlock_t * lock)176 qat_qp_adf_arb_disable_gen4(const struct qat_queue *txq,
177 void *base_addr, rte_spinlock_t *lock)
178 {
179 uint32_t arb_csr_offset = 0, value;
180
181 rte_spinlock_lock(lock);
182 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
183 (ADF_RING_BUNDLE_SIZE_GEN4 *
184 txq->hw_bundle_number);
185 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
186 arb_csr_offset);
187 value &= ~(0x01 << txq->hw_queue_number);
188 ADF_CSR_WR(base_addr, arb_csr_offset, value);
189 rte_spinlock_unlock(lock);
190 }
191
192 static void
qat_qp_adf_configure_queues_gen4(struct qat_qp * qp)193 qat_qp_adf_configure_queues_gen4(struct qat_qp *qp)
194 {
195 uint32_t q_tx_config, q_resp_config;
196 struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q;
197
198 q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size);
199 q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size,
200 ADF_RING_NEAR_WATERMARK_512,
201 ADF_RING_NEAR_WATERMARK_0);
202
203 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
204 q_tx->hw_bundle_number, q_tx->hw_queue_number,
205 q_tx_config);
206 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
207 q_rx->hw_bundle_number, q_rx->hw_queue_number,
208 q_resp_config);
209 }
210
211 static void
qat_qp_csr_write_tail_gen4(struct qat_qp * qp,struct qat_queue * q)212 qat_qp_csr_write_tail_gen4(struct qat_qp *qp, struct qat_queue *q)
213 {
214 WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
215 q->hw_bundle_number, q->hw_queue_number, q->tail);
216 }
217
218 static void
qat_qp_csr_write_head_gen4(struct qat_qp * qp,struct qat_queue * q,uint32_t new_head)219 qat_qp_csr_write_head_gen4(struct qat_qp *qp, struct qat_queue *q,
220 uint32_t new_head)
221 {
222 WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
223 q->hw_bundle_number, q->hw_queue_number, new_head);
224 }
225
226 static void
qat_qp_csr_setup_gen4(struct qat_pci_device * qat_dev,void * io_addr,struct qat_qp * qp)227 qat_qp_csr_setup_gen4(struct qat_pci_device *qat_dev,
228 void *io_addr, struct qat_qp *qp)
229 {
230 qat_qp_build_ring_base_gen4(io_addr, &qp->tx_q);
231 qat_qp_build_ring_base_gen4(io_addr, &qp->rx_q);
232 qat_qp_adf_configure_queues_gen4(qp);
233 qat_qp_adf_arb_enable_gen4(&qp->tx_q, qp->mmap_bar_addr,
234 &qat_dev->arb_csr_lock);
235 }
236
237 static struct qat_qp_hw_spec_funcs qat_qp_hw_spec_gen4 = {
238 .qat_qp_rings_per_service = qat_qp_rings_per_service_gen4,
239 .qat_qp_build_ring_base = qat_qp_build_ring_base_gen4,
240 .qat_qp_adf_arb_enable = qat_qp_adf_arb_enable_gen4,
241 .qat_qp_adf_arb_disable = qat_qp_adf_arb_disable_gen4,
242 .qat_qp_adf_configure_queues = qat_qp_adf_configure_queues_gen4,
243 .qat_qp_csr_write_tail = qat_qp_csr_write_tail_gen4,
244 .qat_qp_csr_write_head = qat_qp_csr_write_head_gen4,
245 .qat_qp_csr_setup = qat_qp_csr_setup_gen4,
246 .qat_qp_get_hw_data = qat_qp_get_hw_data_gen4,
247 };
248
249 static int
qat_reset_ring_pairs_gen4(struct qat_pci_device * qat_pci_dev)250 qat_reset_ring_pairs_gen4(struct qat_pci_device *qat_pci_dev)
251 {
252 int ret = 0, i;
253 uint8_t data[4];
254 struct qat_pf2vf_msg pf2vf_msg;
255
256 pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_RP_RESET;
257 pf2vf_msg.block_hdr = -1;
258 for (i = 0; i < QAT_GEN4_BUNDLE_NUM; i++) {
259 pf2vf_msg.msg_data = i;
260 ret = qat_pf2vf_exch_msg(qat_pci_dev, pf2vf_msg, 1, data);
261 if (ret) {
262 QAT_LOG(ERR, "QAT error when reset bundle no %d",
263 i);
264 return ret;
265 }
266 }
267
268 return 0;
269 }
270
271 static const struct rte_mem_resource *
qat_dev_get_transport_bar_gen4(struct rte_pci_device * pci_dev)272 qat_dev_get_transport_bar_gen4(struct rte_pci_device *pci_dev)
273 {
274 return &pci_dev->mem_resource[0];
275 }
276
277 static int
qat_dev_get_misc_bar_gen4(struct rte_mem_resource ** mem_resource,struct rte_pci_device * pci_dev)278 qat_dev_get_misc_bar_gen4(struct rte_mem_resource **mem_resource,
279 struct rte_pci_device *pci_dev)
280 {
281 *mem_resource = &pci_dev->mem_resource[2];
282 return 0;
283 }
284
285 static int
qat_dev_get_extra_size_gen4(void)286 qat_dev_get_extra_size_gen4(void)
287 {
288 return sizeof(struct qat_dev_gen4_extra);
289 }
290
291 static struct qat_dev_hw_spec_funcs qat_dev_hw_spec_gen4 = {
292 .qat_dev_reset_ring_pairs = qat_reset_ring_pairs_gen4,
293 .qat_dev_get_transport_bar = qat_dev_get_transport_bar_gen4,
294 .qat_dev_get_misc_bar = qat_dev_get_misc_bar_gen4,
295 .qat_dev_read_config = qat_dev_read_config_gen4,
296 .qat_dev_get_extra_size = qat_dev_get_extra_size_gen4,
297 };
298
RTE_INIT(qat_dev_gen_4_init)299 RTE_INIT(qat_dev_gen_4_init)
300 {
301 qat_qp_hw_spec[QAT_GEN4] = &qat_qp_hw_spec_gen4;
302 qat_dev_hw_spec[QAT_GEN4] = &qat_dev_hw_spec_gen4;
303 qat_gen_config[QAT_GEN4].dev_gen = QAT_GEN4;
304 qat_gen_config[QAT_GEN4].pf2vf_dev = &qat_pf2vf_gen4;
305 }
306