1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2021 HiSilicon Limited 3 */ 4 5 #ifndef HISI_DMADEV_H 6 #define HISI_DMADEV_H 7 8 #include <rte_byteorder.h> 9 #include <rte_common.h> 10 11 #define BIT(x) (1ul << (x)) 12 #define BITS_PER_LONG (__SIZEOF_LONG__ * 8) 13 #define GENMASK(h, l) \ 14 (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) 15 #define BF_SHF(x) (__builtin_ffsll(x) - 1) 16 #define FIELD_GET(mask, reg) \ 17 ((typeof(mask))(((reg) & (mask)) >> BF_SHF(mask))) 18 19 #define lower_32_bits(x) ((uint32_t)(x)) 20 #define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) 21 22 #define PCI_VENDOR_ID_HUAWEI 0x19e5 23 #define HISI_DMA_DEVICE_ID 0xA122 24 #define HISI_DMA_PCI_REVISION_ID_REG 0x08 25 #define HISI_DMA_REVISION_HIP08B 0x21 26 #define HISI_DMA_REVISION_HIP09A 0x30 27 28 #define HISI_DMA_MAX_HW_QUEUES 4 29 #define HISI_DMA_MAX_DESC_NUM 8192 30 #define HISI_DMA_MIN_DESC_NUM 32 31 32 /** 33 * The HIP08B(HiSilicon IP08) and HIP09B(HiSilicon IP09) are DMA iEPs, they 34 * have the same pci device id but different pci revision. 35 * Unfortunately, they have different register layouts, so two layout 36 * enumerations are defined. 37 */ 38 enum { 39 HISI_DMA_REG_LAYOUT_INVALID = 0, 40 HISI_DMA_REG_LAYOUT_HIP08, 41 HISI_DMA_REG_LAYOUT_HIP09 42 }; 43 44 /** 45 * Hardware PCI bar register MAP: 46 * 47 * -------------- 48 * | Misc-reg-0 | 49 * | | 50 * -------------- -> Queue base 51 * | | 52 * | Queue-0 | 53 * | | 54 * -------------- --- 55 * | | ^ 56 * | Queue-1 | Queue region 57 * | | v 58 * -------------- --- 59 * | ... | 60 * | Queue-x | 61 * | ... | 62 * -------------- 63 * | Misc-reg-1 | 64 * -------------- 65 * 66 * As described above, a single queue register is continuous and occupies the 67 * length of queue-region. The global offset for a single queue register is 68 * calculated by: 69 * offset = queue-base + (queue-id * queue-region) + reg-offset-in-region. 70 * 71 * The first part of queue region is basically the same for HIP08 and HIP09 72 * register layouts, therefore, HISI_QUEUE_* registers are defined for it. 73 */ 74 #define HISI_DMA_QUEUE_SQ_BASE_L_REG 0x0 75 #define HISI_DMA_QUEUE_SQ_BASE_H_REG 0x4 76 #define HISI_DMA_QUEUE_SQ_DEPTH_REG 0x8 77 #define HISI_DMA_QUEUE_SQ_TAIL_REG 0xC 78 #define HISI_DMA_QUEUE_CQ_BASE_L_REG 0x10 79 #define HISI_DMA_QUEUE_CQ_BASE_H_REG 0x14 80 #define HISI_DMA_QUEUE_CQ_DEPTH_REG 0x18 81 #define HISI_DMA_QUEUE_CQ_HEAD_REG 0x1C 82 #define HISI_DMA_QUEUE_CTRL0_REG 0x20 83 #define HISI_DMA_QUEUE_CTRL0_EN_B 0 84 #define HISI_DMA_QUEUE_CTRL0_PAUSE_B 4 85 #define HISI_DMA_QUEUE_CTRL1_REG 0x24 86 #define HISI_DMA_QUEUE_CTRL1_RESET_B 0 87 #define HISI_DMA_QUEUE_FSM_REG 0x30 88 #define HISI_DMA_QUEUE_FSM_STS_M GENMASK(3, 0) 89 #define HISI_DMA_QUEUE_INT_STATUS_REG 0x40 90 #define HISI_DMA_QUEUE_INT_MASK_REG 0x44 91 #define HISI_DMA_QUEUE_ERR_INT_NUM0_REG 0x84 92 #define HISI_DMA_QUEUE_ERR_INT_NUM1_REG 0x88 93 #define HISI_DMA_QUEUE_ERR_INT_NUM2_REG 0x8C 94 #define HISI_DMA_QUEUE_REGION_SIZE 0x100 95 96 /** 97 * HiSilicon IP08 DMA register and field define: 98 */ 99 #define HISI_DMA_HIP08_QUEUE_BASE 0x0 100 #define HISI_DMA_HIP08_QUEUE_CTRL0_ERR_ABORT_B 2 101 #define HISI_DMA_HIP08_QUEUE_INT_MASK_M GENMASK(14, 0) 102 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM3_REG 0x90 103 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM4_REG 0x94 104 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM5_REG 0x98 105 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM6_REG 0x48 106 #define HISI_DMA_HIP08_MODE_REG 0x217C 107 #define HISI_DMA_HIP08_MODE_SEL_B 0 108 #define HISI_DMA_HIP08_DUMP_START_REG 0x2000 109 #define HISI_DMA_HIP08_DUMP_END_REG 0x2280 110 111 /** 112 * HiSilicon IP09 DMA register and field define: 113 */ 114 #define HISI_DMA_HIP09_QUEUE_BASE 0x2000 115 #define HISI_DMA_HIP09_QUEUE_CTRL0_ERR_ABORT_M GENMASK(31, 28) 116 #define HISI_DMA_HIP09_QUEUE_CTRL1_VA_ENABLE_B 2 117 #define HISI_DMA_HIP09_QUEUE_INT_MASK_M 0x1 118 #define HISI_DMA_HIP09_QUEUE_ERR_INT_STATUS_REG 0x48 119 #define HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_REG 0x4C 120 #define HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_M GENMASK(18, 1) 121 #define HISI_DMA_HIP09_QUEUE_CFG_REG(queue_id) (0x800 + \ 122 (queue_id) * 0x20) 123 #define HISI_DMA_HIP09_QUEUE_CFG_LINK_DOWN_MASK_B 16 124 #define HISI_DMA_HIP09_DUMP_REGION_A_START_REG 0x0 125 #define HISI_DMA_HIP09_DUMP_REGION_A_END_REG 0x368 126 #define HISI_DMA_HIP09_DUMP_REGION_B_START_REG 0x800 127 #define HISI_DMA_HIP09_DUMP_REGION_B_END_REG 0xA08 128 #define HISI_DMA_HIP09_DUMP_REGION_C_START_REG 0x1800 129 #define HISI_DMA_HIP09_DUMP_REGION_C_END_REG 0x1A4C 130 #define HISI_DMA_HIP09_DUMP_REGION_D_START_REG 0x1C00 131 #define HISI_DMA_HIP09_DUMP_REGION_D_END_REG 0x1CC4 132 133 /** 134 * In fact, there are multiple states, but it need to pay attention to 135 * the following two states for the driver: 136 */ 137 enum { 138 HISI_DMA_STATE_IDLE = 0, 139 HISI_DMA_STATE_RUN, 140 }; 141 142 /** 143 * Hardware complete status define: 144 */ 145 #define HISI_DMA_STATUS_SUCCESS 0x0 146 #define HISI_DMA_STATUS_INVALID_OPCODE 0x1 147 #define HISI_DMA_STATUS_INVALID_LENGTH 0x2 148 #define HISI_DMA_STATUS_USER_ABORT 0x4 149 #define HISI_DMA_STATUS_REMOTE_READ_ERROR 0x10 150 #define HISI_DMA_STATUS_AXI_READ_ERROR 0x20 151 #define HISI_DMA_STATUS_AXI_WRITE_ERROR 0x40 152 #define HISI_DMA_STATUS_DATA_POISON 0x80 153 #define HISI_DMA_STATUS_SQE_READ_ERROR 0x100 154 #define HISI_DMA_STATUS_SQE_READ_POISION 0x200 155 #define HISI_DMA_STATUS_REMOTE_DATA_POISION 0x400 156 #define HISI_DMA_STATUS_LINK_DOWN_ERROR 0x800 157 158 /** 159 * After scanning the CQ array, the CQ head register needs to be updated. 160 * Updating the register involves write memory barrier operations. 161 * Here use the following method to reduce WMB operations: 162 * a) malloc more CQEs, which correspond to the macro HISI_DMA_CQ_RESERVED. 163 * b) update the CQ head register after accumulated number of completed CQs 164 * is greater than or equal to HISI_DMA_CQ_RESERVED. 165 */ 166 #define HISI_DMA_CQ_RESERVED 64 167 168 struct hisi_dma_sqe { 169 uint32_t dw0; 170 #define SQE_FENCE_FLAG BIT(10) 171 #define SQE_OPCODE_M2M 0x4 172 uint32_t dw1; 173 uint32_t dw2; 174 uint32_t length; 175 uint64_t src_addr; 176 uint64_t dst_addr; 177 }; 178 179 struct hisi_dma_cqe { 180 uint64_t rsv; 181 uint64_t misc; 182 #define CQE_SQ_HEAD_MASK GENMASK(15, 0) 183 #define CQE_VALID_B BIT(48) 184 #define CQE_STATUS_MASK GENMASK(63, 49) 185 }; 186 187 struct hisi_dma_dev { 188 struct hisi_dma_sqe *sqe; 189 volatile struct hisi_dma_cqe *cqe; 190 uint16_t *status; /* the completion status array of SQEs. */ 191 192 volatile void *sq_tail_reg; /**< register address for doorbell. */ 193 volatile void *cq_head_reg; /**< register address for answer CQ. */ 194 195 uint16_t sq_depth_mask; /**< SQ depth - 1, the SQ depth is power of 2 */ 196 uint16_t cq_depth; /* CQ depth */ 197 198 uint16_t ridx; /**< ring index which will assign to the next request. */ 199 /** ring index which returned by hisi_dmadev_completed APIs. */ 200 uint16_t cridx; 201 202 /** 203 * SQE array management fields: 204 * 205 * ----------------------------------------------------- 206 * | SQE0 | SQE1 | SQE2 | ... | SQEx | ... | SQEn-1 | 207 * ----------------------------------------------------- 208 * ^ ^ ^ 209 * | | | 210 * sq_head cq_sq_head sq_tail 211 * 212 * sq_head: index to the oldest completed request, this filed was 213 * updated by hisi_dmadev_completed* APIs. 214 * sq_tail: index of the next new request, this field was updated by 215 * hisi_dmadev_copy API. 216 * cq_sq_head: next index of index that has been completed by hardware, 217 * this filed was updated by hisi_dmadev_completed* APIs. 218 * 219 * [sq_head, cq_sq_head): the SQEs that hardware already completed. 220 * [cq_sq_head, sq_tail): the SQEs that hardware processing. 221 */ 222 uint16_t sq_head; 223 uint16_t sq_tail; 224 uint16_t cq_sq_head; 225 /** 226 * The driver scans the CQE array, if the valid bit changes, the CQE is 227 * considered valid. 228 * Note: One CQE is corresponding to one or several SQEs, e.g. app 229 * submits two copy requests, the hardware processes the two SQEs, 230 * but it may write back only one CQE and the CQE's sq_head field 231 * indicates the index of the second copy request in the SQE 232 * array. 233 */ 234 uint16_t cq_head; /**< CQ index for next scans. */ 235 /** accumulated number of completed CQs 236 * @see HISI_DMA_CQ_RESERVED 237 */ 238 uint16_t cqs_completed; 239 uint8_t cqe_vld; /**< valid bit for CQE, will change for every round. */ 240 241 uint64_t submitted; 242 uint64_t completed; 243 uint64_t errors; 244 uint64_t qfulls; 245 246 /** 247 * The following fields are not accessed in the I/O path, so they are 248 * placed at the end. 249 */ 250 struct rte_dma_dev_data *data; 251 uint8_t revision; /**< PCI revision. */ 252 uint8_t reg_layout; /**< hardware register layout. */ 253 void *io_base; 254 uint8_t queue_id; /**< hardware DMA queue index. */ 255 const struct rte_memzone *iomz; 256 uint32_t iomz_sz; 257 rte_iova_t sqe_iova; 258 rte_iova_t cqe_iova; 259 }; 260 261 #endif /* HISI_DMADEV_H */ 262