1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #ifndef _RTE_ETH_FAILSAFE_PRIVATE_H_ 7 #define _RTE_ETH_FAILSAFE_PRIVATE_H_ 8 9 #include <stdint.h> 10 #include <sys/queue.h> 11 #include <pthread.h> 12 13 #include <rte_atomic.h> 14 #include <rte_dev.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_devargs.h> 17 #include <rte_flow.h> 18 #include <rte_interrupts.h> 19 20 #define FAILSAFE_DRIVER_NAME "Fail-safe PMD" 21 #define FAILSAFE_OWNER_NAME "Fail-safe" 22 23 #define PMD_FAILSAFE_MAC_KVARG "mac" 24 #define PMD_FAILSAFE_HOTPLUG_POLL_KVARG "hotplug_poll" 25 #define PMD_FAILSAFE_PARAM_STRING \ 26 "dev(<ifc>)," \ 27 "exec(<shell command>)," \ 28 "fd(<fd number>)," \ 29 "mac=mac_addr," \ 30 "hotplug_poll=u64" \ 31 "" 32 33 #define FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS 2000 34 35 #define FAILSAFE_MAX_ETHPORTS 2 36 #define FAILSAFE_MAX_ETHADDR 128 37 38 #define DEVARGS_MAXLEN 4096 39 40 enum rxp_service_state { 41 SS_NO_SERVICE = 0, 42 SS_REGISTERED, 43 SS_READY, 44 SS_RUNNING, 45 }; 46 47 /* TYPES */ 48 49 struct rx_proxy { 50 /* epoll file descriptor */ 51 int efd; 52 /* event vector to be used by epoll */ 53 struct rte_epoll_event *evec; 54 /* rte service id */ 55 uint32_t sid; 56 /* service core id */ 57 uint32_t scid; 58 enum rxp_service_state sstate; 59 }; 60 61 struct rxq { 62 struct fs_priv *priv; 63 uint16_t qid; 64 /* next sub_device to poll */ 65 struct sub_device *sdev; 66 unsigned int socket_id; 67 int event_fd; 68 unsigned int enable_events:1; 69 struct rte_eth_rxq_info info; 70 rte_atomic64_t refcnt[]; 71 }; 72 73 struct txq { 74 struct fs_priv *priv; 75 uint16_t qid; 76 unsigned int socket_id; 77 struct rte_eth_txq_info info; 78 rte_atomic64_t refcnt[]; 79 }; 80 81 struct rte_flow { 82 TAILQ_ENTRY(rte_flow) next; 83 /* sub_flows */ 84 struct rte_flow *flows[FAILSAFE_MAX_ETHPORTS]; 85 /* flow description for synchronization */ 86 struct rte_flow_conv_rule rule; 87 uint8_t rule_data[]; 88 }; 89 90 enum dev_state { 91 DEV_UNDEFINED, 92 DEV_PARSED, 93 DEV_PROBED, 94 DEV_ACTIVE, 95 DEV_STARTED, 96 }; 97 98 struct fs_stats { 99 struct rte_eth_stats stats; 100 uint64_t timestamp; 101 }; 102 103 struct sub_device { 104 /* Exhaustive DPDK device description */ 105 struct sub_device *next; 106 struct rte_devargs devargs; 107 struct rte_bus *bus; 108 struct rte_device *dev; 109 struct rte_eth_dev *edev; 110 uint8_t sid; 111 /* Device state machine */ 112 enum dev_state state; 113 /* Last stats snapshot passed to user */ 114 struct fs_stats stats_snapshot; 115 /* Some device are defined as a command line */ 116 char *cmdline; 117 /* Others are retrieved through a file descriptor */ 118 char *fd_str; 119 /* fail-safe device backreference */ 120 struct rte_eth_dev *fs_dev; 121 /* flag calling for recollection */ 122 volatile unsigned int remove:1; 123 /* flow isolation state */ 124 int flow_isolated:1; 125 /* RMV callback registration state */ 126 unsigned int rmv_callback:1; 127 /* LSC callback registration state */ 128 unsigned int lsc_callback:1; 129 }; 130 131 struct fs_priv { 132 struct rte_eth_dev *dev; 133 /* 134 * Set of sub_devices. 135 * subs[0] is the preferred device 136 * any other is just another slave 137 */ 138 struct sub_device *subs; 139 uint8_t subs_head; /* if head == tail, no subs */ 140 uint8_t subs_tail; /* first invalid */ 141 uint8_t subs_tx; /* current emitting device */ 142 uint8_t current_probed; 143 /* flow mapping */ 144 TAILQ_HEAD(sub_flows, rte_flow) flow_list; 145 /* current number of mac_addr slots allocated. */ 146 uint32_t nb_mac_addr; 147 struct ether_addr mac_addrs[FAILSAFE_MAX_ETHADDR]; 148 uint32_t mac_addr_pool[FAILSAFE_MAX_ETHADDR]; 149 uint32_t nb_mcast_addr; 150 struct ether_addr *mcast_addrs; 151 /* current capabilities */ 152 struct rte_eth_dev_owner my_owner; /* Unique owner. */ 153 struct rte_intr_handle intr_handle; /* Port interrupt handle. */ 154 /* 155 * Fail-safe state machine. 156 * This level will be tracking state of the EAL and eth 157 * layer at large as defined by the user application. 158 * It will then steer the sub_devices toward the same 159 * synchronized state. 160 */ 161 enum dev_state state; 162 struct rte_eth_stats stats_accumulator; 163 /* 164 * Rx interrupts/events proxy. 165 * The PMD issues Rx events to the EAL on behalf of its subdevices, 166 * it does that by registering an event-fd for each of its queues with 167 * the EAL. A PMD service thread listens to all the Rx events from the 168 * subdevices, when an Rx event is issued by a subdevice it will be 169 * caught by this service with will trigger an Rx event in the 170 * appropriate failsafe Rx queue. 171 */ 172 struct rx_proxy rxp; 173 pthread_mutex_t hotplug_mutex; 174 /* Hot-plug mutex is locked by the alarm mechanism. */ 175 volatile unsigned int alarm_lock:1; 176 unsigned int pending_alarm:1; /* An alarm is pending */ 177 /* flow isolation state */ 178 int flow_isolated:1; 179 }; 180 181 /* FAILSAFE_INTR */ 182 183 int failsafe_rx_intr_install(struct rte_eth_dev *dev); 184 void failsafe_rx_intr_uninstall(struct rte_eth_dev *dev); 185 int failsafe_rx_intr_install_subdevice(struct sub_device *sdev); 186 void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev); 187 188 /* MISC */ 189 190 int failsafe_hotplug_alarm_install(struct rte_eth_dev *dev); 191 int failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev); 192 193 /* RX / TX */ 194 195 void failsafe_set_burst_fn(struct rte_eth_dev *dev, int force_safe); 196 197 uint16_t failsafe_rx_burst(void *rxq, 198 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 199 uint16_t failsafe_tx_burst(void *txq, 200 struct rte_mbuf **tx_pkts, uint16_t nb_pkts); 201 202 uint16_t failsafe_rx_burst_fast(void *rxq, 203 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 204 uint16_t failsafe_tx_burst_fast(void *txq, 205 struct rte_mbuf **tx_pkts, uint16_t nb_pkts); 206 207 /* ARGS */ 208 209 int failsafe_args_parse(struct rte_eth_dev *dev, const char *params); 210 void failsafe_args_free(struct rte_eth_dev *dev); 211 int failsafe_args_count_subdevice(struct rte_eth_dev *dev, const char *params); 212 int failsafe_args_parse_subs(struct rte_eth_dev *dev); 213 214 /* EAL */ 215 216 int failsafe_eal_init(struct rte_eth_dev *dev); 217 int failsafe_eal_uninit(struct rte_eth_dev *dev); 218 219 /* ETH_DEV */ 220 221 int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev); 222 void failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev); 223 void failsafe_dev_remove(struct rte_eth_dev *dev); 224 void failsafe_stats_increment(struct rte_eth_stats *to, 225 struct rte_eth_stats *from); 226 int failsafe_eth_rmv_event_callback(uint16_t port_id, 227 enum rte_eth_event_type type, 228 void *arg, void *out); 229 int failsafe_eth_lsc_event_callback(uint16_t port_id, 230 enum rte_eth_event_type event, 231 void *cb_arg, void *out); 232 int failsafe_eth_new_event_callback(uint16_t port_id, 233 enum rte_eth_event_type event, 234 void *cb_arg, void *out); 235 236 /* GLOBALS */ 237 238 extern const char pmd_failsafe_driver_name[]; 239 extern const struct eth_dev_ops failsafe_ops; 240 extern const struct rte_flow_ops fs_flow_ops; 241 extern uint64_t failsafe_hotplug_poll; 242 extern int failsafe_mac_from_arg; 243 244 /* HELPERS */ 245 246 /* dev: (struct rte_eth_dev *) fail-safe device */ 247 #define PRIV(dev) \ 248 ((struct fs_priv *)(dev)->data->dev_private) 249 250 /* sdev: (struct sub_device *) */ 251 #define ETH(sdev) \ 252 ((sdev)->edev) 253 254 /* sdev: (struct sub_device *) */ 255 #define PORT_ID(sdev) \ 256 (ETH(sdev)->data->port_id) 257 258 /* sdev: (struct sub_device *) */ 259 #define SUB_ID(sdev) \ 260 ((sdev)->sid) 261 262 /** 263 * Stateful iterator construct over fail-safe sub-devices: 264 * s: (struct sub_device *), iterator 265 * i: (uint8_t), increment 266 * dev: (struct rte_eth_dev *), fail-safe ethdev 267 * state: (enum dev_state), minimum acceptable device state 268 */ 269 #define FOREACH_SUBDEV_STATE(s, i, dev, state) \ 270 for (s = fs_find_next((dev), 0, state, &i); \ 271 s != NULL; \ 272 s = fs_find_next((dev), i + 1, state, &i)) 273 274 /** 275 * Iterator construct over fail-safe sub-devices: 276 * s: (struct sub_device *), iterator 277 * i: (uint8_t), increment 278 * dev: (struct rte_eth_dev *), fail-safe ethdev 279 */ 280 #define FOREACH_SUBDEV(s, i, dev) \ 281 FOREACH_SUBDEV_STATE(s, i, dev, DEV_UNDEFINED) 282 283 /* dev: (struct rte_eth_dev *) fail-safe device */ 284 #define PREFERRED_SUBDEV(dev) \ 285 (&PRIV(dev)->subs[0]) 286 287 /* dev: (struct rte_eth_dev *) fail-safe device */ 288 #define TX_SUBDEV(dev) \ 289 (PRIV(dev)->subs_tx >= PRIV(dev)->subs_tail ? NULL \ 290 : (PRIV(dev)->subs[PRIV(dev)->subs_tx].state < DEV_PROBED ? NULL \ 291 : &PRIV(dev)->subs[PRIV(dev)->subs_tx])) 292 293 /** 294 * s: (struct sub_device *) 295 * ops: (struct eth_dev_ops) member 296 */ 297 #define SUBOPS(s, ops) \ 298 (ETH(s)->dev_ops->ops) 299 300 /** 301 * Atomic guard 302 */ 303 304 /** 305 * a: (rte_atomic64_t) 306 */ 307 #define FS_ATOMIC_P(a) \ 308 rte_atomic64_set(&(a), 1) 309 310 /** 311 * a: (rte_atomic64_t) 312 */ 313 #define FS_ATOMIC_V(a) \ 314 rte_atomic64_set(&(a), 0) 315 316 /** 317 * s: (struct sub_device *) 318 * i: uint16_t qid 319 */ 320 #define FS_ATOMIC_RX(s, i) \ 321 rte_atomic64_read( \ 322 &((struct rxq *)((s)->fs_dev->data->rx_queues[i]))->refcnt[(s)->sid] \ 323 ) 324 /** 325 * s: (struct sub_device *) 326 * i: uint16_t qid 327 */ 328 #define FS_ATOMIC_TX(s, i) \ 329 rte_atomic64_read( \ 330 &((struct txq *)((s)->fs_dev->data->tx_queues[i]))->refcnt[(s)->sid] \ 331 ) 332 333 #ifdef RTE_EXEC_ENV_BSDAPP 334 #define FS_THREADID_TYPE void* 335 #define FS_THREADID_FMT "p" 336 #else 337 #define FS_THREADID_TYPE unsigned long 338 #define FS_THREADID_FMT "lu" 339 #endif 340 341 extern int failsafe_logtype; 342 343 #define LOG__(l, m, ...) \ 344 rte_log(RTE_LOG_ ## l, failsafe_logtype, \ 345 "net_failsafe: " m "%c", __VA_ARGS__) 346 347 #define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n') 348 #define DEBUG(...) LOG_(DEBUG, __VA_ARGS__) 349 #define INFO(...) LOG_(INFO, __VA_ARGS__) 350 #define WARN(...) LOG_(WARNING, __VA_ARGS__) 351 #define ERROR(...) LOG_(ERR, __VA_ARGS__) 352 353 /* inlined functions */ 354 355 static inline struct sub_device * 356 fs_find_next(struct rte_eth_dev *dev, 357 uint8_t sid, 358 enum dev_state min_state, 359 uint8_t *sid_out) 360 { 361 struct sub_device *subs; 362 uint8_t tail; 363 364 subs = PRIV(dev)->subs; 365 tail = PRIV(dev)->subs_tail; 366 while (sid < tail) { 367 if (subs[sid].state >= min_state) 368 break; 369 sid++; 370 } 371 *sid_out = sid; 372 if (sid >= tail) 373 return NULL; 374 return &subs[sid]; 375 } 376 377 /* 378 * Lock hot-plug mutex. 379 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism. 380 */ 381 static inline int 382 fs_lock(struct rte_eth_dev *dev, unsigned int is_alarm) 383 { 384 int ret; 385 386 if (is_alarm) { 387 ret = pthread_mutex_trylock(&PRIV(dev)->hotplug_mutex); 388 if (ret) { 389 DEBUG("Hot-plug mutex lock trying failed(%s), will try" 390 " again later...", strerror(ret)); 391 return ret; 392 } 393 PRIV(dev)->alarm_lock = 1; 394 } else { 395 ret = pthread_mutex_lock(&PRIV(dev)->hotplug_mutex); 396 if (ret) { 397 ERROR("Cannot lock mutex(%s)", strerror(ret)); 398 return ret; 399 } 400 } 401 DEBUG("Hot-plug mutex was locked by thread %" FS_THREADID_FMT "%s", 402 (FS_THREADID_TYPE)pthread_self(), 403 PRIV(dev)->alarm_lock ? " by the hot-plug alarm" : ""); 404 return ret; 405 } 406 407 /* 408 * Unlock hot-plug mutex. 409 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism. 410 */ 411 static inline void 412 fs_unlock(struct rte_eth_dev *dev, unsigned int is_alarm) 413 { 414 int ret; 415 unsigned int prev_alarm_lock = PRIV(dev)->alarm_lock; 416 417 if (is_alarm) { 418 RTE_ASSERT(PRIV(dev)->alarm_lock == 1); 419 PRIV(dev)->alarm_lock = 0; 420 } 421 ret = pthread_mutex_unlock(&PRIV(dev)->hotplug_mutex); 422 if (ret) 423 ERROR("Cannot unlock hot-plug mutex(%s)", strerror(ret)); 424 else 425 DEBUG("Hot-plug mutex was unlocked by thread %" FS_THREADID_FMT "%s", 426 (FS_THREADID_TYPE)pthread_self(), 427 prev_alarm_lock ? " by the hot-plug alarm" : ""); 428 } 429 430 /* 431 * Switch emitting device. 432 * If banned is set, banned must not be considered for 433 * the role of emitting device. 434 */ 435 static inline void 436 fs_switch_dev(struct rte_eth_dev *dev, 437 struct sub_device *banned) 438 { 439 struct sub_device *txd; 440 enum dev_state req_state; 441 442 req_state = PRIV(dev)->state; 443 txd = TX_SUBDEV(dev); 444 if (PREFERRED_SUBDEV(dev)->state >= req_state && 445 PREFERRED_SUBDEV(dev) != banned) { 446 if (txd != PREFERRED_SUBDEV(dev) && 447 (txd == NULL || 448 (req_state == DEV_STARTED) || 449 (txd && txd->state < DEV_STARTED))) { 450 DEBUG("Switching tx_dev to preferred sub_device"); 451 PRIV(dev)->subs_tx = 0; 452 } 453 } else if ((txd && txd->state < req_state) || 454 txd == NULL || 455 txd == banned) { 456 struct sub_device *sdev = NULL; 457 uint8_t i; 458 459 /* Using acceptable device */ 460 FOREACH_SUBDEV_STATE(sdev, i, dev, req_state) { 461 if (sdev == banned) 462 continue; 463 DEBUG("Switching tx_dev to sub_device %d", 464 i); 465 PRIV(dev)->subs_tx = i; 466 break; 467 } 468 if (i >= PRIV(dev)->subs_tail || sdev == NULL) { 469 DEBUG("No device ready, deactivating tx_dev"); 470 PRIV(dev)->subs_tx = PRIV(dev)->subs_tail; 471 } 472 } else { 473 return; 474 } 475 failsafe_set_burst_fn(dev, 0); 476 rte_wmb(); 477 } 478 479 /* 480 * Adjust error value and rte_errno to the fail-safe actual error value. 481 */ 482 static inline int 483 fs_err(struct sub_device *sdev, int err) 484 { 485 /* A device removal shouldn't be reported as an error. */ 486 if (sdev->remove == 1 || err == -EIO) 487 return rte_errno = 0; 488 return err; 489 } 490 #endif /* _RTE_ETH_FAILSAFE_PRIVATE_H_ */ 491