1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
4 */
5
6 #ifndef _ETH_FAILSAFE_PRIVATE_H_
7 #define _ETH_FAILSAFE_PRIVATE_H_
8
9 #include <stdint.h>
10 #include <sys/queue.h>
11 #include <pthread.h>
12
13 #include <rte_atomic.h>
14 #include <rte_dev.h>
15 #include <rte_ethdev_driver.h>
16 #include <rte_devargs.h>
17 #include <rte_flow.h>
18 #include <rte_interrupts.h>
19
20 #define FAILSAFE_DRIVER_NAME "Fail-safe PMD"
21 #define FAILSAFE_OWNER_NAME "Fail-safe"
22
23 #define PMD_FAILSAFE_MAC_KVARG "mac"
24 #define PMD_FAILSAFE_HOTPLUG_POLL_KVARG "hotplug_poll"
25 #define PMD_FAILSAFE_PARAM_STRING \
26 "dev(<ifc>)," \
27 "exec(<shell command>)," \
28 "fd(<fd number>)," \
29 "mac=mac_addr," \
30 "hotplug_poll=u64" \
31 ""
32
33 #define FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS 2000
34
35 #define FAILSAFE_MAX_ETHPORTS 2
36 #define FAILSAFE_MAX_ETHADDR 128
37
38 #define DEVARGS_MAXLEN 4096
39
40 enum rxp_service_state {
41 SS_NO_SERVICE = 0,
42 SS_REGISTERED,
43 SS_READY,
44 SS_RUNNING,
45 };
46
47 /* TYPES */
48
49 struct rx_proxy {
50 /* epoll file descriptor */
51 int efd;
52 /* event vector to be used by epoll */
53 struct rte_epoll_event *evec;
54 /* rte service id */
55 uint32_t sid;
56 /* service core id */
57 uint32_t scid;
58 enum rxp_service_state sstate;
59 };
60
61 #define FS_RX_PROXY_INIT (struct rx_proxy){ \
62 .efd = -1, \
63 .evec = NULL, \
64 .sid = 0, \
65 .scid = 0, \
66 .sstate = SS_NO_SERVICE, \
67 }
68
69 struct rxq {
70 struct fs_priv *priv;
71 uint16_t qid;
72 /* next sub_device to poll */
73 struct sub_device *sdev;
74 unsigned int socket_id;
75 int event_fd;
76 unsigned int enable_events:1;
77 struct rte_eth_rxq_info info;
78 rte_atomic64_t refcnt[];
79 };
80
81 struct txq {
82 struct fs_priv *priv;
83 uint16_t qid;
84 unsigned int socket_id;
85 struct rte_eth_txq_info info;
86 rte_atomic64_t refcnt[];
87 };
88
89 struct rte_flow {
90 TAILQ_ENTRY(rte_flow) next;
91 /* sub_flows */
92 struct rte_flow *flows[FAILSAFE_MAX_ETHPORTS];
93 /* flow description for synchronization */
94 struct rte_flow_conv_rule rule;
95 uint8_t rule_data[];
96 };
97
98 enum dev_state {
99 DEV_UNDEFINED,
100 DEV_PARSED,
101 DEV_PROBED,
102 DEV_ACTIVE,
103 DEV_STARTED,
104 };
105
106 struct fs_stats {
107 struct rte_eth_stats stats;
108 uint64_t timestamp;
109 };
110
111 /*
112 * Allocated in shared memory.
113 */
114 struct sub_device {
115 /* Exhaustive DPDK device description */
116 struct sub_device *next;
117 struct rte_devargs devargs;
118 struct rte_bus *bus; /* for primary process only. */
119 struct rte_device *dev; /* for primary process only. */
120 uint8_t sid;
121 /* Device state machine */
122 enum dev_state state;
123 /* Last stats snapshot passed to user */
124 struct fs_stats stats_snapshot;
125 /* Some device are defined as a command line */
126 char *cmdline;
127 /* Others are retrieved through a file descriptor */
128 char *fd_str;
129 /* fail-safe device backreference */
130 uint16_t fs_port_id; /* shared between processes */
131 /* sub device port id*/
132 uint16_t sdev_port_id; /* shared between processes */
133 /* flag calling for recollection */
134 volatile unsigned int remove:1;
135 /* flow isolation state */
136 int flow_isolated:1;
137 /* RMV callback registration state */
138 unsigned int rmv_callback:1;
139 /* LSC callback registration state */
140 unsigned int lsc_callback:1;
141 };
142
143 /*
144 * This is referenced by eth_dev->data->dev_private
145 * This is shared between processes.
146 */
147 struct fs_priv {
148 struct rte_eth_dev_data *data; /* backreference to shared data. */
149 /*
150 * Set of sub_devices.
151 * subs[0] is the preferred device
152 * any other is just another sub device
153 */
154 struct sub_device *subs; /* shared between processes */
155 uint8_t subs_head; /* if head == tail, no subs */
156 uint8_t subs_tail; /* first invalid */
157 uint8_t subs_tx; /* current emitting device */
158 uint8_t current_probed;
159 /* flow mapping */
160 TAILQ_HEAD(sub_flows, rte_flow) flow_list;
161 /* current number of mac_addr slots allocated. */
162 uint32_t nb_mac_addr;
163 struct rte_ether_addr mac_addrs[FAILSAFE_MAX_ETHADDR];
164 uint32_t mac_addr_pool[FAILSAFE_MAX_ETHADDR];
165 uint32_t nb_mcast_addr;
166 struct rte_ether_addr *mcast_addrs;
167 /* current capabilities */
168 struct rte_eth_dev_owner my_owner; /* Unique owner. */
169 struct rte_intr_handle intr_handle; /* Port interrupt handle. */
170 /*
171 * Fail-safe state machine.
172 * This level will be tracking state of the EAL and eth
173 * layer at large as defined by the user application.
174 * It will then steer the sub_devices toward the same
175 * synchronized state.
176 */
177 enum dev_state state;
178 struct rte_eth_stats stats_accumulator;
179 /*
180 * Rx interrupts/events proxy.
181 * The PMD issues Rx events to the EAL on behalf of its subdevices,
182 * it does that by registering an event-fd for each of its queues with
183 * the EAL. A PMD service thread listens to all the Rx events from the
184 * subdevices, when an Rx event is issued by a subdevice it will be
185 * caught by this service with will trigger an Rx event in the
186 * appropriate failsafe Rx queue.
187 */
188 struct rx_proxy rxp;
189 pthread_mutex_t hotplug_mutex;
190 /* Hot-plug mutex is locked by the alarm mechanism. */
191 volatile unsigned int alarm_lock:1;
192 unsigned int pending_alarm:1; /* An alarm is pending */
193 /* flow isolation state */
194 int flow_isolated:1;
195 };
196
197 /* FAILSAFE_INTR */
198
199 int failsafe_rx_intr_install(struct rte_eth_dev *dev);
200 void failsafe_rx_intr_uninstall(struct rte_eth_dev *dev);
201 int failsafe_rx_intr_install_subdevice(struct sub_device *sdev);
202 void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev);
203
204 /* MISC */
205
206 int failsafe_hotplug_alarm_install(struct rte_eth_dev *dev);
207 int failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev);
208
209 /* RX / TX */
210
211 void failsafe_set_burst_fn(struct rte_eth_dev *dev, int force_safe);
212
213 uint16_t failsafe_rx_burst(void *rxq,
214 struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
215 uint16_t failsafe_tx_burst(void *txq,
216 struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
217
218 uint16_t failsafe_rx_burst_fast(void *rxq,
219 struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
220 uint16_t failsafe_tx_burst_fast(void *txq,
221 struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
222
223 /* ARGS */
224
225 int failsafe_args_parse(struct rte_eth_dev *dev, const char *params);
226 void failsafe_args_free(struct rte_eth_dev *dev);
227 int failsafe_args_count_subdevice(struct rte_eth_dev *dev, const char *params);
228 int failsafe_args_parse_subs(struct rte_eth_dev *dev);
229
230 /* EAL */
231
232 int failsafe_eal_init(struct rte_eth_dev *dev);
233 int failsafe_eal_uninit(struct rte_eth_dev *dev);
234
235 /* ETH_DEV */
236
237 int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev);
238 void failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev);
239 int failsafe_eth_dev_close(struct rte_eth_dev *dev);
240 void failsafe_dev_remove(struct rte_eth_dev *dev);
241 void failsafe_stats_increment(struct rte_eth_stats *to,
242 struct rte_eth_stats *from);
243 int failsafe_eth_rmv_event_callback(uint16_t port_id,
244 enum rte_eth_event_type type,
245 void *arg, void *out);
246 int failsafe_eth_lsc_event_callback(uint16_t port_id,
247 enum rte_eth_event_type event,
248 void *cb_arg, void *out);
249 int failsafe_eth_new_event_callback(uint16_t port_id,
250 enum rte_eth_event_type event,
251 void *cb_arg, void *out);
252
253 /* GLOBALS */
254
255 extern const char pmd_failsafe_driver_name[];
256 extern const struct eth_dev_ops failsafe_ops;
257 extern const struct rte_flow_ops fs_flow_ops;
258 extern uint64_t failsafe_hotplug_poll;
259 extern int failsafe_mac_from_arg;
260
261 /* HELPERS */
262
263 /* dev: (struct rte_eth_dev *) fail-safe device */
264 #define PRIV(dev) \
265 ((struct fs_priv *)(dev)->data->dev_private)
266
267 /* sdev: (struct sub_device *) */
268 #define ETH(sdev) \
269 ((sdev)->sdev_port_id == RTE_MAX_ETHPORTS ? \
270 NULL : &rte_eth_devices[(sdev)->sdev_port_id])
271
272 /* sdev: (struct sub_device *) */
273 #define PORT_ID(sdev) \
274 ((sdev)->sdev_port_id)
275
276 /* sdev: (struct sub_device *) */
277 #define SUB_ID(sdev) \
278 ((sdev)->sid)
279
280 /**
281 * Stateful iterator construct over fail-safe sub-devices:
282 * s: (struct sub_device *), iterator
283 * i: (uint8_t), increment
284 * dev: (struct rte_eth_dev *), fail-safe ethdev
285 * state: (enum dev_state), minimum acceptable device state
286 */
287 #define FOREACH_SUBDEV_STATE(s, i, dev, state) \
288 for (s = fs_find_next((dev), 0, state, &i); \
289 s != NULL; \
290 s = fs_find_next((dev), i + 1, state, &i))
291
292 /**
293 * Iterator construct over fail-safe sub-devices:
294 * s: (struct sub_device *), iterator
295 * i: (uint8_t), increment
296 * dev: (struct rte_eth_dev *), fail-safe ethdev
297 */
298 #define FOREACH_SUBDEV(s, i, dev) \
299 FOREACH_SUBDEV_STATE(s, i, dev, DEV_UNDEFINED)
300
301 /* dev: (struct rte_eth_dev *) fail-safe device */
302 #define PREFERRED_SUBDEV(dev) \
303 (&PRIV(dev)->subs[0])
304
305 /* dev: (struct rte_eth_dev *) fail-safe device */
306 #define TX_SUBDEV(dev) \
307 (PRIV(dev)->subs_tx >= PRIV(dev)->subs_tail ? NULL \
308 : (PRIV(dev)->subs[PRIV(dev)->subs_tx].state < DEV_PROBED ? NULL \
309 : &PRIV(dev)->subs[PRIV(dev)->subs_tx]))
310
311 /**
312 * s: (struct sub_device *)
313 * ops: (struct eth_dev_ops) member
314 */
315 #define SUBOPS(s, ops) \
316 (ETH(s)->dev_ops->ops)
317
318 /**
319 * Atomic guard
320 */
321
322 /**
323 * a: (rte_atomic64_t)
324 */
325 #define FS_ATOMIC_P(a) \
326 rte_atomic64_set(&(a), 1)
327
328 /**
329 * a: (rte_atomic64_t)
330 */
331 #define FS_ATOMIC_V(a) \
332 rte_atomic64_set(&(a), 0)
333
334 /**
335 * s: (struct sub_device *)
336 * i: uint16_t qid
337 */
338 #define FS_ATOMIC_RX(s, i) \
339 rte_atomic64_read( \
340 &((struct rxq *) \
341 (fs_dev(s)->data->rx_queues[i]))->refcnt[(s)->sid])
342 /**
343 * s: (struct sub_device *)
344 * i: uint16_t qid
345 */
346 #define FS_ATOMIC_TX(s, i) \
347 rte_atomic64_read( \
348 &((struct txq *) \
349 (fs_dev(s)->data->tx_queues[i]))->refcnt[(s)->sid])
350
351 #ifdef RTE_EXEC_ENV_FREEBSD
352 #define FS_THREADID_TYPE void*
353 #define FS_THREADID_FMT "p"
354 #else
355 #define FS_THREADID_TYPE unsigned long
356 #define FS_THREADID_FMT "lu"
357 #endif
358
359 extern int failsafe_logtype;
360
361 #define LOG__(l, m, ...) \
362 rte_log(RTE_LOG_ ## l, failsafe_logtype, \
363 "net_failsafe: " m "%c", __VA_ARGS__)
364
365 #define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n')
366 #define DEBUG(...) LOG_(DEBUG, __VA_ARGS__)
367 #define INFO(...) LOG_(INFO, __VA_ARGS__)
368 #define WARN(...) LOG_(WARNING, __VA_ARGS__)
369 #define ERROR(...) LOG_(ERR, __VA_ARGS__)
370
371 /* inlined functions */
372
373 static inline struct sub_device *
fs_find_next(struct rte_eth_dev * dev,uint8_t sid,enum dev_state min_state,uint8_t * sid_out)374 fs_find_next(struct rte_eth_dev *dev,
375 uint8_t sid,
376 enum dev_state min_state,
377 uint8_t *sid_out)
378 {
379 struct sub_device *subs;
380 uint8_t tail;
381
382 subs = PRIV(dev)->subs;
383 tail = PRIV(dev)->subs_tail;
384 while (sid < tail) {
385 if (subs[sid].state >= min_state)
386 break;
387 sid++;
388 }
389 *sid_out = sid;
390 if (sid >= tail)
391 return NULL;
392 return &subs[sid];
393 }
394
395 static inline struct rte_eth_dev *
fs_dev(struct sub_device * sdev)396 fs_dev(struct sub_device *sdev) {
397 return &rte_eth_devices[sdev->fs_port_id];
398 }
399
400 /*
401 * Lock hot-plug mutex.
402 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism.
403 */
404 static inline int
fs_lock(struct rte_eth_dev * dev,unsigned int is_alarm)405 fs_lock(struct rte_eth_dev *dev, unsigned int is_alarm)
406 {
407 int ret;
408
409 if (is_alarm) {
410 ret = pthread_mutex_trylock(&PRIV(dev)->hotplug_mutex);
411 if (ret) {
412 DEBUG("Hot-plug mutex lock trying failed(%s), will try"
413 " again later...", strerror(ret));
414 return ret;
415 }
416 PRIV(dev)->alarm_lock = 1;
417 } else {
418 ret = pthread_mutex_lock(&PRIV(dev)->hotplug_mutex);
419 if (ret) {
420 ERROR("Cannot lock mutex(%s)", strerror(ret));
421 return ret;
422 }
423 }
424 return ret;
425 }
426
427 /*
428 * Unlock hot-plug mutex.
429 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism.
430 */
431 static inline void
fs_unlock(struct rte_eth_dev * dev,unsigned int is_alarm)432 fs_unlock(struct rte_eth_dev *dev, unsigned int is_alarm)
433 {
434 int ret;
435
436 if (is_alarm) {
437 RTE_ASSERT(PRIV(dev)->alarm_lock == 1);
438 PRIV(dev)->alarm_lock = 0;
439 }
440 ret = pthread_mutex_unlock(&PRIV(dev)->hotplug_mutex);
441 if (ret)
442 ERROR("Cannot unlock hot-plug mutex(%s)", strerror(ret));
443 }
444
445 /*
446 * Switch emitting device.
447 * If banned is set, banned must not be considered for
448 * the role of emitting device.
449 */
450 static inline void
fs_switch_dev(struct rte_eth_dev * dev,struct sub_device * banned)451 fs_switch_dev(struct rte_eth_dev *dev,
452 struct sub_device *banned)
453 {
454 struct sub_device *txd;
455 enum dev_state req_state;
456
457 req_state = PRIV(dev)->state;
458 txd = TX_SUBDEV(dev);
459 if (PREFERRED_SUBDEV(dev)->state >= req_state &&
460 PREFERRED_SUBDEV(dev) != banned) {
461 if (txd != PREFERRED_SUBDEV(dev) &&
462 (txd == NULL ||
463 (req_state == DEV_STARTED) ||
464 (txd && txd->state < DEV_STARTED))) {
465 DEBUG("Switching tx_dev to preferred sub_device");
466 PRIV(dev)->subs_tx = 0;
467 }
468 } else if ((txd && txd->state < req_state) ||
469 txd == NULL ||
470 txd == banned) {
471 struct sub_device *sdev = NULL;
472 uint8_t i;
473
474 /* Using acceptable device */
475 FOREACH_SUBDEV_STATE(sdev, i, dev, req_state) {
476 if (sdev == banned)
477 continue;
478 DEBUG("Switching tx_dev to sub_device %d",
479 i);
480 PRIV(dev)->subs_tx = i;
481 break;
482 }
483 if (i >= PRIV(dev)->subs_tail || sdev == NULL) {
484 DEBUG("No device ready, deactivating tx_dev");
485 PRIV(dev)->subs_tx = PRIV(dev)->subs_tail;
486 }
487 } else {
488 return;
489 }
490 failsafe_set_burst_fn(dev, 0);
491 rte_wmb();
492 }
493
494 /*
495 * Adjust error value and rte_errno to the fail-safe actual error value.
496 */
497 static inline int
fs_err(struct sub_device * sdev,int err)498 fs_err(struct sub_device *sdev, int err)
499 {
500 /* A device removal shouldn't be reported as an error. */
501 if (sdev->remove == 1 || err == -EIO)
502 return rte_errno = 0;
503 return err;
504 }
505 #endif /* _ETH_FAILSAFE_PRIVATE_H_ */
506