/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2017-2018 Intel Corporation. * All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "rte_eventdev.h" #include "rte_eventdev_pmd.h" #include "rte_event_timer_adapter.h" #include "rte_event_timer_adapter_pmd.h" #define DATA_MZ_NAME_MAX_LEN 64 #define DATA_MZ_NAME_FORMAT "rte_event_timer_adapter_data_%d" static int evtim_logtype; static int evtim_svc_logtype; static int evtim_buffer_logtype; static struct rte_event_timer_adapter adapters[RTE_EVENT_TIMER_ADAPTER_NUM_MAX]; static const struct rte_event_timer_adapter_ops sw_event_adapter_timer_ops; #define EVTIM_LOG(level, logtype, ...) \ rte_log(RTE_LOG_ ## level, logtype, \ RTE_FMT("EVTIMER: %s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) \ "\n", __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,))) #define EVTIM_LOG_ERR(...) EVTIM_LOG(ERR, evtim_logtype, __VA_ARGS__) #ifdef RTE_LIBRTE_EVENTDEV_DEBUG #define EVTIM_LOG_DBG(...) \ EVTIM_LOG(DEBUG, evtim_logtype, __VA_ARGS__) #define EVTIM_BUF_LOG_DBG(...) \ EVTIM_LOG(DEBUG, evtim_buffer_logtype, __VA_ARGS__) #define EVTIM_SVC_LOG_DBG(...) \ EVTIM_LOG(DEBUG, evtim_svc_logtype, __VA_ARGS__) #else #define EVTIM_LOG_DBG(...) (void)0 #define EVTIM_BUF_LOG_DBG(...) (void)0 #define EVTIM_SVC_LOG_DBG(...) (void)0 #endif static int default_port_conf_cb(uint16_t id, uint8_t event_dev_id, uint8_t *event_port_id, void *conf_arg) { struct rte_event_timer_adapter *adapter; struct rte_eventdev *dev; struct rte_event_dev_config dev_conf; struct rte_event_port_conf *port_conf, def_port_conf = {0}; int started; uint8_t port_id; uint8_t dev_id; int ret; RTE_SET_USED(event_dev_id); adapter = &adapters[id]; dev = &rte_eventdevs[adapter->data->event_dev_id]; dev_id = dev->data->dev_id; dev_conf = dev->data->dev_conf; started = dev->data->dev_started; if (started) rte_event_dev_stop(dev_id); port_id = dev_conf.nb_event_ports; dev_conf.nb_event_ports += 1; ret = rte_event_dev_configure(dev_id, &dev_conf); if (ret < 0) { EVTIM_LOG_ERR("failed to configure event dev %u\n", dev_id); if (started) if (rte_event_dev_start(dev_id)) return -EIO; return ret; } if (conf_arg != NULL) port_conf = conf_arg; else { port_conf = &def_port_conf; ret = rte_event_port_default_conf_get(dev_id, port_id, port_conf); if (ret < 0) return ret; } ret = rte_event_port_setup(dev_id, port_id, port_conf); if (ret < 0) { EVTIM_LOG_ERR("failed to setup event port %u on event dev %u\n", port_id, dev_id); return ret; } *event_port_id = port_id; if (started) ret = rte_event_dev_start(dev_id); return ret; } struct rte_event_timer_adapter * __rte_experimental rte_event_timer_adapter_create(const struct rte_event_timer_adapter_conf *conf) { return rte_event_timer_adapter_create_ext(conf, default_port_conf_cb, NULL); } struct rte_event_timer_adapter * __rte_experimental rte_event_timer_adapter_create_ext( const struct rte_event_timer_adapter_conf *conf, rte_event_timer_adapter_port_conf_cb_t conf_cb, void *conf_arg) { uint16_t adapter_id; struct rte_event_timer_adapter *adapter; const struct rte_memzone *mz; char mz_name[DATA_MZ_NAME_MAX_LEN]; int n, ret; struct rte_eventdev *dev; if (conf == NULL) { rte_errno = EINVAL; return NULL; } /* Check eventdev ID */ if (!rte_event_pmd_is_valid_dev(conf->event_dev_id)) { rte_errno = EINVAL; return NULL; } dev = &rte_eventdevs[conf->event_dev_id]; adapter_id = conf->timer_adapter_id; /* Check that adapter_id is in range */ if (adapter_id >= RTE_EVENT_TIMER_ADAPTER_NUM_MAX) { rte_errno = EINVAL; return NULL; } /* Check adapter ID not already allocated */ adapter = &adapters[adapter_id]; if (adapter->allocated) { rte_errno = EEXIST; return NULL; } /* Create shared data area. */ n = snprintf(mz_name, sizeof(mz_name), DATA_MZ_NAME_FORMAT, adapter_id); if (n >= (int)sizeof(mz_name)) { rte_errno = EINVAL; return NULL; } mz = rte_memzone_reserve(mz_name, sizeof(struct rte_event_timer_adapter_data), conf->socket_id, 0); if (mz == NULL) /* rte_errno set by rte_memzone_reserve */ return NULL; adapter->data = mz->addr; memset(adapter->data, 0, sizeof(struct rte_event_timer_adapter_data)); adapter->data->mz = mz; adapter->data->event_dev_id = conf->event_dev_id; adapter->data->id = adapter_id; adapter->data->socket_id = conf->socket_id; adapter->data->conf = *conf; /* copy conf structure */ /* Query eventdev PMD for timer adapter capabilities and ops */ ret = dev->dev_ops->timer_adapter_caps_get(dev, adapter->data->conf.flags, &adapter->data->caps, &adapter->ops); if (ret < 0) { rte_errno = -ret; goto free_memzone; } if (!(adapter->data->caps & RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) { FUNC_PTR_OR_NULL_RET_WITH_ERRNO(conf_cb, EINVAL); ret = conf_cb(adapter->data->id, adapter->data->event_dev_id, &adapter->data->event_port_id, conf_arg); if (ret < 0) { rte_errno = -ret; goto free_memzone; } } /* If eventdev PMD did not provide ops, use default software * implementation. */ if (adapter->ops == NULL) adapter->ops = &sw_event_adapter_timer_ops; /* Allow driver to do some setup */ FUNC_PTR_OR_NULL_RET_WITH_ERRNO(adapter->ops->init, ENOTSUP); ret = adapter->ops->init(adapter); if (ret < 0) { rte_errno = -ret; goto free_memzone; } /* Set fast-path function pointers */ adapter->arm_burst = adapter->ops->arm_burst; adapter->arm_tmo_tick_burst = adapter->ops->arm_tmo_tick_burst; adapter->cancel_burst = adapter->ops->cancel_burst; adapter->allocated = 1; return adapter; free_memzone: rte_memzone_free(adapter->data->mz); return NULL; } int __rte_experimental rte_event_timer_adapter_get_info(const struct rte_event_timer_adapter *adapter, struct rte_event_timer_adapter_info *adapter_info) { ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL); if (adapter->ops->get_info) /* let driver set values it knows */ adapter->ops->get_info(adapter, adapter_info); /* Set common values */ adapter_info->conf = adapter->data->conf; adapter_info->event_dev_port_id = adapter->data->event_port_id; adapter_info->caps = adapter->data->caps; return 0; } int __rte_experimental rte_event_timer_adapter_start(const struct rte_event_timer_adapter *adapter) { int ret; ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL); FUNC_PTR_OR_ERR_RET(adapter->ops->start, -EINVAL); ret = adapter->ops->start(adapter); if (ret < 0) return ret; adapter->data->started = 1; return 0; } int __rte_experimental rte_event_timer_adapter_stop(const struct rte_event_timer_adapter *adapter) { int ret; ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL); FUNC_PTR_OR_ERR_RET(adapter->ops->stop, -EINVAL); if (adapter->data->started == 0) { EVTIM_LOG_ERR("event timer adapter %"PRIu8" already stopped", adapter->data->id); return 0; } ret = adapter->ops->stop(adapter); if (ret < 0) return ret; adapter->data->started = 0; return 0; } struct rte_event_timer_adapter * __rte_experimental rte_event_timer_adapter_lookup(uint16_t adapter_id) { char name[DATA_MZ_NAME_MAX_LEN]; const struct rte_memzone *mz; struct rte_event_timer_adapter_data *data; struct rte_event_timer_adapter *adapter; int ret; struct rte_eventdev *dev; if (adapters[adapter_id].allocated) return &adapters[adapter_id]; /* Adapter is already loaded */ snprintf(name, DATA_MZ_NAME_MAX_LEN, DATA_MZ_NAME_FORMAT, adapter_id); mz = rte_memzone_lookup(name); if (mz == NULL) { rte_errno = ENOENT; return NULL; } data = mz->addr; adapter = &adapters[data->id]; adapter->data = data; dev = &rte_eventdevs[adapter->data->event_dev_id]; /* Query eventdev PMD for timer adapter capabilities and ops */ ret = dev->dev_ops->timer_adapter_caps_get(dev, adapter->data->conf.flags, &adapter->data->caps, &adapter->ops); if (ret < 0) { rte_errno = EINVAL; return NULL; } /* If eventdev PMD did not provide ops, use default software * implementation. */ if (adapter->ops == NULL) adapter->ops = &sw_event_adapter_timer_ops; /* Set fast-path function pointers */ adapter->arm_burst = adapter->ops->arm_burst; adapter->arm_tmo_tick_burst = adapter->ops->arm_tmo_tick_burst; adapter->cancel_burst = adapter->ops->cancel_burst; adapter->allocated = 1; return adapter; } int __rte_experimental rte_event_timer_adapter_free(struct rte_event_timer_adapter *adapter) { int ret; ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL); FUNC_PTR_OR_ERR_RET(adapter->ops->uninit, -EINVAL); if (adapter->data->started == 1) { EVTIM_LOG_ERR("event timer adapter %"PRIu8" must be stopped " "before freeing", adapter->data->id); return -EBUSY; } /* free impl priv data */ ret = adapter->ops->uninit(adapter); if (ret < 0) return ret; /* free shared data area */ ret = rte_memzone_free(adapter->data->mz); if (ret < 0) return ret; adapter->data = NULL; adapter->allocated = 0; return 0; } int __rte_experimental rte_event_timer_adapter_service_id_get(struct rte_event_timer_adapter *adapter, uint32_t *service_id) { ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL); if (adapter->data->service_inited && service_id != NULL) *service_id = adapter->data->service_id; return adapter->data->service_inited ? 0 : -ESRCH; } int __rte_experimental rte_event_timer_adapter_stats_get(struct rte_event_timer_adapter *adapter, struct rte_event_timer_adapter_stats *stats) { ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL); FUNC_PTR_OR_ERR_RET(adapter->ops->stats_get, -EINVAL); if (stats == NULL) return -EINVAL; return adapter->ops->stats_get(adapter, stats); } int __rte_experimental rte_event_timer_adapter_stats_reset(struct rte_event_timer_adapter *adapter) { ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL); FUNC_PTR_OR_ERR_RET(adapter->ops->stats_reset, -EINVAL); return adapter->ops->stats_reset(adapter); } /* * Software event timer adapter buffer helper functions */ #define NSECPERSEC 1E9 /* Optimizations used to index into the buffer require that the buffer size * be a power of 2. */ #define EVENT_BUFFER_SZ 4096 #define EVENT_BUFFER_BATCHSZ 32 #define EVENT_BUFFER_MASK (EVENT_BUFFER_SZ - 1) struct event_buffer { uint16_t head; uint16_t tail; struct rte_event events[EVENT_BUFFER_SZ]; } __rte_cache_aligned; static inline bool event_buffer_full(struct event_buffer *bufp) { return (bufp->head - bufp->tail) == EVENT_BUFFER_SZ; } static inline bool event_buffer_batch_ready(struct event_buffer *bufp) { return (bufp->head - bufp->tail) >= EVENT_BUFFER_BATCHSZ; } static void event_buffer_init(struct event_buffer *bufp) { bufp->head = bufp->tail = 0; memset(&bufp->events, 0, sizeof(struct rte_event) * EVENT_BUFFER_SZ); } static int event_buffer_add(struct event_buffer *bufp, struct rte_event *eventp) { uint16_t head_idx; struct rte_event *buf_eventp; if (event_buffer_full(bufp)) return -1; /* Instead of modulus, bitwise AND with mask to get head_idx. */ head_idx = bufp->head & EVENT_BUFFER_MASK; buf_eventp = &bufp->events[head_idx]; rte_memcpy(buf_eventp, eventp, sizeof(struct rte_event)); /* Wrap automatically when overflow occurs. */ bufp->head++; return 0; } static void event_buffer_flush(struct event_buffer *bufp, uint8_t dev_id, uint8_t port_id, uint16_t *nb_events_flushed, uint16_t *nb_events_inv) { uint16_t head_idx, tail_idx, n = 0; struct rte_event *events = bufp->events; /* Instead of modulus, bitwise AND with mask to get index. */ head_idx = bufp->head & EVENT_BUFFER_MASK; tail_idx = bufp->tail & EVENT_BUFFER_MASK; /* Determine the largest contigous run we can attempt to enqueue to the * event device. */ if (head_idx > tail_idx) n = head_idx - tail_idx; else if (head_idx < tail_idx) n = EVENT_BUFFER_SZ - tail_idx; else { *nb_events_flushed = 0; return; } *nb_events_inv = 0; *nb_events_flushed = rte_event_enqueue_burst(dev_id, port_id, &events[tail_idx], n); if (*nb_events_flushed != n && rte_errno == EINVAL) { EVTIM_LOG_ERR("failed to enqueue invalid event - dropping it"); (*nb_events_inv)++; } bufp->tail = bufp->tail + *nb_events_flushed + *nb_events_inv; } /* * Software event timer adapter implementation */ struct rte_event_timer_adapter_sw_data { /* List of messages for outstanding timers */ TAILQ_HEAD(, msg) msgs_tailq_head; /* Lock to guard tailq and armed count */ rte_spinlock_t msgs_tailq_sl; /* Identifier of service executing timer management logic. */ uint32_t service_id; /* The cycle count at which the adapter should next tick */ uint64_t next_tick_cycles; /* Incremented as the service moves through phases of an iteration */ volatile int service_phase; /* The tick resolution used by adapter instance. May have been * adjusted from what user requested */ uint64_t timer_tick_ns; /* Maximum timeout in nanoseconds allowed by adapter instance. */ uint64_t max_tmo_ns; /* Ring containing messages to arm or cancel event timers */ struct rte_ring *msg_ring; /* Mempool containing msg objects */ struct rte_mempool *msg_pool; /* Buffered timer expiry events to be enqueued to an event device. */ struct event_buffer buffer; /* Statistics */ struct rte_event_timer_adapter_stats stats; /* The number of threads currently adding to the message ring */ rte_atomic16_t message_producer_count; }; enum msg_type {MSG_TYPE_ARM, MSG_TYPE_CANCEL}; struct msg { enum msg_type type; struct rte_event_timer *evtim; struct rte_timer tim; TAILQ_ENTRY(msg) msgs; }; static void sw_event_timer_cb(struct rte_timer *tim, void *arg) { int ret; uint16_t nb_evs_flushed = 0; uint16_t nb_evs_invalid = 0; uint64_t opaque; struct rte_event_timer *evtim; struct rte_event_timer_adapter *adapter; struct rte_event_timer_adapter_sw_data *sw_data; evtim = arg; opaque = evtim->impl_opaque[1]; adapter = (struct rte_event_timer_adapter *)(uintptr_t)opaque; sw_data = adapter->data->adapter_priv; ret = event_buffer_add(&sw_data->buffer, &evtim->ev); if (ret < 0) { /* If event buffer is full, put timer back in list with * immediate expiry value, so that we process it again on the * next iteration. */ rte_timer_reset_sync(tim, 0, SINGLE, rte_lcore_id(), sw_event_timer_cb, evtim); sw_data->stats.evtim_retry_count++; EVTIM_LOG_DBG("event buffer full, resetting rte_timer with " "immediate expiry value"); } else { struct msg *m = container_of(tim, struct msg, tim); TAILQ_REMOVE(&sw_data->msgs_tailq_head, m, msgs); EVTIM_BUF_LOG_DBG("buffered an event timer expiry event"); evtim->state = RTE_EVENT_TIMER_NOT_ARMED; /* Free the msg object containing the rte_timer now that * we've buffered its event successfully. */ rte_mempool_put(sw_data->msg_pool, m); /* Bump the count when we successfully add an expiry event to * the buffer. */ sw_data->stats.evtim_exp_count++; } if (event_buffer_batch_ready(&sw_data->buffer)) { event_buffer_flush(&sw_data->buffer, adapter->data->event_dev_id, adapter->data->event_port_id, &nb_evs_flushed, &nb_evs_invalid); sw_data->stats.ev_enq_count += nb_evs_flushed; sw_data->stats.ev_inv_count += nb_evs_invalid; } } static __rte_always_inline uint64_t get_timeout_cycles(struct rte_event_timer *evtim, struct rte_event_timer_adapter *adapter) { uint64_t timeout_ns; struct rte_event_timer_adapter_sw_data *sw_data; sw_data = adapter->data->adapter_priv; timeout_ns = evtim->timeout_ticks * sw_data->timer_tick_ns; return timeout_ns * rte_get_timer_hz() / NSECPERSEC; } /* This function returns true if one or more (adapter) ticks have occurred since * the last time it was called. */ static inline bool adapter_did_tick(struct rte_event_timer_adapter *adapter) { uint64_t cycles_per_adapter_tick, start_cycles; uint64_t *next_tick_cyclesp; struct rte_event_timer_adapter_sw_data *sw_data; sw_data = adapter->data->adapter_priv; next_tick_cyclesp = &sw_data->next_tick_cycles; cycles_per_adapter_tick = sw_data->timer_tick_ns * (rte_get_timer_hz() / NSECPERSEC); start_cycles = rte_get_timer_cycles(); /* Note: initially, *next_tick_cyclesp == 0, so the clause below will * execute, and set things going. */ if (start_cycles >= *next_tick_cyclesp) { /* Snap the current cycle count to the preceding adapter tick * boundary. */ start_cycles -= start_cycles % cycles_per_adapter_tick; *next_tick_cyclesp = start_cycles + cycles_per_adapter_tick; return true; } return false; } /* Check that event timer timeout value is in range */ static __rte_always_inline int check_timeout(struct rte_event_timer *evtim, const struct rte_event_timer_adapter *adapter) { uint64_t tmo_nsec; struct rte_event_timer_adapter_sw_data *sw_data; sw_data = adapter->data->adapter_priv; tmo_nsec = evtim->timeout_ticks * sw_data->timer_tick_ns; if (tmo_nsec > sw_data->max_tmo_ns) return -1; if (tmo_nsec < sw_data->timer_tick_ns) return -2; return 0; } /* Check that event timer event queue sched type matches destination event queue * sched type */ static __rte_always_inline int check_destination_event_queue(struct rte_event_timer *evtim, const struct rte_event_timer_adapter *adapter) { int ret; uint32_t sched_type; ret = rte_event_queue_attr_get(adapter->data->event_dev_id, evtim->ev.queue_id, RTE_EVENT_QUEUE_ATTR_SCHEDULE_TYPE, &sched_type); if ((ret < 0 && ret != -EOVERFLOW) || evtim->ev.sched_type != sched_type) return -1; return 0; } #define NB_OBJS 32 static int sw_event_timer_adapter_service_func(void *arg) { int i, num_msgs; uint64_t cycles, opaque; uint16_t nb_evs_flushed = 0; uint16_t nb_evs_invalid = 0; struct rte_event_timer_adapter *adapter; struct rte_event_timer_adapter_sw_data *sw_data; struct rte_event_timer *evtim = NULL; struct rte_timer *tim = NULL; struct msg *msg, *msgs[NB_OBJS]; adapter = arg; sw_data = adapter->data->adapter_priv; sw_data->service_phase = 1; rte_smp_wmb(); while (rte_atomic16_read(&sw_data->message_producer_count) > 0 || !rte_ring_empty(sw_data->msg_ring)) { num_msgs = rte_ring_dequeue_burst(sw_data->msg_ring, (void **)msgs, NB_OBJS, NULL); for (i = 0; i < num_msgs; i++) { int ret = 0; RTE_SET_USED(ret); msg = msgs[i]; evtim = msg->evtim; switch (msg->type) { case MSG_TYPE_ARM: EVTIM_SVC_LOG_DBG("dequeued ARM message from " "ring"); tim = &msg->tim; rte_timer_init(tim); cycles = get_timeout_cycles(evtim, adapter); ret = rte_timer_reset(tim, cycles, SINGLE, rte_lcore_id(), sw_event_timer_cb, evtim); RTE_ASSERT(ret == 0); evtim->impl_opaque[0] = (uintptr_t)tim; evtim->impl_opaque[1] = (uintptr_t)adapter; TAILQ_INSERT_TAIL(&sw_data->msgs_tailq_head, msg, msgs); break; case MSG_TYPE_CANCEL: EVTIM_SVC_LOG_DBG("dequeued CANCEL message " "from ring"); opaque = evtim->impl_opaque[0]; tim = (struct rte_timer *)(uintptr_t)opaque; RTE_ASSERT(tim != NULL); ret = rte_timer_stop(tim); RTE_ASSERT(ret == 0); /* Free the msg object for the original arm * request. */ struct msg *m; m = container_of(tim, struct msg, tim); TAILQ_REMOVE(&sw_data->msgs_tailq_head, m, msgs); rte_mempool_put(sw_data->msg_pool, m); /* Free the msg object for the current msg */ rte_mempool_put(sw_data->msg_pool, msg); evtim->impl_opaque[0] = 0; evtim->impl_opaque[1] = 0; break; } } } sw_data->service_phase = 2; rte_smp_wmb(); if (adapter_did_tick(adapter)) { rte_timer_manage(); event_buffer_flush(&sw_data->buffer, adapter->data->event_dev_id, adapter->data->event_port_id, &nb_evs_flushed, &nb_evs_invalid); sw_data->stats.ev_enq_count += nb_evs_flushed; sw_data->stats.ev_inv_count += nb_evs_invalid; sw_data->stats.adapter_tick_count++; } sw_data->service_phase = 0; rte_smp_wmb(); return 0; } /* The adapter initialization function rounds the mempool size up to the next * power of 2, so we can take the difference between that value and what the * user requested, and use the space for caches. This avoids a scenario where a * user can't arm the number of timers the adapter was configured with because * mempool objects have been lost to caches. * * nb_actual should always be a power of 2, so we can iterate over the powers * of 2 to see what the largest cache size we can use is. */ static int compute_msg_mempool_cache_size(uint64_t nb_requested, uint64_t nb_actual) { int i; int size; int cache_size = 0; for (i = 0; ; i++) { size = 1 << i; if (RTE_MAX_LCORE * size < (int)(nb_actual - nb_requested) && size < RTE_MEMPOOL_CACHE_MAX_SIZE && size <= nb_actual / 1.5) cache_size = size; else break; } return cache_size; } #define SW_MIN_INTERVAL 1E5 static int sw_event_timer_adapter_init(struct rte_event_timer_adapter *adapter) { int ret; struct rte_event_timer_adapter_sw_data *sw_data; uint64_t nb_timers; unsigned int flags; struct rte_service_spec service; static bool timer_subsystem_inited; // static initialized to false /* Allocate storage for SW implementation data */ char priv_data_name[RTE_RING_NAMESIZE]; snprintf(priv_data_name, RTE_RING_NAMESIZE, "sw_evtim_adap_priv_%"PRIu8, adapter->data->id); adapter->data->adapter_priv = rte_zmalloc_socket( priv_data_name, sizeof(struct rte_event_timer_adapter_sw_data), RTE_CACHE_LINE_SIZE, adapter->data->socket_id); if (adapter->data->adapter_priv == NULL) { EVTIM_LOG_ERR("failed to allocate space for private data"); rte_errno = ENOMEM; return -1; } if (adapter->data->conf.timer_tick_ns < SW_MIN_INTERVAL) { EVTIM_LOG_ERR("failed to create adapter with requested tick " "interval"); rte_errno = EINVAL; return -1; } sw_data = adapter->data->adapter_priv; sw_data->timer_tick_ns = adapter->data->conf.timer_tick_ns; sw_data->max_tmo_ns = adapter->data->conf.max_tmo_ns; TAILQ_INIT(&sw_data->msgs_tailq_head); rte_spinlock_init(&sw_data->msgs_tailq_sl); rte_atomic16_init(&sw_data->message_producer_count); /* Rings require power of 2, so round up to next such value */ nb_timers = rte_align64pow2(adapter->data->conf.nb_timers); char msg_ring_name[RTE_RING_NAMESIZE]; snprintf(msg_ring_name, RTE_RING_NAMESIZE, "sw_evtim_adap_msg_ring_%"PRIu8, adapter->data->id); flags = adapter->data->conf.flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT ? RING_F_SP_ENQ | RING_F_SC_DEQ : RING_F_SC_DEQ; sw_data->msg_ring = rte_ring_create(msg_ring_name, nb_timers, adapter->data->socket_id, flags); if (sw_data->msg_ring == NULL) { EVTIM_LOG_ERR("failed to create message ring"); rte_errno = ENOMEM; goto free_priv_data; } char pool_name[RTE_RING_NAMESIZE]; snprintf(pool_name, RTE_RING_NAMESIZE, "sw_evtim_adap_msg_pool_%"PRIu8, adapter->data->id); /* Both the arming/canceling thread and the service thread will do puts * to the mempool, but if the SP_PUT flag is enabled, we can specify * single-consumer get for the mempool. */ flags = adapter->data->conf.flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT ? MEMPOOL_F_SC_GET : 0; /* The usable size of a ring is count - 1, so subtract one here to * make the counts agree. */ int pool_size = nb_timers - 1; int cache_size = compute_msg_mempool_cache_size( adapter->data->conf.nb_timers, nb_timers); sw_data->msg_pool = rte_mempool_create(pool_name, pool_size, sizeof(struct msg), cache_size, 0, NULL, NULL, NULL, NULL, adapter->data->socket_id, flags); if (sw_data->msg_pool == NULL) { EVTIM_LOG_ERR("failed to create message object mempool"); rte_errno = ENOMEM; goto free_msg_ring; } event_buffer_init(&sw_data->buffer); /* Register a service component to run adapter logic */ memset(&service, 0, sizeof(service)); snprintf(service.name, RTE_SERVICE_NAME_MAX, "sw_evimer_adap_svc_%"PRIu8, adapter->data->id); service.socket_id = adapter->data->socket_id; service.callback = sw_event_timer_adapter_service_func; service.callback_userdata = adapter; service.capabilities &= ~(RTE_SERVICE_CAP_MT_SAFE); ret = rte_service_component_register(&service, &sw_data->service_id); if (ret < 0) { EVTIM_LOG_ERR("failed to register service %s with id %"PRIu32 ": err = %d", service.name, sw_data->service_id, ret); rte_errno = ENOSPC; goto free_msg_pool; } EVTIM_LOG_DBG("registered service %s with id %"PRIu32, service.name, sw_data->service_id); adapter->data->service_id = sw_data->service_id; adapter->data->service_inited = 1; if (!timer_subsystem_inited) { rte_timer_subsystem_init(); timer_subsystem_inited = true; } return 0; free_msg_pool: rte_mempool_free(sw_data->msg_pool); free_msg_ring: rte_ring_free(sw_data->msg_ring); free_priv_data: rte_free(sw_data); return -1; } static int sw_event_timer_adapter_uninit(struct rte_event_timer_adapter *adapter) { int ret; struct msg *m1, *m2; struct rte_event_timer_adapter_sw_data *sw_data = adapter->data->adapter_priv; rte_spinlock_lock(&sw_data->msgs_tailq_sl); /* Cancel outstanding rte_timers and free msg objects */ m1 = TAILQ_FIRST(&sw_data->msgs_tailq_head); while (m1 != NULL) { EVTIM_LOG_DBG("freeing outstanding timer"); m2 = TAILQ_NEXT(m1, msgs); rte_timer_stop_sync(&m1->tim); rte_mempool_put(sw_data->msg_pool, m1); m1 = m2; } rte_spinlock_unlock(&sw_data->msgs_tailq_sl); ret = rte_service_component_unregister(sw_data->service_id); if (ret < 0) { EVTIM_LOG_ERR("failed to unregister service component"); return ret; } rte_ring_free(sw_data->msg_ring); rte_mempool_free(sw_data->msg_pool); rte_free(adapter->data->adapter_priv); return 0; } static inline int32_t get_mapped_count_for_service(uint32_t service_id) { int32_t core_count, i, mapped_count = 0; uint32_t lcore_arr[RTE_MAX_LCORE]; core_count = rte_service_lcore_list(lcore_arr, RTE_MAX_LCORE); for (i = 0; i < core_count; i++) if (rte_service_map_lcore_get(service_id, lcore_arr[i]) == 1) mapped_count++; return mapped_count; } static int sw_event_timer_adapter_start(const struct rte_event_timer_adapter *adapter) { int mapped_count; struct rte_event_timer_adapter_sw_data *sw_data; sw_data = adapter->data->adapter_priv; /* Mapping the service to more than one service core can introduce * delays while one thread is waiting to acquire a lock, so only allow * one core to be mapped to the service. */ mapped_count = get_mapped_count_for_service(sw_data->service_id); if (mapped_count == 1) return rte_service_component_runstate_set(sw_data->service_id, 1); return mapped_count < 1 ? -ENOENT : -ENOTSUP; } static int sw_event_timer_adapter_stop(const struct rte_event_timer_adapter *adapter) { int ret; struct rte_event_timer_adapter_sw_data *sw_data = adapter->data->adapter_priv; ret = rte_service_component_runstate_set(sw_data->service_id, 0); if (ret < 0) return ret; /* Wait for the service to complete its final iteration before * stopping. */ while (sw_data->service_phase != 0) rte_pause(); rte_smp_rmb(); return 0; } static void sw_event_timer_adapter_get_info(const struct rte_event_timer_adapter *adapter, struct rte_event_timer_adapter_info *adapter_info) { struct rte_event_timer_adapter_sw_data *sw_data; sw_data = adapter->data->adapter_priv; adapter_info->min_resolution_ns = sw_data->timer_tick_ns; adapter_info->max_tmo_ns = sw_data->max_tmo_ns; } static int sw_event_timer_adapter_stats_get(const struct rte_event_timer_adapter *adapter, struct rte_event_timer_adapter_stats *stats) { struct rte_event_timer_adapter_sw_data *sw_data; sw_data = adapter->data->adapter_priv; *stats = sw_data->stats; return 0; } static int sw_event_timer_adapter_stats_reset( const struct rte_event_timer_adapter *adapter) { struct rte_event_timer_adapter_sw_data *sw_data; sw_data = adapter->data->adapter_priv; memset(&sw_data->stats, 0, sizeof(sw_data->stats)); return 0; } static __rte_always_inline uint16_t __sw_event_timer_arm_burst(const struct rte_event_timer_adapter *adapter, struct rte_event_timer **evtims, uint16_t nb_evtims) { uint16_t i; int ret; struct rte_event_timer_adapter_sw_data *sw_data; struct msg *msgs[nb_evtims]; #ifdef RTE_LIBRTE_EVENTDEV_DEBUG /* Check that the service is running. */ if (rte_service_runstate_get(adapter->data->service_id) != 1) { rte_errno = EINVAL; return 0; } #endif sw_data = adapter->data->adapter_priv; ret = rte_mempool_get_bulk(sw_data->msg_pool, (void **)msgs, nb_evtims); if (ret < 0) { rte_errno = ENOSPC; return 0; } /* Let the service know we're producing messages for it to process */ rte_atomic16_inc(&sw_data->message_producer_count); /* If the service is managing timers, wait for it to finish */ while (sw_data->service_phase == 2) rte_pause(); rte_smp_rmb(); for (i = 0; i < nb_evtims; i++) { /* Don't modify the event timer state in these cases */ if (evtims[i]->state == RTE_EVENT_TIMER_ARMED) { rte_errno = EALREADY; break; } else if (!(evtims[i]->state == RTE_EVENT_TIMER_NOT_ARMED || evtims[i]->state == RTE_EVENT_TIMER_CANCELED)) { rte_errno = EINVAL; break; } ret = check_timeout(evtims[i], adapter); if (ret == -1) { evtims[i]->state = RTE_EVENT_TIMER_ERROR_TOOLATE; rte_errno = EINVAL; break; } if (ret == -2) { evtims[i]->state = RTE_EVENT_TIMER_ERROR_TOOEARLY; rte_errno = EINVAL; break; } if (check_destination_event_queue(evtims[i], adapter) < 0) { evtims[i]->state = RTE_EVENT_TIMER_ERROR; rte_errno = EINVAL; break; } /* Checks passed, set up a message to enqueue */ msgs[i]->type = MSG_TYPE_ARM; msgs[i]->evtim = evtims[i]; /* Set the payload pointer if not set. */ if (evtims[i]->ev.event_ptr == NULL) evtims[i]->ev.event_ptr = evtims[i]; /* msg objects that get enqueued successfully will be freed * either by a future cancel operation or by the timer * expiration callback. */ if (rte_ring_enqueue(sw_data->msg_ring, msgs[i]) < 0) { rte_errno = ENOSPC; break; } EVTIM_LOG_DBG("enqueued ARM message to ring"); evtims[i]->state = RTE_EVENT_TIMER_ARMED; } /* Let the service know we're done producing messages */ rte_atomic16_dec(&sw_data->message_producer_count); if (i < nb_evtims) rte_mempool_put_bulk(sw_data->msg_pool, (void **)&msgs[i], nb_evtims - i); return i; } static uint16_t sw_event_timer_arm_burst(const struct rte_event_timer_adapter *adapter, struct rte_event_timer **evtims, uint16_t nb_evtims) { return __sw_event_timer_arm_burst(adapter, evtims, nb_evtims); } static uint16_t sw_event_timer_cancel_burst(const struct rte_event_timer_adapter *adapter, struct rte_event_timer **evtims, uint16_t nb_evtims) { uint16_t i; int ret; struct rte_event_timer_adapter_sw_data *sw_data; struct msg *msgs[nb_evtims]; #ifdef RTE_LIBRTE_EVENTDEV_DEBUG /* Check that the service is running. */ if (rte_service_runstate_get(adapter->data->service_id) != 1) { rte_errno = EINVAL; return 0; } #endif sw_data = adapter->data->adapter_priv; ret = rte_mempool_get_bulk(sw_data->msg_pool, (void **)msgs, nb_evtims); if (ret < 0) { rte_errno = ENOSPC; return 0; } /* Let the service know we're producing messages for it to process */ rte_atomic16_inc(&sw_data->message_producer_count); /* If the service could be modifying event timer states, wait */ while (sw_data->service_phase == 2) rte_pause(); rte_smp_rmb(); for (i = 0; i < nb_evtims; i++) { /* Don't modify the event timer state in these cases */ if (evtims[i]->state == RTE_EVENT_TIMER_CANCELED) { rte_errno = EALREADY; break; } else if (evtims[i]->state != RTE_EVENT_TIMER_ARMED) { rte_errno = EINVAL; break; } msgs[i]->type = MSG_TYPE_CANCEL; msgs[i]->evtim = evtims[i]; if (rte_ring_enqueue(sw_data->msg_ring, msgs[i]) < 0) { rte_errno = ENOSPC; break; } EVTIM_LOG_DBG("enqueued CANCEL message to ring"); evtims[i]->state = RTE_EVENT_TIMER_CANCELED; } /* Let the service know we're done producing messages */ rte_atomic16_dec(&sw_data->message_producer_count); if (i < nb_evtims) rte_mempool_put_bulk(sw_data->msg_pool, (void **)&msgs[i], nb_evtims - i); return i; } static uint16_t sw_event_timer_arm_tmo_tick_burst(const struct rte_event_timer_adapter *adapter, struct rte_event_timer **evtims, uint64_t timeout_ticks, uint16_t nb_evtims) { int i; for (i = 0; i < nb_evtims; i++) evtims[i]->timeout_ticks = timeout_ticks; return __sw_event_timer_arm_burst(adapter, evtims, nb_evtims); } static const struct rte_event_timer_adapter_ops sw_event_adapter_timer_ops = { .init = sw_event_timer_adapter_init, .uninit = sw_event_timer_adapter_uninit, .start = sw_event_timer_adapter_start, .stop = sw_event_timer_adapter_stop, .get_info = sw_event_timer_adapter_get_info, .stats_get = sw_event_timer_adapter_stats_get, .stats_reset = sw_event_timer_adapter_stats_reset, .arm_burst = sw_event_timer_arm_burst, .arm_tmo_tick_burst = sw_event_timer_arm_tmo_tick_burst, .cancel_burst = sw_event_timer_cancel_burst, }; RTE_INIT(event_timer_adapter_init_log) { evtim_logtype = rte_log_register("lib.eventdev.adapter.timer"); if (evtim_logtype >= 0) rte_log_set_level(evtim_logtype, RTE_LOG_NOTICE); evtim_buffer_logtype = rte_log_register("lib.eventdev.adapter.timer." "buffer"); if (evtim_buffer_logtype >= 0) rte_log_set_level(evtim_buffer_logtype, RTE_LOG_NOTICE); evtim_svc_logtype = rte_log_register("lib.eventdev.adapter.timer.svc"); if (evtim_svc_logtype >= 0) rte_log_set_level(evtim_svc_logtype, RTE_LOG_NOTICE); }