5 #ifndef _RTE_RCU_QSBR_H_ 6 #define _RTE_RCU_QSBR_H_ 44 extern int rte_rcu_log_type;
46 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG 47 #define __RTE_RCU_DP_LOG(level, fmt, args...) \ 48 rte_log(RTE_LOG_ ## level, rte_rcu_log_type, \ 49 "%s(): " fmt "\n", __func__, ## args) 51 #define __RTE_RCU_DP_LOG(level, fmt, args...) 54 #if defined(RTE_LIBRTE_RCU_DEBUG) 55 #define __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, level, fmt, args...) do {\ 56 if (v->qsbr_cnt[thread_id].lock_cnt) \ 57 rte_log(RTE_LOG_ ## level, rte_rcu_log_type, \ 58 "%s(): " fmt "\n", __func__, ## args); \ 61 #define __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, level, fmt, args...) 68 #define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(uint64_t) * 8) 69 #define __RTE_QSBR_THRID_ARRAY_SIZE(max_threads) \ 70 RTE_ALIGN(RTE_ALIGN_MUL_CEIL(max_threads, \ 71 __RTE_QSBR_THRID_ARRAY_ELM_SIZE) >> 3, RTE_CACHE_LINE_SIZE) 72 #define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t *) \ 73 ((struct rte_rcu_qsbr_cnt *)(v + 1) + v->max_threads) + i) 74 #define __RTE_QSBR_THRID_INDEX_SHIFT 6 75 #define __RTE_QSBR_THRID_MASK 0x3f 76 #define RTE_QSBR_THRID_INVALID 0xffffffff 79 struct rte_rcu_qsbr_cnt {
90 #define __RTE_QSBR_CNT_THR_OFFLINE 0 91 #define __RTE_QSBR_CNT_INIT 1 92 #define __RTE_QSBR_CNT_MAX ((uint64_t)~0) 93 #define __RTE_QSBR_TOKEN_SIZE sizeof(uint64_t) 101 struct rte_rcu_qsbr {
104 uint64_t acked_token;
111 uint32_t num_threads;
113 uint32_t max_threads;
139 #define RTE_RCU_QSBR_DQ_NAMESIZE RTE_RING_NAMESIZE 149 #define RTE_RCU_QSBR_DQ_MT_UNSAFE 1 193 struct rte_rcu_qsbr *
v;
202 struct rte_rcu_qsbr_dq;
307 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
309 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR,
"Lock counter %u\n",
310 v->qsbr_cnt[thread_id].lock_cnt);
317 t = __atomic_load_n(&v->token, __ATOMIC_RELAXED);
322 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
323 t, __ATOMIC_RELAXED);
332 #ifdef RTE_ARCH_X86_64 336 __atomic_thread_fence(__ATOMIC_SEQ_CST);
363 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
365 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR,
"Lock counter %u\n",
366 v->qsbr_cnt[thread_id].lock_cnt);
373 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
374 __RTE_QSBR_CNT_THR_OFFLINE, __ATOMIC_RELEASE);
401 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
403 #if defined(RTE_LIBRTE_RCU_DEBUG) 405 __atomic_fetch_add(&v->qsbr_cnt[thread_id].lock_cnt,
406 1, __ATOMIC_ACQUIRE);
434 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
436 #if defined(RTE_LIBRTE_RCU_DEBUG) 438 __atomic_fetch_sub(&v->qsbr_cnt[thread_id].lock_cnt,
439 1, __ATOMIC_RELEASE);
441 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, WARNING,
442 "Lock counter %u. Nested locks?\n",
443 v->qsbr_cnt[thread_id].lock_cnt);
465 RTE_ASSERT(v != NULL);
472 t = __atomic_add_fetch(&v->token, 1, __ATOMIC_RELEASE);
494 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
496 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR,
"Lock counter %u\n",
497 v->qsbr_cnt[thread_id].lock_cnt);
504 t = __atomic_load_n(&v->token, __ATOMIC_ACQUIRE);
511 if (t != __atomic_load_n(&v->qsbr_cnt[thread_id].cnt, __ATOMIC_RELAXED))
512 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
513 t, __ATOMIC_RELEASE);
515 __RTE_RCU_DP_LOG(DEBUG,
"%s: update: token = %" PRIu64
", Thread ID = %d",
516 __func__, t, thread_id);
523 __rte_rcu_qsbr_check_selective(
struct rte_rcu_qsbr *v, uint64_t t,
bool wait)
528 uint64_t *reg_thread_id;
529 uint64_t acked_token = __RTE_QSBR_CNT_MAX;
531 for (i = 0, reg_thread_id = __RTE_QSBR_THRID_ARRAY_ELM(v, 0);
533 i++, reg_thread_id++) {
537 bmap = __atomic_load_n(reg_thread_id, __ATOMIC_ACQUIRE);
538 id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
541 j = __builtin_ctzl(bmap);
542 __RTE_RCU_DP_LOG(DEBUG,
543 "%s: check: token = %" PRIu64
", wait = %d, Bit Map = 0x%" PRIx64
", Thread ID = %d",
544 __func__, t, wait, bmap,
id + j);
546 &v->qsbr_cnt[
id + j].cnt,
548 __RTE_RCU_DP_LOG(DEBUG,
549 "%s: status: token = %" PRIu64
", wait = %d, Thread QS cnt = %" PRIu64
", Thread ID = %d",
550 __func__, t, wait, c,
id+j);
556 __RTE_QSBR_CNT_THR_OFFLINE && c < t)) {
565 bmap = __atomic_load_n(reg_thread_id,
575 if (c != __RTE_QSBR_CNT_THR_OFFLINE && acked_token > c)
586 if (acked_token != __RTE_QSBR_CNT_MAX)
587 __atomic_store_n(&v->acked_token, acked_token,
597 __rte_rcu_qsbr_check_all(
struct rte_rcu_qsbr *v, uint64_t t,
bool wait)
600 struct rte_rcu_qsbr_cnt *cnt;
602 uint64_t acked_token = __RTE_QSBR_CNT_MAX;
604 for (i = 0, cnt = v->qsbr_cnt; i < v->max_threads; i++, cnt++) {
605 __RTE_RCU_DP_LOG(DEBUG,
606 "%s: check: token = %" PRIu64
", wait = %d, Thread ID = %d",
607 __func__, t, wait, i);
609 c = __atomic_load_n(&cnt->cnt, __ATOMIC_ACQUIRE);
610 __RTE_RCU_DP_LOG(DEBUG,
611 "%s: status: token = %" PRIu64
", wait = %d, Thread QS cnt = %" PRIu64
", Thread ID = %d",
612 __func__, t, wait, c, i);
617 if (
likely(c == __RTE_QSBR_CNT_THR_OFFLINE || c >= t))
630 if (
likely(c != __RTE_QSBR_CNT_THR_OFFLINE && acked_token > c))
638 if (acked_token != __RTE_QSBR_CNT_MAX)
639 __atomic_store_n(&v->acked_token, acked_token,
679 RTE_ASSERT(v != NULL);
682 if (
likely(t <= v->acked_token)) {
683 __RTE_RCU_DP_LOG(DEBUG,
684 "%s: check: token = %" PRIu64
", wait = %d",
686 __RTE_RCU_DP_LOG(DEBUG,
687 "%s: status: least acked token = %" PRIu64,
688 __func__, v->acked_token);
692 if (
likely(v->num_threads == v->max_threads))
693 return __rte_rcu_qsbr_check_all(v, t, wait);
695 return __rte_rcu_qsbr_check_selective(v, t, wait);
754 struct rte_rcu_qsbr_dq *
820 unsigned int *freed,
unsigned int *pending,
unsigned int *available);
int rte_rcu_qsbr_thread_register(struct rte_rcu_qsbr *v, unsigned int thread_id)
#define __rte_always_inline
__rte_experimental int rte_rcu_qsbr_dq_reclaim(struct rte_rcu_qsbr_dq *dq, unsigned int n, unsigned int *freed, unsigned int *pending, unsigned int *available)
__rte_experimental int rte_rcu_qsbr_dq_delete(struct rte_rcu_qsbr_dq *dq)
int rte_rcu_qsbr_init(struct rte_rcu_qsbr *v, uint32_t max_threads)
static __rte_always_inline int rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait)
static __rte_always_inline void rte_rcu_qsbr_quiescent(struct rte_rcu_qsbr *v, unsigned int thread_id)
static __rte_always_inline void rte_rcu_qsbr_thread_offline(struct rte_rcu_qsbr *v, unsigned int thread_id)
int rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v)
static __rte_always_inline void rte_rcu_qsbr_unlock(__rte_unused struct rte_rcu_qsbr *v, __rte_unused unsigned int thread_id)
void(* rte_rcu_qsbr_free_resource_t)(void *p, void *e, unsigned int n)
rte_rcu_qsbr_free_resource_t free_fn
static void rte_pause(void)
uint32_t trigger_reclaim_limit
uint32_t max_reclaim_size
#define __rte_cache_aligned
static __rte_always_inline void rte_rcu_qsbr_thread_online(struct rte_rcu_qsbr *v, unsigned int thread_id)
static __rte_always_inline void rte_rcu_qsbr_lock(__rte_unused struct rte_rcu_qsbr *v, __rte_unused unsigned int thread_id)
static void rte_smp_mb(void)
__rte_experimental struct rte_rcu_qsbr_dq * rte_rcu_qsbr_dq_create(const struct rte_rcu_qsbr_dq_parameters *params)
static __rte_always_inline uint64_t rte_rcu_qsbr_start(struct rte_rcu_qsbr *v)
int rte_rcu_qsbr_thread_unregister(struct rte_rcu_qsbr *v, unsigned int thread_id)
size_t rte_rcu_qsbr_get_memsize(uint32_t max_threads)
__rte_experimental int rte_rcu_qsbr_dq_enqueue(struct rte_rcu_qsbr_dq *dq, void *e)
void rte_rcu_qsbr_synchronize(struct rte_rcu_qsbr *v, unsigned int thread_id)