rcutorture: Check for immediate deboosting at reader end This commit adds a check for failure to have fully deboosted a multi-segmented RCU reader at the end of the full read-side critical section. This check only happens for fully task-level readers, because a reader in a handler might have interrupted an already-boosted task-level RCU reader, which would result in false positives. Although most uses of RCU priority boosting serve as debugging aids, this might change, and in fact might already have changed. And allowing (for example) RCU priority boosting to persist until the next scheduler tick could cause an aggressively real-time system to miss sub-millisecond deadlines. So we do need to find this sort of problem during testing, and preferably not in the field. The name and type of the rcu_torture_ops function pointer may need to change should other end-of-reader checks be needed. But let's start simple. Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa6d30c..14faa11 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h
@@ -695,4 +695,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re void synchronize_rcu_trivial_preempt(void); #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU +#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +bool rcu_is_task_rcu_boosted(void); +#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +static inline bool rcu_is_task_rcu_boosted(void) { return false; } +#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) + + #endif /* __LINUX_RCU_H */
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 3c8e4cd..bb5ab86 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c
@@ -426,6 +426,7 @@ struct rcu_torture_ops { void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len); void (*set_gpwrap_lag)(unsigned long lag); int (*get_gpwrap_count)(int cpu); + bool (*is_task_rcu_boosted)(void); long cbflood_max; int irq_capable; int can_boost; @@ -635,6 +636,7 @@ static struct rcu_torture_ops rcu_ops = { .format_gp_seqs = rcutorture_format_gp_seqs, .set_gpwrap_lag = rcu_set_gpwrap_lag, .get_gpwrap_count = rcu_get_gpwrap_count, + .is_task_rcu_boosted = rcu_is_task_rcu_boosted, .irq_capable = 1, .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), .extendables = RCUTORTURE_MAX_EXTEND, @@ -2518,6 +2520,7 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) return false; rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); rcu_torture_one_read_end(&rtors, trsp); + WARN_ON_ONCE(cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted()); return true; }
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967..c5dbdf8 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h
@@ -923,7 +923,7 @@ void rcu_read_unlock_strict(void) * * The in_atomic_preempt_off() check ensures that we come here holding * the last preempt_count (which will get dropped once we return to - * __rcu_read_unlock(). + * __rcu_read_unlock()). */ rdp = this_cpu_ptr(&rcu_data); rdp->cpu_no_qs.b.norm = false; @@ -1320,6 +1320,36 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ } +#ifdef CONFIG_RCU_TORTURE_TEST + +/* + * Is the current task RCU priority boosted? This is used by + * rcutorture to check that tasks are always deboosted once then exit + * an RCU read-side critical section, no matter how many overlapping + * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(), + * or local_irq_disable() made up that reader. + * + * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex) + * are safe because tasks release ->boost_mtx when they own it, they + * cannot be boosted unless current->rcu_blocked_node is non-NULL, + * current->rcu_blocked_node is modified only by the current task, + * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner + * switching among other tasks cannot force an equality comparison. + */ +bool rcu_is_task_rcu_boosted(void) +{ + struct rcu_node *rnp; + struct task_struct *t = current; + + rnp = t->rcu_blocked_node; + if (!rnp) + return false; + return rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t; +} +EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted); + +#endif // #ifdef CONFIG_RCU_TORTURE_TEST + #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)