rcu: Provide diagnostic option to slow down grace-period scans
Grace-period scans of the rcu_node combining tree normally
proceed quite quickly, so that it is very difficult to reproduce
races against them. This commit therefore allows grace-period
pre-initialization and cleanup to be artificially slowed down,
increasing race-reproduction probability. A pair of pairs of new
Kconfig parameters are provided, RCU_TORTURE_TEST_SLOW_PREINIT to
enable the slowing down of propagating CPU-hotplug changes up the
combining tree along with RCU_TORTURE_TEST_SLOW_PREINIT_DELAY to
specify the delay in jiffies, and RCU_TORTURE_TEST_SLOW_CLEANUP
to enable the slowing down of the end-of-grace-period cleanup scan
along with RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY to specify the delay
in jiffies. Boot-time parameters named rcutree.gp_preinit_delay and
rcutree.gp_cleanup_delay allow these delays to be specified at boot time.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index 5368ba7..64528a2 100644
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -2968,11 +2968,23 @@
Set maximum number of finished RCU callbacks to
process in one batch.
+ rcutree.gp_cleanup_delay= [KNL]
+ Set the number of jiffies to delay each step of
+ RCU grace-period cleanup. This only has effect
+ when CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP is set.
+
rcutree.gp_init_delay= [KNL]
Set the number of jiffies to delay each step of
RCU grace-period initialization. This only has
- effect when CONFIG_RCU_TORTURE_TEST_SLOW_INIT is
- set.
+ effect when CONFIG_RCU_TORTURE_TEST_SLOW_INIT
+ is set.
+
+ rcutree.gp_preinit_delay= [KNL]
+ Set the number of jiffies to delay each step of
+ RCU grace-period pre-initialization, that is,
+ the propagation of recent CPU-hotplug changes up
+ the rcu_node combining tree. This only has effect
+ when CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT is set.
rcutree.rcu_fanout_leaf= [KNL]
Increase the number of CPUs assigned to each
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index ff7349d..664b052 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -158,11 +158,19 @@
static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO;
module_param(kthread_prio, int, 0644);
-/* Delay in jiffies for grace-period initialization delays. */
+/* Delay in jiffies for grace-period rcu_node-loop delays. */
+static int gp_preinit_delay = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT)
+ ? CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT_DELAY
+ : 0;
+module_param(gp_preinit_delay, int, 0644);
static int gp_init_delay = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_INIT)
? CONFIG_RCU_TORTURE_TEST_SLOW_INIT_DELAY
: 0;
module_param(gp_init_delay, int, 0644);
+static int gp_cleanup_delay = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP)
+ ? CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY
+ : 0;
+module_param(gp_cleanup_delay, int, 0644);
/*
* Track the rcutorture test sequence number and the update version
@@ -1725,6 +1733,12 @@
rcu_gp_kthread_wake(rsp);
}
+static void rcu_gp_slow(int enable, struct rcu_state *rsp, int delay)
+{
+ if (enable && delay > 0 && !(rsp->gpnum % (rcu_num_nodes * 3 * delay)))
+ schedule_timeout_uninterruptible(delay);
+}
+
/*
* Initialize a new grace period. Return 0 if no grace period required.
*/
@@ -1767,6 +1781,8 @@
* will handle subsequent offline CPUs.
*/
rcu_for_each_leaf_node(rsp, rnp) {
+ rcu_gp_slow(IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT),
+ rsp, gp_preinit_delay);
raw_spin_lock_irq(&rnp->lock);
smp_mb__after_unlock_lock();
if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
@@ -1823,6 +1839,8 @@
* process finishes, because this kthread handles both.
*/
rcu_for_each_node_breadth_first(rsp, rnp) {
+ rcu_gp_slow(IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_INIT),
+ rsp, gp_init_delay);
raw_spin_lock_irq(&rnp->lock);
smp_mb__after_unlock_lock();
rdp = this_cpu_ptr(rsp->rda);
@@ -1840,10 +1858,6 @@
raw_spin_unlock_irq(&rnp->lock);
cond_resched_rcu_qs();
WRITE_ONCE(rsp->gp_activity, jiffies);
- if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_INIT) &&
- gp_init_delay > 0 &&
- !(rsp->gpnum % (rcu_num_nodes * 3 * gp_init_delay)))
- schedule_timeout_uninterruptible(gp_init_delay);
}
return 1;
@@ -1938,6 +1952,8 @@
raw_spin_unlock_irq(&rnp->lock);
cond_resched_rcu_qs();
WRITE_ONCE(rsp->gp_activity, jiffies);
+ rcu_gp_slow(IS_ENABLED(CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP),
+ rsp, gp_cleanup_delay);
}
rnp = rcu_get_root(rsp);
raw_spin_lock_irq(&rnp->lock);
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 1ad74c0..984acf3 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1248,12 +1248,37 @@
Say N here if you want the RCU torture tests to start only
after being manually enabled via /proc.
+config RCU_TORTURE_TEST_SLOW_PREINIT
+ bool "Slow down RCU grace-period pre-initialization to expose races"
+ depends on RCU_TORTURE_TEST
+ help
+ This option delays grace-period pre-initialization (the
+ propagation of CPU-hotplug changes up the rcu_node combining
+ tree) for a few jiffies between initializing each pair of
+ consecutive rcu_node structures. This helps to expose races
+ involving grace-period pre-initialization, in other words, it
+ makes your kernel less stable. It can also greatly increase
+ grace-period latency, especially on systems with large numbers
+ of CPUs. This is useful when torture-testing RCU, but in
+ almost no other circumstance.
+
+ Say Y here if you want your system to crash and hang more often.
+ Say N if you want a sane system.
+
+config RCU_TORTURE_TEST_SLOW_PREINIT_DELAY
+ int "How much to slow down RCU grace-period pre-initialization"
+ range 0 5
+ default 3
+ help
+ This option specifies the number of jiffies to wait between
+ each rcu_node structure pre-initialization step.
+
config RCU_TORTURE_TEST_SLOW_INIT
bool "Slow down RCU grace-period initialization to expose races"
depends on RCU_TORTURE_TEST
help
- This option makes grace-period initialization block for a
- few jiffies between initializing each pair of consecutive
+ This option delays grace-period initialization for a few
+ jiffies between initializing each pair of consecutive
rcu_node structures. This helps to expose races involving
grace-period initialization, in other words, it makes your
kernel less stable. It can also greatly increase grace-period
@@ -1272,6 +1297,29 @@
This option specifies the number of jiffies to wait between
each rcu_node structure initialization.
+config RCU_TORTURE_TEST_SLOW_CLEANUP
+ bool "Slow down RCU grace-period cleanup to expose races"
+ depends on RCU_TORTURE_TEST
+ help
+ This option delays grace-period cleanup for a few jiffies
+ between cleaning up each pair of consecutive rcu_node
+ structures. This helps to expose races involving grace-period
+ cleanup, in other words, it makes your kernel less stable.
+ It can also greatly increase grace-period latency, especially
+ on systems with large numbers of CPUs. This is useful when
+ torture-testing RCU, but in almost no other circumstance.
+
+ Say Y here if you want your system to crash and hang more often.
+ Say N if you want a sane system.
+
+config RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY
+ int "How much to slow down RCU grace-period cleanup"
+ range 0 5
+ default 3
+ help
+ This option specifies the number of jiffies to wait between
+ each rcu_node structure cleanup operation.
+
config RCU_CPU_STALL_TIMEOUT
int "RCU CPU stall timeout in seconds"
depends on RCU_STALL_COMMON
diff --git a/tools/testing/selftests/rcutorture/configs/rcu/CFcommon b/tools/testing/selftests/rcutorture/configs/rcu/CFcommon
index 4970121..f824b4c 100644
--- a/tools/testing/selftests/rcutorture/configs/rcu/CFcommon
+++ b/tools/testing/selftests/rcutorture/configs/rcu/CFcommon
@@ -1,3 +1,5 @@
CONFIG_RCU_TORTURE_TEST=y
CONFIG_PRINTK_TIME=y
+CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP=y
CONFIG_RCU_TORTURE_TEST_SLOW_INIT=y
+CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT=y
