rcutorture: Break up too-long rcu_torture_fwd_prog() function
This commit splits rcu_torture_fwd_prog_nr() and rcu_torture_fwd_prog_cr()
functions out of rcu_torture_fwd_prog() in order to reduce indentation
pain and because rcu_torture_fwd_prog() was getting a bit too long.
In addition, this will enable easier conditional execution of the
rcu_torture_fwd_prog_cr() function, which can give false-positive
failures in some NO_HZ_FULL configurations due to overloading the
housekeeping CPUs.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 9dc4059..278a134 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -1629,8 +1629,8 @@
#define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */
static long n_launders_hist[2 * MAX_FWD_CB_JIFFIES / HZ];
-/* Callback function for continuous-flood RCU callbacks. */
static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
+/* Callback function for continuous-flood RCU callbacks. */
{
int i;
struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh);
@@ -1650,15 +1650,70 @@
spin_unlock(&rcu_fwd_lock);
}
-/* Carry out grace-period forward-progress testing. */
-static int rcu_torture_fwd_prog(void *args)
+/* Carry out need_resched()/cond_resched() forward-progress testing. */
+static void rcu_torture_fwd_prog_nr(int *tested, int *tested_tries)
{
unsigned long cver;
unsigned long dur;
struct fwd_cb_state fcs;
unsigned long gps;
- int i;
int idx;
+ int sd;
+ int sd4;
+ bool selfpropcb = false;
+ unsigned long stopat;
+ static DEFINE_TORTURE_RANDOM(trs);
+
+ if (cur_ops->call && cur_ops->sync && cur_ops->cb_barrier) {
+ init_rcu_head_on_stack(&fcs.rh);
+ selfpropcb = true;
+ }
+
+ /* Tight loop containing cond_resched(). */
+ if (selfpropcb) {
+ WRITE_ONCE(fcs.stop, 0);
+ cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);
+ }
+ cver = READ_ONCE(rcu_torture_current_version);
+ gps = cur_ops->get_gp_seq();
+ sd = cur_ops->stall_dur() + 1;
+ sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div;
+ dur = sd4 + torture_random(&trs) % (sd - sd4);
+ rcu_fwd_startat = jiffies;
+ stopat = rcu_fwd_startat + dur;
+ while (time_before(jiffies, stopat) && !torture_must_stop()) {
+ idx = cur_ops->readlock();
+ udelay(10);
+ cur_ops->readunlock(idx);
+ if (!fwd_progress_need_resched || need_resched())
+ cond_resched();
+ }
+ (*tested_tries)++;
+ if (!time_before(jiffies, stopat) && !torture_must_stop()) {
+ (*tested)++;
+ cver = READ_ONCE(rcu_torture_current_version) - cver;
+ gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
+ WARN_ON(!cver && gps < 2);
+ pr_alert("%s: Duration %ld cver %ld gps %ld\n", __func__, dur, cver, gps);
+ }
+ if (selfpropcb) {
+ WRITE_ONCE(fcs.stop, 1);
+ cur_ops->sync(); /* Wait for running CB to complete. */
+ cur_ops->cb_barrier(); /* Wait for queued callbacks. */
+ }
+
+ if (selfpropcb) {
+ WARN_ON(READ_ONCE(fcs.stop) != 2);
+ destroy_rcu_head_on_stack(&fcs.rh);
+ }
+}
+
+/* Carry out call_rcu() forward-progress testing. */
+static void rcu_torture_fwd_prog_cr(void)
+{
+ unsigned long cver;
+ unsigned long gps;
+ int i;
int j;
long n_launders;
long n_launders_cb_snap;
@@ -1667,136 +1722,97 @@
long n_max_gps;
struct rcu_fwd_cb *rfcp;
struct rcu_fwd_cb *rfcpn;
- int sd;
- int sd4;
- bool selfpropcb = false;
unsigned long stopat;
unsigned long stoppedat;
+
+ /* Loop continuously posting RCU callbacks. */
+ WRITE_ONCE(rcu_fwd_cb_nodelay, true);
+ cur_ops->sync(); /* Later readers see above write. */
+ rcu_fwd_startat = jiffies;
+ stopat = rcu_fwd_startat + MAX_FWD_CB_JIFFIES;
+ n_launders = 0;
+ n_launders_cb = 0;
+ n_launders_sa = 0;
+ n_max_cbs = 0;
+ n_max_gps = 0;
+ for (i = 0; i < ARRAY_SIZE(n_launders_hist); i++)
+ n_launders_hist[i] = 0;
+ cver = READ_ONCE(rcu_torture_current_version);
+ gps = cur_ops->get_gp_seq();
+ while (time_before(jiffies, stopat) && !torture_must_stop()) {
+ rfcp = READ_ONCE(rcu_fwd_cb_head);
+ rfcpn = NULL;
+ if (rfcp)
+ rfcpn = READ_ONCE(rfcp->rfc_next);
+ if (rfcpn) {
+ if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS &&
+ ++n_max_gps >= MIN_FWD_CBS_LAUNDERED)
+ break;
+ rcu_fwd_cb_head = rfcpn;
+ n_launders++;
+ n_launders_sa++;
+ } else {
+ rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL);
+ if (WARN_ON_ONCE(!rfcp)) {
+ schedule_timeout_interruptible(1);
+ continue;
+ }
+ n_max_cbs++;
+ n_launders_sa = 0;
+ rfcp->rfc_gps = 0;
+ }
+ cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
+ cond_resched();
+ }
+ stoppedat = jiffies;
+ n_launders_cb_snap = READ_ONCE(n_launders_cb);
+ cver = READ_ONCE(rcu_torture_current_version) - cver;
+ gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
+ cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */
+ for (;;) {
+ rfcp = rcu_fwd_cb_head;
+ if (!rfcp)
+ break;
+ rcu_fwd_cb_head = rfcp->rfc_next;
+ kfree(rfcp);
+ }
+ rcu_fwd_cb_tail = &rcu_fwd_cb_head;
+ WRITE_ONCE(rcu_fwd_cb_nodelay, false);
+ if (!torture_must_stop()) {
+ WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED);
+ pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n",
+ __func__,
+ stoppedat - rcu_fwd_startat, jiffies - stoppedat,
+ n_launders + n_max_cbs - n_launders_cb_snap,
+ n_launders, n_launders_sa,
+ n_max_gps, n_max_cbs, cver, gps);
+ for (i = ARRAY_SIZE(n_launders_hist) - 1; i > 0; i--)
+ if (n_launders_hist[i] > 0)
+ break;
+ pr_alert("Callback-invocation histogram:");
+ for (j = 0; j <= i; j++)
+ pr_cont(" %ds: %ld", j + 1, n_launders_hist[j]);
+ pr_cont("\n");
+ }
+}
+
+/* Carry out grace-period forward-progress testing. */
+static int rcu_torture_fwd_prog(void *args)
+{
int tested = 0;
int tested_tries = 0;
- static DEFINE_TORTURE_RANDOM(trs);
VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started");
if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST))
set_user_nice(current, MAX_NICE);
- if (cur_ops->call && cur_ops->sync && cur_ops->cb_barrier) {
- init_rcu_head_on_stack(&fcs.rh);
- selfpropcb = true;
- }
do {
schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
-
- /* Tight loop containing cond_resched(). */
- if (selfpropcb) {
- WRITE_ONCE(fcs.stop, 0);
- cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);
- }
- cver = READ_ONCE(rcu_torture_current_version);
- gps = cur_ops->get_gp_seq();
- sd = cur_ops->stall_dur() + 1;
- sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div;
- dur = sd4 + torture_random(&trs) % (sd - sd4);
- rcu_fwd_startat = jiffies;
- stopat = rcu_fwd_startat + dur;
- while (time_before(jiffies, stopat) && !torture_must_stop()) {
- idx = cur_ops->readlock();
- udelay(10);
- cur_ops->readunlock(idx);
- if (!fwd_progress_need_resched || need_resched())
- cond_resched();
- }
- tested_tries++;
- if (!time_before(jiffies, stopat) && !torture_must_stop()) {
- tested++;
- cver = READ_ONCE(rcu_torture_current_version) - cver;
- gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
- WARN_ON(!cver && gps < 2);
- pr_alert("%s: Duration %ld cver %ld gps %ld\n", __func__, dur, cver, gps);
- }
- if (selfpropcb) {
- WRITE_ONCE(fcs.stop, 1);
- cur_ops->sync(); /* Wait for running CB to complete. */
- cur_ops->cb_barrier(); /* Wait for queued callbacks. */
- }
-
- /* Loop continuously posting RCU callbacks. */
- WRITE_ONCE(rcu_fwd_cb_nodelay, true);
- cur_ops->sync(); /* Later readers see above write. */
- rcu_fwd_startat = jiffies;
- stopat = rcu_fwd_startat + MAX_FWD_CB_JIFFIES;
- n_launders = 0;
- n_launders_cb = 0;
- n_launders_sa = 0;
- n_max_cbs = 0;
- n_max_gps = 0;
- for (i = 0; i < ARRAY_SIZE(n_launders_hist); i++)
- n_launders_hist[i] = 0;
- cver = READ_ONCE(rcu_torture_current_version);
- gps = cur_ops->get_gp_seq();
- while (time_before(jiffies, stopat) && !torture_must_stop()) {
- rfcp = READ_ONCE(rcu_fwd_cb_head);
- rfcpn = NULL;
- if (rfcp)
- rfcpn = READ_ONCE(rfcp->rfc_next);
- if (rfcpn) {
- if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS &&
- ++n_max_gps >= MIN_FWD_CBS_LAUNDERED)
- break;
- rcu_fwd_cb_head = rfcpn;
- n_launders++;
- n_launders_sa++;
- } else {
- rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL);
- if (WARN_ON_ONCE(!rfcp)) {
- schedule_timeout_interruptible(1);
- continue;
- }
- n_max_cbs++;
- n_launders_sa = 0;
- rfcp->rfc_gps = 0;
- }
- cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
- cond_resched();
- }
- stoppedat = jiffies;
- n_launders_cb_snap = READ_ONCE(n_launders_cb);
- cver = READ_ONCE(rcu_torture_current_version) - cver;
- gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
- cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */
- for (;;) {
- rfcp = rcu_fwd_cb_head;
- if (!rfcp)
- break;
- rcu_fwd_cb_head = rfcp->rfc_next;
- kfree(rfcp);
- }
- rcu_fwd_cb_tail = &rcu_fwd_cb_head;
- WRITE_ONCE(rcu_fwd_cb_nodelay, false);
- if (!torture_must_stop()) {
- WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED);
- pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n",
- __func__,
- stoppedat - rcu_fwd_startat,
- jiffies - stoppedat,
- n_launders + n_max_cbs - n_launders_cb_snap,
- n_launders, n_launders_sa,
- n_max_gps, n_max_cbs, cver, gps);
- for (i = ARRAY_SIZE(n_launders_hist) - 1; i > 0; i--)
- if (n_launders_hist[i] > 0)
- break;
- pr_alert("Callback-invocation histogram:");
- for (j = 0; j <= i; j++)
- pr_cont(" %ds: %ld", j + 1, n_launders_hist[j]);
- pr_cont("\n");
- }
+ rcu_torture_fwd_prog_nr(&tested, &tested_tries);
+ rcu_torture_fwd_prog_cr();
/* Avoid slow periods, better to test when busy. */
stutter_wait("rcu_torture_fwd_prog");
} while (!torture_must_stop());
- if (selfpropcb) {
- WARN_ON(READ_ONCE(fcs.stop) != 2);
- destroy_rcu_head_on_stack(&fcs.rh);
- }
/* Short runs might not contain a valid forward-progress attempt. */
WARN_ON(!tested && tested_tries >= 5);
pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries);
