diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 5ae4dcc6ba27..8b347b9659aa 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -918,17 +918,18 @@ static void rcu_torture_enable_rt_throttle(void) old_rt_runtime = -1; } -static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long start, unsigned long end) +static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start) { int cpu; static int dbg_done; + unsigned long end = jiffies; bool gp_done; unsigned long j; static unsigned long last_persist; unsigned long lp; unsigned long mininterval = test_boost_duration * HZ - HZ / 2; - if (end - start > mininterval) { + if (end - *start > mininterval) { // Recheck after checking time to avoid false positives. smp_mb(); // Time check before grace-period check. if (cur_ops->poll_gp_state(gp_state)) @@ -945,7 +946,7 @@ static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long start n_rcu_torture_boost_failure++; if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) { pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n", - current->rt_priority, gp_state, end - start); + current->rt_priority, gp_state, end - *start); cur_ops->gp_kthread_dbg(); // Recheck after print to flag grace period ending during splat. gp_done = cur_ops->poll_gp_state(gp_state); @@ -955,6 +956,8 @@ static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long start } return true; // failed + } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) { + *start = jiffies; } return false; // passed @@ -995,7 +998,7 @@ static int rcu_torture_boost(void *arg) while (time_before(jiffies, endtime)) { // Has current GP gone too long? if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) - failed = rcu_torture_boost_failed(gp_state, gp_state_time, jiffies); + failed = rcu_torture_boost_failed(gp_state, &gp_state_time); // If we don't have a grace period in flight, start one. if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) { gp_state = cur_ops->start_gp_poll(); @@ -1016,7 +1019,7 @@ static int rcu_torture_boost(void *arg) // In case the grace period extended beyond the end of the loop. if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) - rcu_torture_boost_failed(gp_state, gp_state_time, jiffies); + rcu_torture_boost_failed(gp_state, &gp_state_time); /* * Set the start time of the next test interval. diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index af92d9fee0d4..8bde1b53b0c9 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -723,6 +723,10 @@ static void check_cpu_stall(struct rcu_data *rdp) * count this as an RCU priority boosting failure. A return of true says * RCU priority boosting is to blame, and false says otherwise. If false * is returned, the first of the CPUs to blame is stored through cpup. + * + * If cpup is NULL, then a lockless quick check is carried out, suitable + * for high-rate usage. On the other hand, if cpup is non-NULL, each + * rcu_node structure's ->lock is acquired, ruling out high-rate usage. */ bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) { @@ -731,6 +735,12 @@ bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) struct rcu_node *rnp; rcu_for_each_leaf_node(rnp) { + if (!cpup) { + if (READ_ONCE(rnp->qsmask)) + return false; + else + continue; + } raw_spin_lock_irqsave_rcu_node(rnp, flags); if (!rnp->qsmask) { // No CPUs without quiescent states for this rnp.