p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

rcu/nocb: Reduce ->nocb_lock contention with separate ->nocb_gp_lock 

The sleep/wakeup of the no-CBs grace-period kthreads is synchronized
using the ->nocb_lock of the first CPU corresponding to that kthread.
This commit provides a separate ->nocb_gp_lock for this purpose, thus
reducing contention on ->nocb_lock.

Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
This commit is contained in:
Paul E. McKenney 2019-06-02 13:41:08 -07:00
parent 523bddd553
commit 4fd8c5f153
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
kernel/rcu/tree.h
View File

@ -202,7 +202,8 @@ struct rcu_data {
struct timer_list nocb_timer; /* Enforce finite deferral. */ struct timer_list nocb_timer; /* Enforce finite deferral. */
/* The following fields are used by GP kthread, hence own cacheline. */ /* The following fields are used by GP kthread, hence own cacheline. */
bool nocb_gp_sleep ____cacheline_internodealigned_in_smp; raw_spinlock_t nocb_gp_lock ____cacheline_internodealigned_in_smp;
bool nocb_gp_sleep;
/* Is the nocb GP thread asleep? */ /* Is the nocb GP thread asleep? */
struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */ struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */
bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */ bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */

9
kernel/rcu/tree_plugin.h
View File

@ -1604,9 +1604,9 @@ static void wake_nocb_gp(struct rcu_data *rdp, bool force,
del_timer(&rdp->nocb_timer); del_timer(&rdp->nocb_timer);
rcu_nocb_unlock_irqrestore(rdp, flags); rcu_nocb_unlock_irqrestore(rdp, flags);
smp_mb(); /* enqueue before ->nocb_gp_sleep. */ smp_mb(); /* enqueue before ->nocb_gp_sleep. */
rcu_nocb_lock_irqsave(rdp_gp, flags); raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
WRITE_ONCE(rdp_gp->nocb_gp_sleep, false); WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);
rcu_nocb_unlock_irqrestore(rdp_gp, flags); raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
wake_up_process(rdp_gp->nocb_gp_kthread); wake_up_process(rdp_gp->nocb_gp_kthread);
} else { } else {
rcu_nocb_unlock_irqrestore(rdp, flags); rcu_nocb_unlock_irqrestore(rdp, flags);
@ -1761,9 +1761,9 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)
trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait")); trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait"));
} }
if (!rcu_nocb_poll) { if (!rcu_nocb_poll) {
rcu_nocb_lock_irqsave(my_rdp, flags); raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
WRITE_ONCE(my_rdp->nocb_gp_sleep, true); WRITE_ONCE(my_rdp->nocb_gp_sleep, true);
rcu_nocb_unlock_irqrestore(my_rdp, flags); raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
} }
WARN_ON(signal_pending(current)); WARN_ON(signal_pending(current));
} }
@ -1943,6 +1943,7 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
init_swait_queue_head(&rdp->nocb_cb_wq); init_swait_queue_head(&rdp->nocb_cb_wq);
init_swait_queue_head(&rdp->nocb_gp_wq); init_swait_queue_head(&rdp->nocb_gp_wq);
raw_spin_lock_init(&rdp->nocb_lock); raw_spin_lock_init(&rdp->nocb_lock);
raw_spin_lock_init(&rdp->nocb_gp_lock);
timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);
} }