openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'rtt-speedup.2020.09.16a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into bpf-next 

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov 2020-09-23 19:32:09 -07:00
parent f00f2f7fe8 f747c7e15d
commit 182bf3f3dd
2 changed files with 45 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/linux/rcupdate_trace.h
View File

@ -50,6 +50,7 @@ static inline void rcu_read_lock_trace(void)
struct task_struct *t = current; struct task_struct *t = current;
WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1); WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1);
barrier();
if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) &&
t->trc_reader_special.b.need_mb) t->trc_reader_special.b.need_mb)
smp_mb(); // Pairs with update-side barriers smp_mb(); // Pairs with update-side barriers
@ -72,6 +73,9 @@ static inline void rcu_read_unlock_trace(void)
rcu_lock_release(&rcu_trace_lock_map); rcu_lock_release(&rcu_trace_lock_map);
nesting = READ_ONCE(t->trc_reader_nesting) - 1; nesting = READ_ONCE(t->trc_reader_nesting) - 1;
barrier(); // Critical section before disabling.
// Disable IPI-based setting of .need_qs.
WRITE_ONCE(t->trc_reader_nesting, INT_MIN);
if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) { if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) {
WRITE_ONCE(t->trc_reader_nesting, nesting); WRITE_ONCE(t->trc_reader_nesting, nesting);
return; // We assume shallow reader nesting. return; // We assume shallow reader nesting.

53
kernel/rcu/tasks.h
View File

@ -28,6 +28,8 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
* @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.
* @gp_func: This flavor's grace-period-wait function. * @gp_func: This flavor's grace-period-wait function.
* @gp_state: Grace period's most recent state transition (debugging). * @gp_state: Grace period's most recent state transition (debugging).
* @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping.
* @init_fract: Initial backoff sleep interval.
* @gp_jiffies: Time of last @gp_state transition. * @gp_jiffies: Time of last @gp_state transition.
* @gp_start: Most recent grace-period start in jiffies. * @gp_start: Most recent grace-period start in jiffies.
* @n_gps: Number of grace periods completed since boot. * @n_gps: Number of grace periods completed since boot.
@ -48,6 +50,8 @@ struct rcu_tasks {
struct wait_queue_head cbs_wq; struct wait_queue_head cbs_wq;
raw_spinlock_t cbs_lock; raw_spinlock_t cbs_lock;
int gp_state; int gp_state;
int gp_sleep;
int init_fract;
unsigned long gp_jiffies; unsigned long gp_jiffies;
unsigned long gp_start; unsigned long gp_start;
unsigned long n_gps; unsigned long n_gps;
@ -81,7 +85,7 @@ static struct rcu_tasks rt_name = \
DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu); DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu);
/* Avoid IPIing CPUs early in the grace period. */ /* Avoid IPIing CPUs early in the grace period. */
#define RCU_TASK_IPI_DELAY (HZ / 2) #define RCU_TASK_IPI_DELAY (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) ? HZ / 2 : 0)
static int rcu_task_ipi_delay __read_mostly = RCU_TASK_IPI_DELAY; static int rcu_task_ipi_delay __read_mostly = RCU_TASK_IPI_DELAY;
module_param(rcu_task_ipi_delay, int, 0644); module_param(rcu_task_ipi_delay, int, 0644);
@ -231,7 +235,7 @@ static int __noreturn rcu_tasks_kthread(void *arg)
cond_resched(); cond_resched();
} }
/* Paranoid sleep to keep this from entering a tight loop */ /* Paranoid sleep to keep this from entering a tight loop */
schedule_timeout_idle(HZ/10); schedule_timeout_idle(rtp->gp_sleep);
set_tasks_gp_state(rtp, RTGS_WAIT_CBS); set_tasks_gp_state(rtp, RTGS_WAIT_CBS);
} }
@ -329,8 +333,10 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp)
*/ */
lastreport = jiffies; lastreport = jiffies;
/* Start off with HZ/10 wait and slowly back off to 1 HZ wait. */ // Start off with initial wait and slowly back off to 1 HZ wait.
fract = 10; fract = rtp->init_fract;
if (fract > HZ)
fract = HZ;
for (;;) { for (;;) {
bool firstreport; bool firstreport;
@ -553,6 +559,8 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks);
static int __init rcu_spawn_tasks_kthread(void) static int __init rcu_spawn_tasks_kthread(void)
{ {
rcu_tasks.gp_sleep = HZ / 10;
rcu_tasks.init_fract = 10;
rcu_tasks.pregp_func = rcu_tasks_pregp_step; rcu_tasks.pregp_func = rcu_tasks_pregp_step;
rcu_tasks.pertask_func = rcu_tasks_pertask; rcu_tasks.pertask_func = rcu_tasks_pertask;
rcu_tasks.postscan_func = rcu_tasks_postscan; rcu_tasks.postscan_func = rcu_tasks_postscan;
@ -685,6 +693,7 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks_rude);
static int __init rcu_spawn_tasks_rude_kthread(void) static int __init rcu_spawn_tasks_rude_kthread(void)
{ {
rcu_tasks_rude.gp_sleep = HZ / 10;
rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude); rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude);
return 0; return 0;
} }
@ -745,9 +754,9 @@ static DEFINE_PER_CPU(bool, trc_ipi_to_cpu);
// The number of detections of task quiescent state relying on // The number of detections of task quiescent state relying on
// heavyweight readers executing explicit memory barriers. // heavyweight readers executing explicit memory barriers.
unsigned long n_heavy_reader_attempts; static unsigned long n_heavy_reader_attempts;
unsigned long n_heavy_reader_updates; static unsigned long n_heavy_reader_updates;
unsigned long n_heavy_reader_ofl_updates; static unsigned long n_heavy_reader_ofl_updates;
void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func);
DEFINE_RCU_TASKS(rcu_tasks_trace, rcu_tasks_wait_gp, call_rcu_tasks_trace, DEFINE_RCU_TASKS(rcu_tasks_trace, rcu_tasks_wait_gp, call_rcu_tasks_trace,
@ -821,6 +830,12 @@ static void trc_read_check_handler(void *t_in)
WRITE_ONCE(t->trc_reader_checked, true); WRITE_ONCE(t->trc_reader_checked, true);
goto reset_ipi; goto reset_ipi;
} }
// If we are racing with an rcu_read_unlock_trace(), try again later.
if (unlikely(t->trc_reader_nesting < 0)) {
if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end)))
wake_up(&trc_wait);
goto reset_ipi;
}
WRITE_ONCE(t->trc_reader_checked, true); WRITE_ONCE(t->trc_reader_checked, true);
// Get here if the task is in a read-side critical section. Set // Get here if the task is in a read-side critical section. Set
@ -911,7 +926,8 @@ static void trc_wait_for_one_reader(struct task_struct *t,
// If currently running, send an IPI, either way, add to list. // If currently running, send an IPI, either way, add to list.
trc_add_holdout(t, bhp); trc_add_holdout(t, bhp);
if (task_curr(t) && time_after(jiffies, rcu_tasks_trace.gp_start + rcu_task_ipi_delay)) { if (task_curr(t) &&
time_after(jiffies + 1, rcu_tasks_trace.gp_start + rcu_task_ipi_delay)) {
// The task is currently running, so try IPIing it. // The task is currently running, so try IPIing it.
cpu = task_cpu(t); cpu = task_cpu(t);
@ -1072,15 +1088,17 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp)
if (ret) if (ret)
break; // Count reached zero. break; // Count reached zero.
// Stall warning time, so make a list of the offenders. // Stall warning time, so make a list of the offenders.
rcu_read_lock();
for_each_process_thread(g, t) for_each_process_thread(g, t)
if (READ_ONCE(t->trc_reader_special.b.need_qs)) if (READ_ONCE(t->trc_reader_special.b.need_qs))
trc_add_holdout(t, &holdouts); trc_add_holdout(t, &holdouts);
rcu_read_unlock();
firstreport = true; firstreport = true;
list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) {
if (READ_ONCE(t->trc_reader_special.b.need_qs)) { if (READ_ONCE(t->trc_reader_special.b.need_qs))
show_stalled_task_trace(t, &firstreport); show_stalled_task_trace(t, &firstreport);
trc_del_holdout(t); trc_del_holdout(t); // Release task_struct reference.
} }
if (firstreport) if (firstreport)
pr_err("INFO: rcu_tasks_trace detected stalls? (Counter/taskslist mismatch?)\n"); pr_err("INFO: rcu_tasks_trace detected stalls? (Counter/taskslist mismatch?)\n");
show_stalled_ipi_trace(); show_stalled_ipi_trace();
@ -1163,6 +1181,17 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks_trace);
static int __init rcu_spawn_tasks_trace_kthread(void) static int __init rcu_spawn_tasks_trace_kthread(void)
{ {
if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) {
rcu_tasks_trace.gp_sleep = HZ / 10;
rcu_tasks_trace.init_fract = 10;
} else {
rcu_tasks_trace.gp_sleep = HZ / 200;
if (rcu_tasks_trace.gp_sleep <= 0)
rcu_tasks_trace.gp_sleep = 1;
rcu_tasks_trace.init_fract = HZ / 5;
if (rcu_tasks_trace.init_fract <= 0)
rcu_tasks_trace.init_fract = 1;
}
rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step; rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step;
rcu_tasks_trace.pertask_func = rcu_tasks_trace_pertask; rcu_tasks_trace.pertask_func = rcu_tasks_trace_pertask;
rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan; rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan;