p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

rcu: Convert ACCESS_ONCE() to READ_ONCE() and WRITE_ONCE() 

This commit moves from the old ACCESS_ONCE() API to the new READ_ONCE()
and WRITE_ONCE() APIs.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck:  Updated to include kernel/torture.c as suggested by Jason Low. ]
This commit is contained in:
Paul E. McKenney 2015-03-03 14:57:58 -08:00
parent 030bbdbf4c
commit 7d0ae8086b
10 changed files with 193 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/linux/rculist.h
View File

@ -29,8 +29,8 @@
*/ */
static inline void INIT_LIST_HEAD_RCU(struct list_head *list) static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
{ {
ACCESS_ONCE(list->next) = list; WRITE_ONCE(list->next, list);
ACCESS_ONCE(list->prev) = list; WRITE_ONCE(list->prev, list);
} }
/* /*
@ -288,7 +288,7 @@ static inline void list_splice_init_rcu(struct list_head *list,
#define list_first_or_null_rcu(ptr, type, member) \ #define list_first_or_null_rcu(ptr, type, member) \
({ \ ({ \
struct list_head *__ptr = (ptr); \ struct list_head *__ptr = (ptr); \
struct list_head *__next = ACCESS_ONCE(__ptr->next); \ struct list_head *__next = READ_ONCE(__ptr->next); \
likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \ likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \
}) })

16
include/linux/rcupdate.h
View File

@ -364,8 +364,8 @@ extern struct srcu_struct tasks_rcu_exit_srcu;
#define rcu_note_voluntary_context_switch(t) \ #define rcu_note_voluntary_context_switch(t) \
do { \ do { \
rcu_all_qs(); \ rcu_all_qs(); \
if (ACCESS_ONCE((t)->rcu_tasks_holdout)) \ if (READ_ONCE((t)->rcu_tasks_holdout)) \
ACCESS_ONCE((t)->rcu_tasks_holdout) = false; \ WRITE_ONCE((t)->rcu_tasks_holdout, false); \
} while (0) } while (0)
#else /* #ifdef CONFIG_TASKS_RCU */ #else /* #ifdef CONFIG_TASKS_RCU */
#define TASKS_RCU(x) do { } while (0) #define TASKS_RCU(x) do { } while (0)
@ -609,7 +609,7 @@ static inline void rcu_preempt_sleep_check(void)
#define __rcu_access_pointer(p, space) \ #define __rcu_access_pointer(p, space) \
({ \ ({ \
typeof(*p) *_________p1 = (typeof(*p) *__force)ACCESS_ONCE(p); \ typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \
rcu_dereference_sparse(p, space); \ rcu_dereference_sparse(p, space); \
((typeof(*p) __force __kernel *)(_________p1)); \ ((typeof(*p) __force __kernel *)(_________p1)); \
}) })
@ -630,7 +630,7 @@ static inline void rcu_preempt_sleep_check(void)
#define __rcu_access_index(p, space) \ #define __rcu_access_index(p, space) \
({ \ ({ \
typeof(p) _________p1 = ACCESS_ONCE(p); \ typeof(p) _________p1 = READ_ONCE(p); \
rcu_dereference_sparse(p, space); \ rcu_dereference_sparse(p, space); \
(_________p1); \ (_________p1); \
}) })
@ -659,7 +659,7 @@ static inline void rcu_preempt_sleep_check(void)
*/ */
#define lockless_dereference(p) \ #define lockless_dereference(p) \
({ \ ({ \
typeof(p) _________p1 = ACCESS_ONCE(p); \ typeof(p) _________p1 = READ_ONCE(p); \
smp_read_barrier_depends(); /* Dependency order vs. p above. */ \ smp_read_barrier_depends(); /* Dependency order vs. p above. */ \
(_________p1); \ (_________p1); \
}) })
@ -702,7 +702,7 @@ static inline void rcu_preempt_sleep_check(void)
* @p: The pointer to read * @p: The pointer to read
* *
* Return the value of the specified RCU-protected pointer, but omit the * Return the value of the specified RCU-protected pointer, but omit the
* smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful * smp_read_barrier_depends() and keep the READ_ONCE(). This is useful
* when the value of this pointer is accessed, but the pointer is not * when the value of this pointer is accessed, but the pointer is not
* dereferenced, for example, when testing an RCU-protected pointer against * dereferenced, for example, when testing an RCU-protected pointer against
* NULL. Although rcu_access_pointer() may also be used in cases where * NULL. Although rcu_access_pointer() may also be used in cases where
@ -791,7 +791,7 @@ static inline void rcu_preempt_sleep_check(void)
* @p: The index to read * @p: The index to read
* *
* Return the value of the specified RCU-protected index, but omit the * Return the value of the specified RCU-protected index, but omit the
* smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful * smp_read_barrier_depends() and keep the READ_ONCE(). This is useful
* when the value of this index is accessed, but the index is not * when the value of this index is accessed, but the index is not
* dereferenced, for example, when testing an RCU-protected index against * dereferenced, for example, when testing an RCU-protected index against
* -1. Although rcu_access_index() may also be used in cases where * -1. Although rcu_access_index() may also be used in cases where
@ -827,7 +827,7 @@ static inline void rcu_preempt_sleep_check(void)
* @c: The conditions under which the dereference will take place * @c: The conditions under which the dereference will take place
* *
* Return the value of the specified RCU-protected pointer, but omit * Return the value of the specified RCU-protected pointer, but omit
* both the smp_read_barrier_depends() and the ACCESS_ONCE(). This * both the smp_read_barrier_depends() and the READ_ONCE(). This
* is useful in cases where update-side locks prevent the value of the * is useful in cases where update-side locks prevent the value of the
* pointer from changing. Please note that this primitive does -not- * pointer from changing. Please note that this primitive does -not-
* prevent the compiler from repeating this reference or combining it * prevent the compiler from repeating this reference or combining it

2
kernel/rcu/rcutorture.c
View File

@ -1413,7 +1413,7 @@ static int rcu_torture_barrier_cbs(void *arg)
do { do {
wait_event(barrier_cbs_wq[myid], wait_event(barrier_cbs_wq[myid],
(newphase = (newphase =
ACCESS_ONCE(barrier_phase)) != lastphase || READ_ONCE(barrier_phase)) != lastphase ||
torture_must_stop()); torture_must_stop());
lastphase = newphase; lastphase = newphase;
smp_mb(); /* ensure barrier_phase load before ->call(). */ smp_mb(); /* ensure barrier_phase load before ->call(). */

10
kernel/rcu/srcu.c
View File

@ -151,7 +151,7 @@ static unsigned long srcu_readers_seq_idx(struct srcu_struct *sp, int idx)
unsigned long t; unsigned long t;
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
t = ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->seq[idx]); t = READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->seq[idx]);
sum += t; sum += t;
} }
return sum; return sum;
@ -168,7 +168,7 @@ static unsigned long srcu_readers_active_idx(struct srcu_struct *sp, int idx)
unsigned long t; unsigned long t;
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
t = ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[idx]); t = READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[idx]);
sum += t; sum += t;
} }
return sum; return sum;
@ -265,8 +265,8 @@ static int srcu_readers_active(struct srcu_struct *sp)
unsigned long sum = 0; unsigned long sum = 0;
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
sum += ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[0]); sum += READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[0]);
sum += ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[1]); sum += READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[1]);
} }
return sum; return sum;
} }
@ -296,7 +296,7 @@ int __srcu_read_lock(struct srcu_struct *sp)
{ {
int idx; int idx;
idx = ACCESS_ONCE(sp->completed) & 0x1; idx = READ_ONCE(sp->completed) & 0x1;
preempt_disable(); preempt_disable();
__this_cpu_inc(sp->per_cpu_ref->c[idx]); __this_cpu_inc(sp->per_cpu_ref->c[idx]);
smp_mb(); /* B */ /* Avoid leaking the critical section. */ smp_mb(); /* B */ /* Avoid leaking the critical section. */

12
kernel/rcu/tiny_plugin.h
View File

@ -144,16 +144,17 @@ static void check_cpu_stall(struct rcu_ctrlblk *rcp)
return; return;
rcp->ticks_this_gp++; rcp->ticks_this_gp++;
j = jiffies; j = jiffies;
js = ACCESS_ONCE(rcp->jiffies_stall); js = READ_ONCE(rcp->jiffies_stall);
if (rcp->rcucblist && ULONG_CMP_GE(j, js)) { if (rcp->rcucblist && ULONG_CMP_GE(j, js)) {
pr_err("INFO: %s stall on CPU (%lu ticks this GP) idle=%llx (t=%lu jiffies q=%ld)\n", pr_err("INFO: %s stall on CPU (%lu ticks this GP) idle=%llx (t=%lu jiffies q=%ld)\n",
rcp->name, rcp->ticks_this_gp, DYNTICK_TASK_EXIT_IDLE, rcp->name, rcp->ticks_this_gp, DYNTICK_TASK_EXIT_IDLE,
jiffies - rcp->gp_start, rcp->qlen); jiffies - rcp->gp_start, rcp->qlen);
dump_stack(); dump_stack();
ACCESS_ONCE(rcp->jiffies_stall) = jiffies + WRITE_ONCE(rcp->jiffies_stall,
3 * rcu_jiffies_till_stall_check() + 3; jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
} else if (ULONG_CMP_GE(j, js)) { } else if (ULONG_CMP_GE(j, js)) {
ACCESS_ONCE(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check(); WRITE_ONCE(rcp->jiffies_stall,
jiffies + rcu_jiffies_till_stall_check());
} }
} }
@ -161,7 +162,8 @@ static void reset_cpu_stall_ticks(struct rcu_ctrlblk *rcp)
{ {
rcp->ticks_this_gp = 0; rcp->ticks_this_gp = 0;
rcp->gp_start = jiffies; rcp->gp_start = jiffies;
ACCESS_ONCE(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check(); WRITE_ONCE(rcp->jiffies_stall,
jiffies + rcu_jiffies_till_stall_check());
} }
static void check_cpu_stalls(void) static void check_cpu_stalls(void)

184
kernel/rcu/tree.c
View File

@ -191,17 +191,17 @@ unsigned long rcutorture_vernum;
*/ */
unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp) unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
{ {
return ACCESS_ONCE(rnp->qsmaskinitnext); return READ_ONCE(rnp->qsmaskinitnext);
} }
/* /*
* Return true if an RCU grace period is in progress. The ACCESS_ONCE()s * Return true if an RCU grace period is in progress. The READ_ONCE()s
* permit this function to be invoked without holding the root rcu_node * permit this function to be invoked without holding the root rcu_node
* structure's ->lock, but of course results can be subject to change. * structure's ->lock, but of course results can be subject to change.
*/ */
static int rcu_gp_in_progress(struct rcu_state *rsp) static int rcu_gp_in_progress(struct rcu_state *rsp)
{ {
return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum); return READ_ONCE(rsp->completed) != READ_ONCE(rsp->gpnum);
} }
/* /*
@ -278,8 +278,8 @@ static void rcu_momentary_dyntick_idle(void)
if (!(resched_mask & rsp->flavor_mask)) if (!(resched_mask & rsp->flavor_mask))
continue; continue;
smp_mb(); /* rcu_sched_qs_mask before cond_resched_completed. */ smp_mb(); /* rcu_sched_qs_mask before cond_resched_completed. */
if (ACCESS_ONCE(rdp->mynode->completed) != if (READ_ONCE(rdp->mynode->completed) !=
ACCESS_ONCE(rdp->cond_resched_completed)) READ_ONCE(rdp->cond_resched_completed))
continue; continue;
/* /*
@ -491,9 +491,9 @@ void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
break; break;
} }
if (rsp != NULL) { if (rsp != NULL) {
*flags = ACCESS_ONCE(rsp->gp_flags); *flags = READ_ONCE(rsp->gp_flags);
*gpnum = ACCESS_ONCE(rsp->gpnum); *gpnum = READ_ONCE(rsp->gpnum);
*completed = ACCESS_ONCE(rsp->completed); *completed = READ_ONCE(rsp->completed);
return; return;
} }
*flags = 0; *flags = 0;
@ -539,10 +539,10 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp)
static int rcu_future_needs_gp(struct rcu_state *rsp) static int rcu_future_needs_gp(struct rcu_state *rsp)
{ {
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
int idx = (ACCESS_ONCE(rnp->completed) + 1) & 0x1; int idx = (READ_ONCE(rnp->completed) + 1) & 0x1;
int *fp = &rnp->need_future_gp[idx]; int *fp = &rnp->need_future_gp[idx];
return ACCESS_ONCE(*fp); return READ_ONCE(*fp);
} }
/* /*
@ -565,7 +565,7 @@ cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp)
return 1; /* Yes, this CPU has newly registered callbacks. */ return 1; /* Yes, this CPU has newly registered callbacks. */
for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++)
if (rdp->nxttail[i - 1] != rdp->nxttail[i] && if (rdp->nxttail[i - 1] != rdp->nxttail[i] &&
ULONG_CMP_LT(ACCESS_ONCE(rsp->completed), ULONG_CMP_LT(READ_ONCE(rsp->completed),
rdp->nxtcompleted[i])) rdp->nxtcompleted[i]))
return 1; /* Yes, CBs for future grace period. */ return 1; /* Yes, CBs for future grace period. */
return 0; /* No grace period needed. */ return 0; /* No grace period needed. */
@ -1011,9 +1011,9 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
return 1; return 1;
} else { } else {
if (ULONG_CMP_LT(ACCESS_ONCE(rdp->gpnum) + ULONG_MAX / 4, if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4,
rdp->mynode->gpnum)) rdp->mynode->gpnum))
ACCESS_ONCE(rdp->gpwrap) = true; WRITE_ONCE(rdp->gpwrap, true);
return 0; return 0;
} }
} }
@ -1093,12 +1093,12 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
if (ULONG_CMP_GE(jiffies, if (ULONG_CMP_GE(jiffies,
rdp->rsp->gp_start + jiffies_till_sched_qs) || rdp->rsp->gp_start + jiffies_till_sched_qs) ||
ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
if (!(ACCESS_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) { if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) {
ACCESS_ONCE(rdp->cond_resched_completed) = WRITE_ONCE(rdp->cond_resched_completed,
ACCESS_ONCE(rdp->mynode->completed); READ_ONCE(rdp->mynode->completed));
smp_mb(); /* ->cond_resched_completed before *rcrmp. */ smp_mb(); /* ->cond_resched_completed before *rcrmp. */
ACCESS_ONCE(*rcrmp) = WRITE_ONCE(*rcrmp,
ACCESS_ONCE(*rcrmp) + rdp->rsp->flavor_mask; READ_ONCE(*rcrmp) + rdp->rsp->flavor_mask);
resched_cpu(rdp->cpu); /* Force CPU into scheduler. */ resched_cpu(rdp->cpu); /* Force CPU into scheduler. */
rdp->rsp->jiffies_resched += 5; /* Enable beating. */ rdp->rsp->jiffies_resched += 5; /* Enable beating. */
} else if (ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) { } else if (ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
@ -1119,9 +1119,9 @@ static void record_gp_stall_check_time(struct rcu_state *rsp)
rsp->gp_start = j; rsp->gp_start = j;
smp_wmb(); /* Record start time before stall time. */ smp_wmb(); /* Record start time before stall time. */
j1 = rcu_jiffies_till_stall_check(); j1 = rcu_jiffies_till_stall_check();
ACCESS_ONCE(rsp->jiffies_stall) = j + j1; WRITE_ONCE(rsp->jiffies_stall, j + j1);
rsp->jiffies_resched = j + j1 / 2; rsp->jiffies_resched = j + j1 / 2;
rsp->n_force_qs_gpstart = ACCESS_ONCE(rsp->n_force_qs); rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs);
} }
/* /*
@ -1133,7 +1133,7 @@ static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp)
unsigned long j; unsigned long j;
j = jiffies; j = jiffies;
gpa = ACCESS_ONCE(rsp->gp_activity); gpa = READ_ONCE(rsp->gp_activity);
if (j - gpa > 2 * HZ) if (j - gpa > 2 * HZ)
pr_err("%s kthread starved for %ld jiffies!\n", pr_err("%s kthread starved for %ld jiffies!\n",
rsp->name, j - gpa); rsp->name, j - gpa);
@ -1173,12 +1173,13 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
/* Only let one CPU complain about others per time interval. */ /* Only let one CPU complain about others per time interval. */
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
delta = jiffies - ACCESS_ONCE(rsp->jiffies_stall); delta = jiffies - READ_ONCE(rsp->jiffies_stall);
if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) { if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) {
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
return; return;
} }
ACCESS_ONCE(rsp->jiffies_stall) = jiffies + 3 * rcu_jiffies_till_stall_check() + 3; WRITE_ONCE(rsp->jiffies_stall,
jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
/* /*
@ -1212,12 +1213,12 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
if (ndetected) { if (ndetected) {
rcu_dump_cpu_stacks(rsp); rcu_dump_cpu_stacks(rsp);
} else { } else {
if (ACCESS_ONCE(rsp->gpnum) != gpnum || if (READ_ONCE(rsp->gpnum) != gpnum ||
ACCESS_ONCE(rsp->completed) == gpnum) { READ_ONCE(rsp->completed) == gpnum) {
pr_err("INFO: Stall ended before state dump start\n"); pr_err("INFO: Stall ended before state dump start\n");
} else { } else {
j = jiffies; j = jiffies;
gpa = ACCESS_ONCE(rsp->gp_activity); gpa = READ_ONCE(rsp->gp_activity);
pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n", pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
rsp->name, j - gpa, j, gpa, rsp->name, j - gpa, j, gpa,
jiffies_till_next_fqs, jiffies_till_next_fqs,
@ -1262,9 +1263,9 @@ static void print_cpu_stall(struct rcu_state *rsp)
rcu_dump_cpu_stacks(rsp); rcu_dump_cpu_stacks(rsp);
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
if (ULONG_CMP_GE(jiffies, ACCESS_ONCE(rsp->jiffies_stall))) if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
ACCESS_ONCE(rsp->jiffies_stall) = jiffies + WRITE_ONCE(rsp->jiffies_stall,
3 * rcu_jiffies_till_stall_check() + 3; jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
/* /*
@ -1307,20 +1308,20 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
* Given this check, comparisons of jiffies, rsp->jiffies_stall, * Given this check, comparisons of jiffies, rsp->jiffies_stall,
* and rsp->gp_start suffice to forestall false positives. * and rsp->gp_start suffice to forestall false positives.
*/ */
gpnum = ACCESS_ONCE(rsp->gpnum); gpnum = READ_ONCE(rsp->gpnum);
smp_rmb(); /* Pick up ->gpnum first... */ smp_rmb(); /* Pick up ->gpnum first... */
js = ACCESS_ONCE(rsp->jiffies_stall); js = READ_ONCE(rsp->jiffies_stall);
smp_rmb(); /* ...then ->jiffies_stall before the rest... */ smp_rmb(); /* ...then ->jiffies_stall before the rest... */
gps = ACCESS_ONCE(rsp->gp_start); gps = READ_ONCE(rsp->gp_start);
smp_rmb(); /* ...and finally ->gp_start before ->completed. */ smp_rmb(); /* ...and finally ->gp_start before ->completed. */
completed = ACCESS_ONCE(rsp->completed); completed = READ_ONCE(rsp->completed);
if (ULONG_CMP_GE(completed, gpnum) || if (ULONG_CMP_GE(completed, gpnum) ||
ULONG_CMP_LT(j, js) || ULONG_CMP_LT(j, js) ||
ULONG_CMP_GE(gps, js)) ULONG_CMP_GE(gps, js))
return; /* No stall or GP completed since entering function. */ return; /* No stall or GP completed since entering function. */
rnp = rdp->mynode; rnp = rdp->mynode;
if (rcu_gp_in_progress(rsp) && if (rcu_gp_in_progress(rsp) &&
(ACCESS_ONCE(rnp->qsmask) & rdp->grpmask)) { (READ_ONCE(rnp->qsmask) & rdp->grpmask)) {
/* We haven't checked in, so go dump stack. */ /* We haven't checked in, so go dump stack. */
print_cpu_stall(rsp); print_cpu_stall(rsp);
@ -1347,7 +1348,7 @@ void rcu_cpu_stall_reset(void)
struct rcu_state *rsp; struct rcu_state *rsp;
for_each_rcu_flavor(rsp) for_each_rcu_flavor(rsp)
ACCESS_ONCE(rsp->jiffies_stall) = jiffies + ULONG_MAX / 2; WRITE_ONCE(rsp->jiffies_stall, jiffies + ULONG_MAX / 2);
} }
/* /*
@ -1457,7 +1458,7 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp,
* doing some extra useless work. * doing some extra useless work.
*/ */
if (rnp->gpnum != rnp->completed || if (rnp->gpnum != rnp->completed ||
ACCESS_ONCE(rnp_root->gpnum) != ACCESS_ONCE(rnp_root->completed)) { READ_ONCE(rnp_root->gpnum) != READ_ONCE(rnp_root->completed)) {
rnp->need_future_gp[c & 0x1]++; rnp->need_future_gp[c & 0x1]++;
trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf")); trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf"));
goto out; goto out;
@ -1542,7 +1543,7 @@ static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
static void rcu_gp_kthread_wake(struct rcu_state *rsp) static void rcu_gp_kthread_wake(struct rcu_state *rsp)
{ {
if (current == rsp->gp_kthread || if (current == rsp->gp_kthread ||
!ACCESS_ONCE(rsp->gp_flags) || !READ_ONCE(rsp->gp_flags) ||
!rsp->gp_kthread) !rsp->gp_kthread)
return; return;
wake_up(&rsp->gp_wq); wake_up(&rsp->gp_wq);
@ -1677,7 +1678,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
/* Handle the ends of any preceding grace periods first. */ /* Handle the ends of any preceding grace periods first. */
if (rdp->completed == rnp->completed && if (rdp->completed == rnp->completed &&
!unlikely(ACCESS_ONCE(rdp->gpwrap))) { !unlikely(READ_ONCE(rdp->gpwrap))) {
/* No grace period end, so just accelerate recent callbacks. */ /* No grace period end, so just accelerate recent callbacks. */
ret = rcu_accelerate_cbs(rsp, rnp, rdp); ret = rcu_accelerate_cbs(rsp, rnp, rdp);
@ -1692,7 +1693,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend")); trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend"));
} }
if (rdp->gpnum != rnp->gpnum || unlikely(ACCESS_ONCE(rdp->gpwrap))) { if (rdp->gpnum != rnp->gpnum || unlikely(READ_ONCE(rdp->gpwrap))) {
/* /*
* If the current grace period is waiting for this CPU, * If the current grace period is waiting for this CPU,
* set up to detect a quiescent state, otherwise don't * set up to detect a quiescent state, otherwise don't
@ -1704,7 +1705,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr); rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask); rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
zero_cpu_stall_ticks(rdp); zero_cpu_stall_ticks(rdp);
ACCESS_ONCE(rdp->gpwrap) = false; WRITE_ONCE(rdp->gpwrap, false);
} }
return ret; return ret;
} }
@ -1717,9 +1718,9 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
local_irq_save(flags); local_irq_save(flags);
rnp = rdp->mynode; rnp = rdp->mynode;
if ((rdp->gpnum == ACCESS_ONCE(rnp->gpnum) && if ((rdp->gpnum == READ_ONCE(rnp->gpnum) &&
rdp->completed == ACCESS_ONCE(rnp->completed) && rdp->completed == READ_ONCE(rnp->completed) &&
!unlikely(ACCESS_ONCE(rdp->gpwrap))) || /* w/out lock. */ !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */
!raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */ !raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */
local_irq_restore(flags); local_irq_restore(flags);
return; return;
@ -1740,15 +1741,15 @@ static int rcu_gp_init(struct rcu_state *rsp)
struct rcu_data *rdp; struct rcu_data *rdp;
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
ACCESS_ONCE(rsp->gp_activity) = jiffies; WRITE_ONCE(rsp->gp_activity, jiffies);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq(&rnp->lock);
smp_mb__after_unlock_lock(); smp_mb__after_unlock_lock();
if (!ACCESS_ONCE(rsp->gp_flags)) { if (!READ_ONCE(rsp->gp_flags)) {
/* Spurious wakeup, tell caller to go back to sleep. */ /* Spurious wakeup, tell caller to go back to sleep. */
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
return 0; return 0;
} }
ACCESS_ONCE(rsp->gp_flags) = 0; /* Clear all flags: New grace period. */ WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */
if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) { if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) {
/* /*
@ -1834,9 +1835,9 @@ static int rcu_gp_init(struct rcu_state *rsp)
rdp = this_cpu_ptr(rsp->rda); rdp = this_cpu_ptr(rsp->rda);
rcu_preempt_check_blocked_tasks(rnp); rcu_preempt_check_blocked_tasks(rnp);
rnp->qsmask = rnp->qsmaskinit; rnp->qsmask = rnp->qsmaskinit;
ACCESS_ONCE(rnp->gpnum) = rsp->gpnum; WRITE_ONCE(rnp->gpnum, rsp->gpnum);
if (WARN_ON_ONCE(rnp->completed != rsp->completed)) if (WARN_ON_ONCE(rnp->completed != rsp->completed))
ACCESS_ONCE(rnp->completed) = rsp->completed; WRITE_ONCE(rnp->completed, rsp->completed);
if (rnp == rdp->mynode) if (rnp == rdp->mynode)
(void)__note_gp_changes(rsp, rnp, rdp); (void)__note_gp_changes(rsp, rnp, rdp);
rcu_preempt_boost_start_gp(rnp); rcu_preempt_boost_start_gp(rnp);
@ -1845,7 +1846,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
rnp->grphi, rnp->qsmask); rnp->grphi, rnp->qsmask);
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
cond_resched_rcu_qs(); cond_resched_rcu_qs();
ACCESS_ONCE(rsp->gp_activity) = jiffies; WRITE_ONCE(rsp->gp_activity, jiffies);
if (gp_init_delay > 0 && if (gp_init_delay > 0 &&
!(rsp->gpnum % (rcu_num_nodes * PER_RCU_NODE_PERIOD))) !(rsp->gpnum % (rcu_num_nodes * PER_RCU_NODE_PERIOD)))
schedule_timeout_uninterruptible(gp_init_delay); schedule_timeout_uninterruptible(gp_init_delay);
@ -1864,7 +1865,7 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
unsigned long maxj; unsigned long maxj;
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
ACCESS_ONCE(rsp->gp_activity) = jiffies; WRITE_ONCE(rsp->gp_activity, jiffies);
rsp->n_force_qs++; rsp->n_force_qs++;
if (fqs_state == RCU_SAVE_DYNTICK) { if (fqs_state == RCU_SAVE_DYNTICK) {
/* Collect dyntick-idle snapshots. */ /* Collect dyntick-idle snapshots. */
@ -1882,11 +1883,11 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj); force_qs_rnp(rsp, rcu_implicit_dynticks_qs, &isidle, &maxj);
} }
/* Clear flag to prevent immediate re-entry. */ /* Clear flag to prevent immediate re-entry. */
if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq(&rnp->lock);
smp_mb__after_unlock_lock(); smp_mb__after_unlock_lock();
ACCESS_ONCE(rsp->gp_flags) = WRITE_ONCE(rsp->gp_flags,
ACCESS_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS; READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS);
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
} }
return fqs_state; return fqs_state;
@ -1903,7 +1904,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
struct rcu_data *rdp; struct rcu_data *rdp;
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
ACCESS_ONCE(rsp->gp_activity) = jiffies; WRITE_ONCE(rsp->gp_activity, jiffies);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq(&rnp->lock);
smp_mb__after_unlock_lock(); smp_mb__after_unlock_lock();
gp_duration = jiffies - rsp->gp_start; gp_duration = jiffies - rsp->gp_start;
@ -1934,7 +1935,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
smp_mb__after_unlock_lock(); smp_mb__after_unlock_lock();
WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)); WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
WARN_ON_ONCE(rnp->qsmask); WARN_ON_ONCE(rnp->qsmask);
ACCESS_ONCE(rnp->completed) = rsp->gpnum; WRITE_ONCE(rnp->completed, rsp->gpnum);
rdp = this_cpu_ptr(rsp->rda); rdp = this_cpu_ptr(rsp->rda);
if (rnp == rdp->mynode) if (rnp == rdp->mynode)
needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; needgp = __note_gp_changes(rsp, rnp, rdp) || needgp;
@ -1942,7 +1943,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
nocb += rcu_future_gp_cleanup(rsp, rnp); nocb += rcu_future_gp_cleanup(rsp, rnp);
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
cond_resched_rcu_qs(); cond_resched_rcu_qs();
ACCESS_ONCE(rsp->gp_activity) = jiffies; WRITE_ONCE(rsp->gp_activity, jiffies);
} }
rnp = rcu_get_root(rsp); rnp = rcu_get_root(rsp);
raw_spin_lock_irq(&rnp->lock); raw_spin_lock_irq(&rnp->lock);
@ -1950,16 +1951,16 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
rcu_nocb_gp_set(rnp, nocb); rcu_nocb_gp_set(rnp, nocb);
/* Declare grace period done. */ /* Declare grace period done. */
ACCESS_ONCE(rsp->completed) = rsp->gpnum; WRITE_ONCE(rsp->completed, rsp->gpnum);
trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end")); trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
rsp->fqs_state = RCU_GP_IDLE; rsp->fqs_state = RCU_GP_IDLE;
rdp = this_cpu_ptr(rsp->rda); rdp = this_cpu_ptr(rsp->rda);
/* Advance CBs to reduce false positives below. */ /* Advance CBs to reduce false positives below. */
needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp; needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;
if (needgp || cpu_needs_another_gp(rsp, rdp)) { if (needgp || cpu_needs_another_gp(rsp, rdp)) {
ACCESS_ONCE(rsp->gp_flags) = RCU_GP_FLAG_INIT; WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), READ_ONCE(rsp->gpnum),
TPS("newreq")); TPS("newreq"));
} }
raw_spin_unlock_irq(&rnp->lock); raw_spin_unlock_irq(&rnp->lock);
@ -1983,20 +1984,20 @@ static int __noreturn rcu_gp_kthread(void *arg)
/* Handle grace-period start. */ /* Handle grace-period start. */
for (;;) { for (;;) {
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), READ_ONCE(rsp->gpnum),
TPS("reqwait")); TPS("reqwait"));
rsp->gp_state = RCU_GP_WAIT_GPS; rsp->gp_state = RCU_GP_WAIT_GPS;
wait_event_interruptible(rsp->gp_wq, wait_event_interruptible(rsp->gp_wq,
ACCESS_ONCE(rsp->gp_flags) & READ_ONCE(rsp->gp_flags) &
RCU_GP_FLAG_INIT); RCU_GP_FLAG_INIT);
/* Locking provides needed memory barrier. */ /* Locking provides needed memory barrier. */
if (rcu_gp_init(rsp)) if (rcu_gp_init(rsp))
break; break;
cond_resched_rcu_qs(); cond_resched_rcu_qs();
ACCESS_ONCE(rsp->gp_activity) = jiffies; WRITE_ONCE(rsp->gp_activity, jiffies);
WARN_ON(signal_pending(current)); WARN_ON(signal_pending(current));
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), READ_ONCE(rsp->gpnum),
TPS("reqwaitsig")); TPS("reqwaitsig"));
} }
@ -2012,39 +2013,39 @@ static int __noreturn rcu_gp_kthread(void *arg)
if (!ret) if (!ret)
rsp->jiffies_force_qs = jiffies + j; rsp->jiffies_force_qs = jiffies + j;
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), READ_ONCE(rsp->gpnum),
TPS("fqswait")); TPS("fqswait"));
rsp->gp_state = RCU_GP_WAIT_FQS; rsp->gp_state = RCU_GP_WAIT_FQS;
ret = wait_event_interruptible_timeout(rsp->gp_wq, ret = wait_event_interruptible_timeout(rsp->gp_wq,
((gf = ACCESS_ONCE(rsp->gp_flags)) & ((gf = READ_ONCE(rsp->gp_flags)) &
RCU_GP_FLAG_FQS) || RCU_GP_FLAG_FQS) ||
(!ACCESS_ONCE(rnp->qsmask) && (!READ_ONCE(rnp->qsmask) &&
!rcu_preempt_blocked_readers_cgp(rnp)), !rcu_preempt_blocked_readers_cgp(rnp)),
j); j);
/* Locking provides needed memory barriers. */ /* Locking provides needed memory barriers. */
/* If grace period done, leave loop. */ /* If grace period done, leave loop. */
if (!ACCESS_ONCE(rnp->qsmask) && if (!READ_ONCE(rnp->qsmask) &&
!rcu_preempt_blocked_readers_cgp(rnp)) !rcu_preempt_blocked_readers_cgp(rnp))
break; break;
/* If time for quiescent-state forcing, do it. */ /* If time for quiescent-state forcing, do it. */
if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) || if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) ||
(gf & RCU_GP_FLAG_FQS)) { (gf & RCU_GP_FLAG_FQS)) {
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), READ_ONCE(rsp->gpnum),
TPS("fqsstart")); TPS("fqsstart"));
fqs_state = rcu_gp_fqs(rsp, fqs_state); fqs_state = rcu_gp_fqs(rsp, fqs_state);
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), READ_ONCE(rsp->gpnum),
TPS("fqsend")); TPS("fqsend"));
cond_resched_rcu_qs(); cond_resched_rcu_qs();
ACCESS_ONCE(rsp->gp_activity) = jiffies; WRITE_ONCE(rsp->gp_activity, jiffies);
} else { } else {
/* Deal with stray signal. */ /* Deal with stray signal. */
cond_resched_rcu_qs(); cond_resched_rcu_qs();
ACCESS_ONCE(rsp->gp_activity) = jiffies; WRITE_ONCE(rsp->gp_activity, jiffies);
WARN_ON(signal_pending(current)); WARN_ON(signal_pending(current));
trace_rcu_grace_period(rsp->name, trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum), READ_ONCE(rsp->gpnum),
TPS("fqswaitsig")); TPS("fqswaitsig"));
} }
j = jiffies_till_next_fqs; j = jiffies_till_next_fqs;
@ -2086,8 +2087,8 @@ rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
*/ */
return false; return false;
} }
ACCESS_ONCE(rsp->gp_flags) = RCU_GP_FLAG_INIT; WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum), trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum),
TPS("newreq")); TPS("newreq"));
/* /*
@ -2359,7 +2360,7 @@ rcu_send_cbs_to_orphanage(int cpu, struct rcu_state *rsp,
rsp->qlen += rdp->qlen; rsp->qlen += rdp->qlen;
rdp->n_cbs_orphaned += rdp->qlen; rdp->n_cbs_orphaned += rdp->qlen;
rdp->qlen_lazy = 0; rdp->qlen_lazy = 0;
ACCESS_ONCE(rdp->qlen) = 0; WRITE_ONCE(rdp->qlen, 0);
} }
/* /*
@ -2580,7 +2581,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
/* If no callbacks are ready, just return. */ /* If no callbacks are ready, just return. */
if (!cpu_has_callbacks_ready_to_invoke(rdp)) { if (!cpu_has_callbacks_ready_to_invoke(rdp)) {
trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, 0); trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, 0);
trace_rcu_batch_end(rsp->name, 0, !!ACCESS_ONCE(rdp->nxtlist), trace_rcu_batch_end(rsp->name, 0, !!READ_ONCE(rdp->nxtlist),
need_resched(), is_idle_task(current), need_resched(), is_idle_task(current),
rcu_is_callbacks_kthread()); rcu_is_callbacks_kthread());
return; return;
@ -2636,7 +2637,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
} }
smp_mb(); /* List handling before counting for rcu_barrier(). */ smp_mb(); /* List handling before counting for rcu_barrier(). */
rdp->qlen_lazy -= count_lazy; rdp->qlen_lazy -= count_lazy;
ACCESS_ONCE(rdp->qlen) = rdp->qlen - count; WRITE_ONCE(rdp->qlen, rdp->qlen - count);
rdp->n_cbs_invoked += count; rdp->n_cbs_invoked += count;
/* Reinstate batch limit if we have worked down the excess. */ /* Reinstate batch limit if we have worked down the excess. */
@ -2793,7 +2794,7 @@ static void force_quiescent_state(struct rcu_state *rsp)
/* Funnel through hierarchy to reduce memory contention. */ /* Funnel through hierarchy to reduce memory contention. */
rnp = __this_cpu_read(rsp->rda->mynode); rnp = __this_cpu_read(rsp->rda->mynode);
for (; rnp != NULL; rnp = rnp->parent) { for (; rnp != NULL; rnp = rnp->parent) {
ret = (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) || ret = (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) ||
!raw_spin_trylock(&rnp->fqslock); !raw_spin_trylock(&rnp->fqslock);
if (rnp_old != NULL) if (rnp_old != NULL)
raw_spin_unlock(&rnp_old->fqslock); raw_spin_unlock(&rnp_old->fqslock);
@ -2809,13 +2810,12 @@ static void force_quiescent_state(struct rcu_state *rsp)
raw_spin_lock_irqsave(&rnp_old->lock, flags); raw_spin_lock_irqsave(&rnp_old->lock, flags);
smp_mb__after_unlock_lock(); smp_mb__after_unlock_lock();
raw_spin_unlock(&rnp_old->fqslock); raw_spin_unlock(&rnp_old->fqslock);
if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
rsp->n_force_qs_lh++; rsp->n_force_qs_lh++;
raw_spin_unlock_irqrestore(&rnp_old->lock, flags); raw_spin_unlock_irqrestore(&rnp_old->lock, flags);
return; /* Someone beat us to it. */ return; /* Someone beat us to it. */
} }
ACCESS_ONCE(rsp->gp_flags) = WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS);
ACCESS_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS;
raw_spin_unlock_irqrestore(&rnp_old->lock, flags); raw_spin_unlock_irqrestore(&rnp_old->lock, flags);
rcu_gp_kthread_wake(rsp); rcu_gp_kthread_wake(rsp);
} }
@ -2881,7 +2881,7 @@ static void rcu_process_callbacks(struct softirq_action *unused)
*/ */
static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
{ {
if (unlikely(!ACCESS_ONCE(rcu_scheduler_fully_active))) if (unlikely(!READ_ONCE(rcu_scheduler_fully_active)))
return; return;
if (likely(!rsp->boost)) { if (likely(!rsp->boost)) {
rcu_do_batch(rsp, rdp); rcu_do_batch(rsp, rdp);
@ -2972,7 +2972,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
WARN_ON_ONCE((unsigned long)head & 0x1); /* Misaligned rcu_head! */ WARN_ON_ONCE((unsigned long)head & 0x1); /* Misaligned rcu_head! */
if (debug_rcu_head_queue(head)) { if (debug_rcu_head_queue(head)) {
/* Probable double call_rcu(), so leak the callback. */ /* Probable double call_rcu(), so leak the callback. */
ACCESS_ONCE(head->func) = rcu_leak_callback; WRITE_ONCE(head->func, rcu_leak_callback);
WARN_ONCE(1, "__call_rcu(): Leaked duplicate callback\n"); WARN_ONCE(1, "__call_rcu(): Leaked duplicate callback\n");
return; return;
} }
@ -3011,7 +3011,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
if (!likely(rdp->nxtlist)) if (!likely(rdp->nxtlist))
init_default_callback_list(rdp); init_default_callback_list(rdp);
} }
ACCESS_ONCE(rdp->qlen) = rdp->qlen + 1; WRITE_ONCE(rdp->qlen, rdp->qlen + 1);
if (lazy) if (lazy)
rdp->qlen_lazy++; rdp->qlen_lazy++;
else else
@ -3450,14 +3450,14 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
} }
/* Has another RCU grace period completed? */ /* Has another RCU grace period completed? */
if (ACCESS_ONCE(rnp->completed) != rdp->completed) { /* outside lock */ if (READ_ONCE(rnp->completed) != rdp->completed) { /* outside lock */
rdp->n_rp_gp_completed++; rdp->n_rp_gp_completed++;
return 1; return 1;
} }
/* Has a new RCU grace period started? */ /* Has a new RCU grace period started? */
if (ACCESS_ONCE(rnp->gpnum) != rdp->gpnum || if (READ_ONCE(rnp->gpnum) != rdp->gpnum ||
unlikely(ACCESS_ONCE(rdp->gpwrap))) { /* outside lock */ unlikely(READ_ONCE(rdp->gpwrap))) { /* outside lock */
rdp->n_rp_gp_started++; rdp->n_rp_gp_started++;
return 1; return 1;
} }
@ -3564,7 +3564,7 @@ static void _rcu_barrier(struct rcu_state *rsp)
{ {
int cpu; int cpu;
struct rcu_data *rdp; struct rcu_data *rdp;
unsigned long snap = ACCESS_ONCE(rsp->n_barrier_done); unsigned long snap = READ_ONCE(rsp->n_barrier_done);
unsigned long snap_done; unsigned long snap_done;
_rcu_barrier_trace(rsp, "Begin", -1, snap); _rcu_barrier_trace(rsp, "Begin", -1, snap);
@ -3606,10 +3606,10 @@ static void _rcu_barrier(struct rcu_state *rsp)
/* /*
* Increment ->n_barrier_done to avoid duplicate work. Use * Increment ->n_barrier_done to avoid duplicate work. Use
* ACCESS_ONCE() to prevent the compiler from speculating * WRITE_ONCE() to prevent the compiler from speculating
* the increment to precede the early-exit check. * the increment to precede the early-exit check.
*/ */
ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1; WRITE_ONCE(rsp->n_barrier_done, rsp->n_barrier_done + 1);
WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 1); WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 1);
_rcu_barrier_trace(rsp, "Inc1", -1, rsp->n_barrier_done); _rcu_barrier_trace(rsp, "Inc1", -1, rsp->n_barrier_done);
smp_mb(); /* Order ->n_barrier_done increment with below mechanism. */ smp_mb(); /* Order ->n_barrier_done increment with below mechanism. */
@ -3645,7 +3645,7 @@ static void _rcu_barrier(struct rcu_state *rsp)
__call_rcu(&rdp->barrier_head, __call_rcu(&rdp->barrier_head,
rcu_barrier_callback, rsp, cpu, 0); rcu_barrier_callback, rsp, cpu, 0);
} }
} else if (ACCESS_ONCE(rdp->qlen)) { } else if (READ_ONCE(rdp->qlen)) {
_rcu_barrier_trace(rsp, "OnlineQ", cpu, _rcu_barrier_trace(rsp, "OnlineQ", cpu,
rsp->n_barrier_done); rsp->n_barrier_done);
smp_call_function_single(cpu, rcu_barrier_func, rsp, 1); smp_call_function_single(cpu, rcu_barrier_func, rsp, 1);
@ -3665,7 +3665,7 @@ static void _rcu_barrier(struct rcu_state *rsp)
/* Increment ->n_barrier_done to prevent duplicate work. */ /* Increment ->n_barrier_done to prevent duplicate work. */
smp_mb(); /* Keep increment after above mechanism. */ smp_mb(); /* Keep increment after above mechanism. */
ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1; WRITE_ONCE(rsp->n_barrier_done, rsp->n_barrier_done + 1);
WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 0); WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 0);
_rcu_barrier_trace(rsp, "Inc2", -1, rsp->n_barrier_done); _rcu_barrier_trace(rsp, "Inc2", -1, rsp->n_barrier_done);
smp_mb(); /* Keep increment before caller's subsequent code. */ smp_mb(); /* Keep increment before caller's subsequent code. */

93
kernel/rcu/tree_plugin.h
View File

@ -570,7 +570,7 @@ static int rcu_preempted_readers_exp(struct rcu_node *rnp)
static int sync_rcu_preempt_exp_done(struct rcu_node *rnp) static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
{ {
return !rcu_preempted_readers_exp(rnp) && return !rcu_preempted_readers_exp(rnp) &&
ACCESS_ONCE(rnp->expmask) == 0; READ_ONCE(rnp->expmask) == 0;
} }
/* /*
@ -716,7 +716,7 @@ void synchronize_rcu_expedited(void)
int trycount = 0; int trycount = 0;
smp_mb(); /* Caller's modifications seen first by other CPUs. */ smp_mb(); /* Caller's modifications seen first by other CPUs. */
snap = ACCESS_ONCE(sync_rcu_preempt_exp_count) + 1; snap = READ_ONCE(sync_rcu_preempt_exp_count) + 1;
smp_mb(); /* Above access cannot bleed into critical section. */ smp_mb(); /* Above access cannot bleed into critical section. */
/* /*
@ -740,7 +740,7 @@ void synchronize_rcu_expedited(void)
*/ */
while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) { while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {
if (ULONG_CMP_LT(snap, if (ULONG_CMP_LT(snap,
ACCESS_ONCE(sync_rcu_preempt_exp_count))) { READ_ONCE(sync_rcu_preempt_exp_count))) {
put_online_cpus(); put_online_cpus();
goto mb_ret; /* Others did our work for us. */ goto mb_ret; /* Others did our work for us. */
} }
@ -752,7 +752,7 @@ void synchronize_rcu_expedited(void)
return; return;
} }
} }
if (ULONG_CMP_LT(snap, ACCESS_ONCE(sync_rcu_preempt_exp_count))) { if (ULONG_CMP_LT(snap, READ_ONCE(sync_rcu_preempt_exp_count))) {
put_online_cpus(); put_online_cpus();
goto unlock_mb_ret; /* Others did our work for us. */ goto unlock_mb_ret; /* Others did our work for us. */
} }
@ -780,8 +780,7 @@ void synchronize_rcu_expedited(void)
/* Clean up and exit. */ /* Clean up and exit. */
smp_mb(); /* ensure expedited GP seen before counter increment. */ smp_mb(); /* ensure expedited GP seen before counter increment. */
ACCESS_ONCE(sync_rcu_preempt_exp_count) = WRITE_ONCE(sync_rcu_preempt_exp_count, sync_rcu_preempt_exp_count + 1);
sync_rcu_preempt_exp_count + 1;
unlock_mb_ret: unlock_mb_ret:
mutex_unlock(&sync_rcu_preempt_exp_mutex); mutex_unlock(&sync_rcu_preempt_exp_mutex);
mb_ret: mb_ret:
@ -994,8 +993,8 @@ static int rcu_boost(struct rcu_node *rnp)
struct task_struct *t; struct task_struct *t;
struct list_head *tb; struct list_head *tb;
if (ACCESS_ONCE(rnp->exp_tasks) == NULL && if (READ_ONCE(rnp->exp_tasks) == NULL &&
ACCESS_ONCE(rnp->boost_tasks) == NULL) READ_ONCE(rnp->boost_tasks) == NULL)
return 0; /* Nothing left to boost. */ return 0; /* Nothing left to boost. */
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
@ -1048,8 +1047,8 @@ static int rcu_boost(struct rcu_node *rnp)
rt_mutex_lock(&rnp->boost_mtx); rt_mutex_lock(&rnp->boost_mtx);
rt_mutex_unlock(&rnp->boost_mtx); /* Then keep lockdep happy. */ rt_mutex_unlock(&rnp->boost_mtx); /* Then keep lockdep happy. */
return ACCESS_ONCE(rnp->exp_tasks) != NULL || return READ_ONCE(rnp->exp_tasks) != NULL ||
ACCESS_ONCE(rnp->boost_tasks) != NULL; READ_ONCE(rnp->boost_tasks) != NULL;
} }
/* /*
@ -1462,7 +1461,7 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void)
* callbacks not yet ready to invoke. * callbacks not yet ready to invoke.
*/ */
if ((rdp->completed != rnp->completed || if ((rdp->completed != rnp->completed ||
unlikely(ACCESS_ONCE(rdp->gpwrap))) && unlikely(READ_ONCE(rdp->gpwrap))) &&
rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL]) rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
note_gp_changes(rsp, rdp); note_gp_changes(rsp, rdp);
@ -1534,7 +1533,7 @@ static void rcu_prepare_for_idle(void)
int tne; int tne;
/* Handle nohz enablement switches conservatively. */ /* Handle nohz enablement switches conservatively. */
tne = ACCESS_ONCE(tick_nohz_active); tne = READ_ONCE(tick_nohz_active);
if (tne != rdtp->tick_nohz_enabled_snap) { if (tne != rdtp->tick_nohz_enabled_snap) {
if (rcu_cpu_has_callbacks(NULL)) if (rcu_cpu_has_callbacks(NULL))
invoke_rcu_core(); /* force nohz to see update. */ invoke_rcu_core(); /* force nohz to see update. */
@ -1760,7 +1759,7 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
atomic_read(&rdtp->dynticks) & 0xfff, atomic_read(&rdtp->dynticks) & 0xfff,
rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting, rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
ACCESS_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart, READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
fast_no_hz); fast_no_hz);
} }
@ -1898,11 +1897,11 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force)
{ {
struct rcu_data *rdp_leader = rdp->nocb_leader; struct rcu_data *rdp_leader = rdp->nocb_leader;
if (!ACCESS_ONCE(rdp_leader->nocb_kthread)) if (!READ_ONCE(rdp_leader->nocb_kthread))
return; return;
if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) { if (READ_ONCE(rdp_leader->nocb_leader_sleep) || force) {
/* Prior smp_mb__after_atomic() orders against prior enqueue. */ /* Prior smp_mb__after_atomic() orders against prior enqueue. */
ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false; WRITE_ONCE(rdp_leader->nocb_leader_sleep, false);
wake_up(&rdp_leader->nocb_wq); wake_up(&rdp_leader->nocb_wq);
} }
} }
@ -1934,14 +1933,14 @@ static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
ret = atomic_long_read(&rdp->nocb_q_count); ret = atomic_long_read(&rdp->nocb_q_count);
#ifdef CONFIG_PROVE_RCU #ifdef CONFIG_PROVE_RCU
rhp = ACCESS_ONCE(rdp->nocb_head); rhp = READ_ONCE(rdp->nocb_head);
if (!rhp) if (!rhp)
rhp = ACCESS_ONCE(rdp->nocb_gp_head); rhp = READ_ONCE(rdp->nocb_gp_head);
if (!rhp) if (!rhp)
rhp = ACCESS_ONCE(rdp->nocb_follower_head); rhp = READ_ONCE(rdp->nocb_follower_head);
/* Having no rcuo kthread but CBs after scheduler starts is bad! */ /* Having no rcuo kthread but CBs after scheduler starts is bad! */
if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp && if (!READ_ONCE(rdp->nocb_kthread) && rhp &&
rcu_scheduler_fully_active) { rcu_scheduler_fully_active) {
/* RCU callback enqueued before CPU first came online??? */ /* RCU callback enqueued before CPU first came online??? */
pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n", pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
@ -1975,12 +1974,12 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
atomic_long_add(rhcount, &rdp->nocb_q_count); atomic_long_add(rhcount, &rdp->nocb_q_count);
/* rcu_barrier() relies on ->nocb_q_count add before xchg. */ /* rcu_barrier() relies on ->nocb_q_count add before xchg. */
old_rhpp = xchg(&rdp->nocb_tail, rhtp); old_rhpp = xchg(&rdp->nocb_tail, rhtp);
ACCESS_ONCE(*old_rhpp) = rhp; WRITE_ONCE(*old_rhpp, rhp);
atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy); atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */ smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
/* If we are not being polled and there is a kthread, awaken it ... */ /* If we are not being polled and there is a kthread, awaken it ... */
t = ACCESS_ONCE(rdp->nocb_kthread); t = READ_ONCE(rdp->nocb_kthread);
if (rcu_nocb_poll || !t) { if (rcu_nocb_poll || !t) {
trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
TPS("WakeNotPoll")); TPS("WakeNotPoll"));
@ -2118,7 +2117,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
for (;;) { for (;;) {
wait_event_interruptible( wait_event_interruptible(
rnp->nocb_gp_wq[c & 0x1], rnp->nocb_gp_wq[c & 0x1],
(d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c))); (d = ULONG_CMP_GE(READ_ONCE(rnp->completed), c)));
if (likely(d)) if (likely(d))
break; break;
WARN_ON(signal_pending(current)); WARN_ON(signal_pending(current));
@ -2145,7 +2144,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
if (!rcu_nocb_poll) { if (!rcu_nocb_poll) {
trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
wait_event_interruptible(my_rdp->nocb_wq, wait_event_interruptible(my_rdp->nocb_wq,
!ACCESS_ONCE(my_rdp->nocb_leader_sleep)); !READ_ONCE(my_rdp->nocb_leader_sleep));
/* Memory barrier handled by smp_mb() calls below and repoll. */ /* Memory barrier handled by smp_mb() calls below and repoll. */
} else if (firsttime) { } else if (firsttime) {
firsttime = false; /* Don't drown trace log with "Poll"! */ firsttime = false; /* Don't drown trace log with "Poll"! */
@ -2159,12 +2158,12 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
*/ */
gotcbs = false; gotcbs = false;
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
rdp->nocb_gp_head = ACCESS_ONCE(rdp->nocb_head); rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head);
if (!rdp->nocb_gp_head) if (!rdp->nocb_gp_head)
continue; /* No CBs here, try next follower. */ continue; /* No CBs here, try next follower. */
/* Move callbacks to wait-for-GP list, which is empty. */ /* Move callbacks to wait-for-GP list, which is empty. */
ACCESS_ONCE(rdp->nocb_head) = NULL; WRITE_ONCE(rdp->nocb_head, NULL);
rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head); rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
gotcbs = true; gotcbs = true;
} }
@ -2184,7 +2183,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
my_rdp->nocb_leader_sleep = true; my_rdp->nocb_leader_sleep = true;
smp_mb(); /* Ensure _sleep true before scan. */ smp_mb(); /* Ensure _sleep true before scan. */
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
if (ACCESS_ONCE(rdp->nocb_head)) { if (READ_ONCE(rdp->nocb_head)) {
/* Found CB, so short-circuit next wait. */ /* Found CB, so short-circuit next wait. */
my_rdp->nocb_leader_sleep = false; my_rdp->nocb_leader_sleep = false;
break; break;
@ -2205,7 +2204,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
/* Each pass through the following loop wakes a follower, if needed. */ /* Each pass through the following loop wakes a follower, if needed. */
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
if (ACCESS_ONCE(rdp->nocb_head)) if (READ_ONCE(rdp->nocb_head))
my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/ my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
if (!rdp->nocb_gp_head) if (!rdp->nocb_gp_head)
continue; /* No CBs, so no need to wake follower. */ continue; /* No CBs, so no need to wake follower. */
@ -2241,7 +2240,7 @@ static void nocb_follower_wait(struct rcu_data *rdp)
trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
"FollowerSleep"); "FollowerSleep");
wait_event_interruptible(rdp->nocb_wq, wait_event_interruptible(rdp->nocb_wq,
ACCESS_ONCE(rdp->nocb_follower_head)); READ_ONCE(rdp->nocb_follower_head));
} else if (firsttime) { } else if (firsttime) {
/* Don't drown trace log with "Poll"! */ /* Don't drown trace log with "Poll"! */
firsttime = false; firsttime = false;
@ -2282,10 +2281,10 @@ static int rcu_nocb_kthread(void *arg)
nocb_follower_wait(rdp); nocb_follower_wait(rdp);
/* Pull the ready-to-invoke callbacks onto local list. */ /* Pull the ready-to-invoke callbacks onto local list. */
list = ACCESS_ONCE(rdp->nocb_follower_head); list = READ_ONCE(rdp->nocb_follower_head);
BUG_ON(!list); BUG_ON(!list);
trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty"); trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
ACCESS_ONCE(rdp->nocb_follower_head) = NULL; WRITE_ONCE(rdp->nocb_follower_head, NULL);
tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head); tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
/* Each pass through the following loop invokes a callback. */ /* Each pass through the following loop invokes a callback. */
@ -2324,7 +2323,7 @@ static int rcu_nocb_kthread(void *arg)
/* Is a deferred wakeup of rcu_nocb_kthread() required? */ /* Is a deferred wakeup of rcu_nocb_kthread() required? */
static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp) static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
{ {
return ACCESS_ONCE(rdp->nocb_defer_wakeup); return READ_ONCE(rdp->nocb_defer_wakeup);
} }
/* Do a deferred wakeup of rcu_nocb_kthread(). */ /* Do a deferred wakeup of rcu_nocb_kthread(). */
@ -2334,8 +2333,8 @@ static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
if (!rcu_nocb_need_deferred_wakeup(rdp)) if (!rcu_nocb_need_deferred_wakeup(rdp))
return; return;
ndw = ACCESS_ONCE(rdp->nocb_defer_wakeup); ndw = READ_ONCE(rdp->nocb_defer_wakeup);
ACCESS_ONCE(rdp->nocb_defer_wakeup) = RCU_NOGP_WAKE_NOT; WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE_NOT);
wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE); wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake")); trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
} }
@ -2448,7 +2447,7 @@ static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
t = kthread_run(rcu_nocb_kthread, rdp_spawn, t = kthread_run(rcu_nocb_kthread, rdp_spawn,
"rcuo%c/%d", rsp->abbr, cpu); "rcuo%c/%d", rsp->abbr, cpu);
BUG_ON(IS_ERR(t)); BUG_ON(IS_ERR(t));
ACCESS_ONCE(rdp_spawn->nocb_kthread) = t; WRITE_ONCE(rdp_spawn->nocb_kthread, t);
} }
/* /*
@ -2663,7 +2662,7 @@ static void rcu_sysidle_enter(int irq)
/* Record start of fully idle period. */ /* Record start of fully idle period. */
j = jiffies; j = jiffies;
ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j; WRITE_ONCE(rdtp->dynticks_idle_jiffies, j);
smp_mb__before_atomic(); smp_mb__before_atomic();
atomic_inc(&rdtp->dynticks_idle); atomic_inc(&rdtp->dynticks_idle);
smp_mb__after_atomic(); smp_mb__after_atomic();
@ -2681,7 +2680,7 @@ static void rcu_sysidle_enter(int irq)
*/ */
void rcu_sysidle_force_exit(void) void rcu_sysidle_force_exit(void)
{ {
int oldstate = ACCESS_ONCE(full_sysidle_state); int oldstate = READ_ONCE(full_sysidle_state);
int newoldstate; int newoldstate;
/* /*
@ -2794,7 +2793,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
smp_mb(); /* Read counters before timestamps. */ smp_mb(); /* Read counters before timestamps. */
/* Pick up timestamps. */ /* Pick up timestamps. */
j = ACCESS_ONCE(rdtp->dynticks_idle_jiffies); j = READ_ONCE(rdtp->dynticks_idle_jiffies);
/* If this CPU entered idle more recently, update maxj timestamp. */ /* If this CPU entered idle more recently, update maxj timestamp. */
if (ULONG_CMP_LT(*maxj, j)) if (ULONG_CMP_LT(*maxj, j))
*maxj = j; *maxj = j;
@ -2831,11 +2830,11 @@ static unsigned long rcu_sysidle_delay(void)
static void rcu_sysidle(unsigned long j) static void rcu_sysidle(unsigned long j)
{ {
/* Check the current state. */ /* Check the current state. */
switch (ACCESS_ONCE(full_sysidle_state)) { switch (READ_ONCE(full_sysidle_state)) {
case RCU_SYSIDLE_NOT: case RCU_SYSIDLE_NOT:
/* First time all are idle, so note a short idle period. */ /* First time all are idle, so note a short idle period. */
ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_SHORT; WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_SHORT);
break; break;
case RCU_SYSIDLE_SHORT: case RCU_SYSIDLE_SHORT:
@ -2873,7 +2872,7 @@ static void rcu_sysidle_cancel(void)
{ {
smp_mb(); smp_mb();
if (full_sysidle_state > RCU_SYSIDLE_SHORT) if (full_sysidle_state > RCU_SYSIDLE_SHORT)
ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_NOT; WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_NOT);
} }
/* /*
@ -2925,7 +2924,7 @@ static void rcu_sysidle_cb(struct rcu_head *rhp)
smp_mb(); /* grace period precedes setting inuse. */ smp_mb(); /* grace period precedes setting inuse. */
rshp = container_of(rhp, struct rcu_sysidle_head, rh); rshp = container_of(rhp, struct rcu_sysidle_head, rh);
ACCESS_ONCE(rshp->inuse) = 0; WRITE_ONCE(rshp->inuse, 0);
} }
/* /*
@ -2936,7 +2935,7 @@ static void rcu_sysidle_cb(struct rcu_head *rhp)
bool rcu_sys_is_idle(void) bool rcu_sys_is_idle(void)
{ {
static struct rcu_sysidle_head rsh; static struct rcu_sysidle_head rsh;
int rss = ACCESS_ONCE(full_sysidle_state); int rss = READ_ONCE(full_sysidle_state);
if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu)) if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
return false; return false;
@ -2964,7 +2963,7 @@ bool rcu_sys_is_idle(void)
} }
rcu_sysidle_report(rcu_state_p, isidle, maxj, false); rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
oldrss = rss; oldrss = rss;
rss = ACCESS_ONCE(full_sysidle_state); rss = READ_ONCE(full_sysidle_state);
} }
} }
@ -3048,7 +3047,7 @@ static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
#ifdef CONFIG_NO_HZ_FULL #ifdef CONFIG_NO_HZ_FULL
if (tick_nohz_full_cpu(smp_processor_id()) && if (tick_nohz_full_cpu(smp_processor_id()) &&
(!rcu_gp_in_progress(rsp) || (!rcu_gp_in_progress(rsp) ||
ULONG_CMP_LT(jiffies, ACCESS_ONCE(rsp->gp_start) + HZ))) ULONG_CMP_LT(jiffies, READ_ONCE(rsp->gp_start) + HZ)))
return 1; return 1;
#endif /* #ifdef CONFIG_NO_HZ_FULL */ #endif /* #ifdef CONFIG_NO_HZ_FULL */
return 0; return 0;
@ -3077,7 +3076,7 @@ static void rcu_bind_gp_kthread(void)
static void rcu_dynticks_task_enter(void) static void rcu_dynticks_task_enter(void)
{ {
#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
ACCESS_ONCE(current->rcu_tasks_idle_cpu) = smp_processor_id(); WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
} }
@ -3085,6 +3084,6 @@ static void rcu_dynticks_task_enter(void)
static void rcu_dynticks_task_exit(void) static void rcu_dynticks_task_exit(void)
{ {
#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
ACCESS_ONCE(current->rcu_tasks_idle_cpu) = -1; WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */ #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
} }

6
kernel/rcu/tree_trace.c
View File

@ -277,7 +277,7 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
seq_printf(m, "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld/%ld\n", seq_printf(m, "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld/%ld\n",
rsp->n_force_qs, rsp->n_force_qs_ngp, rsp->n_force_qs, rsp->n_force_qs_ngp,
rsp->n_force_qs - rsp->n_force_qs_ngp, rsp->n_force_qs - rsp->n_force_qs_ngp,
ACCESS_ONCE(rsp->n_force_qs_lh), rsp->qlen_lazy, rsp->qlen); READ_ONCE(rsp->n_force_qs_lh), rsp->qlen_lazy, rsp->qlen);
for (rnp = &rsp->node[0]; rnp - &rsp->node[0] < rcu_num_nodes; rnp++) { for (rnp = &rsp->node[0]; rnp - &rsp->node[0] < rcu_num_nodes; rnp++) {
if (rnp->level != level) { if (rnp->level != level) {
seq_puts(m, "\n"); seq_puts(m, "\n");
@ -323,8 +323,8 @@ static void show_one_rcugp(struct seq_file *m, struct rcu_state *rsp)
struct rcu_node *rnp = &rsp->node[0]; struct rcu_node *rnp = &rsp->node[0];
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
completed = ACCESS_ONCE(rsp->completed); completed = READ_ONCE(rsp->completed);
gpnum = ACCESS_ONCE(rsp->gpnum); gpnum = READ_ONCE(rsp->gpnum);
if (completed == gpnum) if (completed == gpnum)
gpage = 0; gpage = 0;
else else

30
kernel/rcu/update.c
View File

@ -150,14 +150,14 @@ void __rcu_read_unlock(void)
barrier(); /* critical section before exit code. */ barrier(); /* critical section before exit code. */
t->rcu_read_lock_nesting = INT_MIN; t->rcu_read_lock_nesting = INT_MIN;
barrier(); /* assign before ->rcu_read_unlock_special load */ barrier(); /* assign before ->rcu_read_unlock_special load */
if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special.s))) if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s)))
rcu_read_unlock_special(t); rcu_read_unlock_special(t);
barrier(); /* ->rcu_read_unlock_special load before assign */ barrier(); /* ->rcu_read_unlock_special load before assign */
t->rcu_read_lock_nesting = 0; t->rcu_read_lock_nesting = 0;
} }
#ifdef CONFIG_PROVE_LOCKING #ifdef CONFIG_PROVE_LOCKING
{ {
int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting); int rrln = READ_ONCE(t->rcu_read_lock_nesting);
WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2); WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
} }
@ -389,17 +389,17 @@ module_param(rcu_cpu_stall_timeout, int, 0644);
int rcu_jiffies_till_stall_check(void) int rcu_jiffies_till_stall_check(void)
{ {
int till_stall_check = ACCESS_ONCE(rcu_cpu_stall_timeout); int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout);
/* /*
* Limit check must be consistent with the Kconfig limits * Limit check must be consistent with the Kconfig limits
* for CONFIG_RCU_CPU_STALL_TIMEOUT. * for CONFIG_RCU_CPU_STALL_TIMEOUT.
*/ */
if (till_stall_check < 3) { if (till_stall_check < 3) {
ACCESS_ONCE(rcu_cpu_stall_timeout) = 3; WRITE_ONCE(rcu_cpu_stall_timeout, 3);
till_stall_check = 3; till_stall_check = 3;
} else if (till_stall_check > 300) { } else if (till_stall_check > 300) {
ACCESS_ONCE(rcu_cpu_stall_timeout) = 300; WRITE_ONCE(rcu_cpu_stall_timeout, 300);
till_stall_check = 300; till_stall_check = 300;
} }
return till_stall_check * HZ + RCU_STALL_DELAY_DELTA; return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
@ -550,12 +550,12 @@ static void check_holdout_task(struct task_struct *t,
{ {
int cpu; int cpu;
if (!ACCESS_ONCE(t->rcu_tasks_holdout) || if (!READ_ONCE(t->rcu_tasks_holdout) ||
t->rcu_tasks_nvcsw != ACCESS_ONCE(t->nvcsw) || t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) ||
!ACCESS_ONCE(t->on_rq) || !READ_ONCE(t->on_rq) ||
(IS_ENABLED(CONFIG_NO_HZ_FULL) && (IS_ENABLED(CONFIG_NO_HZ_FULL) &&
!is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) { !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {
ACCESS_ONCE(t->rcu_tasks_holdout) = false; WRITE_ONCE(t->rcu_tasks_holdout, false);
list_del_init(&t->rcu_tasks_holdout_list); list_del_init(&t->rcu_tasks_holdout_list);
put_task_struct(t); put_task_struct(t);
return; return;
@ -639,11 +639,11 @@ static int __noreturn rcu_tasks_kthread(void *arg)
*/ */
rcu_read_lock(); rcu_read_lock();
for_each_process_thread(g, t) { for_each_process_thread(g, t) {
if (t != current && ACCESS_ONCE(t->on_rq) && if (t != current && READ_ONCE(t->on_rq) &&
!is_idle_task(t)) { !is_idle_task(t)) {
get_task_struct(t); get_task_struct(t);
t->rcu_tasks_nvcsw = ACCESS_ONCE(t->nvcsw); t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);
ACCESS_ONCE(t->rcu_tasks_holdout) = true; WRITE_ONCE(t->rcu_tasks_holdout, true);
list_add(&t->rcu_tasks_holdout_list, list_add(&t->rcu_tasks_holdout_list,
&rcu_tasks_holdouts); &rcu_tasks_holdouts);
} }
@ -672,7 +672,7 @@ static int __noreturn rcu_tasks_kthread(void *arg)
struct task_struct *t1; struct task_struct *t1;
schedule_timeout_interruptible(HZ); schedule_timeout_interruptible(HZ);
rtst = ACCESS_ONCE(rcu_task_stall_timeout); rtst = READ_ONCE(rcu_task_stall_timeout);
needreport = rtst > 0 && needreport = rtst > 0 &&
time_after(jiffies, lastreport + rtst); time_after(jiffies, lastreport + rtst);
if (needreport) if (needreport)
@ -728,7 +728,7 @@ static void rcu_spawn_tasks_kthread(void)
static struct task_struct *rcu_tasks_kthread_ptr; static struct task_struct *rcu_tasks_kthread_ptr;
struct task_struct *t; struct task_struct *t;
if (ACCESS_ONCE(rcu_tasks_kthread_ptr)) { if (READ_ONCE(rcu_tasks_kthread_ptr)) {
smp_mb(); /* Ensure caller sees full kthread. */ smp_mb(); /* Ensure caller sees full kthread. */
return; return;
} }
@ -740,7 +740,7 @@ static void rcu_spawn_tasks_kthread(void)
t = kthread_run(rcu_tasks_kthread, NULL, "rcu_tasks_kthread"); t = kthread_run(rcu_tasks_kthread, NULL, "rcu_tasks_kthread");
BUG_ON(IS_ERR(t)); BUG_ON(IS_ERR(t));
smp_mb(); /* Ensure others see full kthread. */ smp_mb(); /* Ensure others see full kthread. */
ACCESS_ONCE(rcu_tasks_kthread_ptr) = t; WRITE_ONCE(rcu_tasks_kthread_ptr, t);
mutex_unlock(&rcu_tasks_kthread_mutex); mutex_unlock(&rcu_tasks_kthread_mutex);
} }

26
kernel/torture.c
View File

@ -409,7 +409,7 @@ static void (*torture_shutdown_hook)(void);
*/ */
void torture_shutdown_absorb(const char *title) void torture_shutdown_absorb(const char *title)
{ {
while (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
pr_notice("torture thread %s parking due to system shutdown\n", pr_notice("torture thread %s parking due to system shutdown\n",
title); title);
schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT); schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
@ -480,9 +480,9 @@ static int torture_shutdown_notify(struct notifier_block *unused1,
unsigned long unused2, void *unused3) unsigned long unused2, void *unused3)
{ {
mutex_lock(&fullstop_mutex); mutex_lock(&fullstop_mutex);
if (ACCESS_ONCE(fullstop) == FULLSTOP_DONTSTOP) { if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected"); VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
ACCESS_ONCE(fullstop) = FULLSTOP_SHUTDOWN; WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
} else { } else {
pr_warn("Concurrent rmmod and shutdown illegal!\n"); pr_warn("Concurrent rmmod and shutdown illegal!\n");
} }
@ -523,13 +523,13 @@ static int stutter;
*/ */
void stutter_wait(const char *title) void stutter_wait(const char *title)
{ {
while (ACCESS_ONCE(stutter_pause_test) || while (READ_ONCE(stutter_pause_test) ||
(torture_runnable && !ACCESS_ONCE(*torture_runnable))) { (torture_runnable && !READ_ONCE(*torture_runnable))) {
if (stutter_pause_test) if (stutter_pause_test)
if (ACCESS_ONCE(stutter_pause_test) == 1) if (READ_ONCE(stutter_pause_test) == 1)
schedule_timeout_interruptible(1); schedule_timeout_interruptible(1);
else else
while (ACCESS_ONCE(stutter_pause_test)) while (READ_ONCE(stutter_pause_test))
cond_resched(); cond_resched();
else else
schedule_timeout_interruptible(round_jiffies_relative(HZ)); schedule_timeout_interruptible(round_jiffies_relative(HZ));
@ -549,14 +549,14 @@ static int torture_stutter(void *arg)
if (!torture_must_stop()) { if (!torture_must_stop()) {
if (stutter > 1) { if (stutter > 1) {
schedule_timeout_interruptible(stutter - 1); schedule_timeout_interruptible(stutter - 1);
ACCESS_ONCE(stutter_pause_test) = 2; WRITE_ONCE(stutter_pause_test, 2);
} }
schedule_timeout_interruptible(1); schedule_timeout_interruptible(1);
ACCESS_ONCE(stutter_pause_test) = 1; WRITE_ONCE(stutter_pause_test, 1);
} }
if (!torture_must_stop()) if (!torture_must_stop())
schedule_timeout_interruptible(stutter); schedule_timeout_interruptible(stutter);
ACCESS_ONCE(stutter_pause_test) = 0; WRITE_ONCE(stutter_pause_test, 0);
torture_shutdown_absorb("torture_stutter"); torture_shutdown_absorb("torture_stutter");
} while (!torture_must_stop()); } while (!torture_must_stop());
torture_kthread_stopping("torture_stutter"); torture_kthread_stopping("torture_stutter");
@ -642,13 +642,13 @@ EXPORT_SYMBOL_GPL(torture_init_end);
bool torture_cleanup_begin(void) bool torture_cleanup_begin(void)
{ {
mutex_lock(&fullstop_mutex); mutex_lock(&fullstop_mutex);
if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
pr_warn("Concurrent rmmod and shutdown illegal!\n"); pr_warn("Concurrent rmmod and shutdown illegal!\n");
mutex_unlock(&fullstop_mutex); mutex_unlock(&fullstop_mutex);
schedule_timeout_uninterruptible(10); schedule_timeout_uninterruptible(10);
return true; return true;
} }
ACCESS_ONCE(fullstop) = FULLSTOP_RMMOD; WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
mutex_unlock(&fullstop_mutex); mutex_unlock(&fullstop_mutex);
torture_shutdown_cleanup(); torture_shutdown_cleanup();
torture_shuffle_cleanup(); torture_shuffle_cleanup();
@ -681,7 +681,7 @@ EXPORT_SYMBOL_GPL(torture_must_stop);
*/ */
bool torture_must_stop_irq(void) bool torture_must_stop_irq(void)
{ {
return ACCESS_ONCE(fullstop) != FULLSTOP_DONTSTOP; return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
} }
EXPORT_SYMBOL_GPL(torture_must_stop_irq); EXPORT_SYMBOL_GPL(torture_must_stop_irq);