p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

timers: Make the cpu base lock raw 

The timers cpu base lock could not be converted to a raw spinlock becaue
the lock held time was non-deterministic due to cascading and long lasting
timer wheel traversals.

The rework of the timer wheel to the new non-cascading model removed also
the wheel traversals and the lock held times are deterministic now. This
allows to make the lock raw and thereby unbreaks NOHz* on preempt-RT.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Link: http://lkml.kernel.org/r/20170627161538.30257-1-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
This commit is contained in:
Sebastian Andrzej Siewior 2017-06-27 18:15:38 +02:00 committed by Thomas Gleixner
parent f5b816786f
commit 2287d8664f
1 changed files with 24 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
kernel/time/timer.c
View File

@ -195,7 +195,7 @@ EXPORT_SYMBOL(jiffies_64);
#endif #endif
struct timer_base { struct timer_base {
spinlock_t lock; raw_spinlock_t lock;
struct timer_list *running_timer; struct timer_list *running_timer;
unsigned long clk; unsigned long clk;
unsigned long next_expiry; unsigned long next_expiry;
@ -913,10 +913,10 @@ static struct timer_base *lock_timer_base(struct timer_list *timer,
if (!(tf & TIMER_MIGRATING)) { if (!(tf & TIMER_MIGRATING)) {
base = get_timer_base(tf); base = get_timer_base(tf);
spin_lock_irqsave(&base->lock, *flags); raw_spin_lock_irqsave(&base->lock, *flags);
if (timer->flags == tf) if (timer->flags == tf)
return base; return base;
spin_unlock_irqrestore(&base->lock, *flags); raw_spin_unlock_irqrestore(&base->lock, *flags);
} }
cpu_relax(); cpu_relax();
} }
@ -986,9 +986,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
/* See the comment in lock_timer_base() */ /* See the comment in lock_timer_base() */
timer->flags |= TIMER_MIGRATING; timer->flags |= TIMER_MIGRATING;
spin_unlock(&base->lock); raw_spin_unlock(&base->lock);
base = new_base; base = new_base;
spin_lock(&base->lock); raw_spin_lock(&base->lock);
WRITE_ONCE(timer->flags, WRITE_ONCE(timer->flags,
(timer->flags & ~TIMER_BASEMASK) | base->cpu); (timer->flags & ~TIMER_BASEMASK) | base->cpu);
} }
@ -1013,7 +1013,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
} }
out_unlock: out_unlock:
spin_unlock_irqrestore(&base->lock, flags); raw_spin_unlock_irqrestore(&base->lock, flags);
return ret; return ret;
} }
@ -1106,16 +1106,16 @@ void add_timer_on(struct timer_list *timer, int cpu)
if (base != new_base) { if (base != new_base) {
timer->flags |= TIMER_MIGRATING; timer->flags |= TIMER_MIGRATING;
spin_unlock(&base->lock); raw_spin_unlock(&base->lock);
base = new_base; base = new_base;
spin_lock(&base->lock); raw_spin_lock(&base->lock);
WRITE_ONCE(timer->flags, WRITE_ONCE(timer->flags,
(timer->flags & ~TIMER_BASEMASK) | cpu); (timer->flags & ~TIMER_BASEMASK) | cpu);
} }
debug_activate(timer, timer->expires); debug_activate(timer, timer->expires);
internal_add_timer(base, timer); internal_add_timer(base, timer);
spin_unlock_irqrestore(&base->lock, flags); raw_spin_unlock_irqrestore(&base->lock, flags);
} }
EXPORT_SYMBOL_GPL(add_timer_on); EXPORT_SYMBOL_GPL(add_timer_on);
@ -1141,7 +1141,7 @@ int del_timer(struct timer_list *timer)
if (timer_pending(timer)) { if (timer_pending(timer)) {
base = lock_timer_base(timer, &flags); base = lock_timer_base(timer, &flags);
ret = detach_if_pending(timer, base, true); ret = detach_if_pending(timer, base, true);
spin_unlock_irqrestore(&base->lock, flags); raw_spin_unlock_irqrestore(&base->lock, flags);
} }
return ret; return ret;
@ -1168,7 +1168,7 @@ int try_to_del_timer_sync(struct timer_list *timer)
if (base->running_timer != timer) if (base->running_timer != timer)
ret = detach_if_pending(timer, base, true); ret = detach_if_pending(timer, base, true);
spin_unlock_irqrestore(&base->lock, flags); raw_spin_unlock_irqrestore(&base->lock, flags);
return ret; return ret;
} }
@ -1299,13 +1299,13 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head)
data = timer->data; data = timer->data;
if (timer->flags & TIMER_IRQSAFE) { if (timer->flags & TIMER_IRQSAFE) {
spin_unlock(&base->lock); raw_spin_unlock(&base->lock);
call_timer_fn(timer, fn, data); call_timer_fn(timer, fn, data);
spin_lock(&base->lock); raw_spin_lock(&base->lock);
} else { } else {
spin_unlock_irq(&base->lock); raw_spin_unlock_irq(&base->lock);
call_timer_fn(timer, fn, data); call_timer_fn(timer, fn, data);
spin_lock_irq(&base->lock); raw_spin_lock_irq(&base->lock);
} }
} }
} }
@ -1474,7 +1474,7 @@ u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
if (cpu_is_offline(smp_processor_id())) if (cpu_is_offline(smp_processor_id()))
return expires; return expires;
spin_lock(&base->lock); raw_spin_lock(&base->lock);
nextevt = __next_timer_interrupt(base); nextevt = __next_timer_interrupt(base);
is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA); is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA);
base->next_expiry = nextevt; base->next_expiry = nextevt;
@ -1502,7 +1502,7 @@ u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
if ((expires - basem) > TICK_NSEC) if ((expires - basem) > TICK_NSEC)
base->is_idle = true; base->is_idle = true;
} }
spin_unlock(&base->lock); raw_spin_unlock(&base->lock);
return cmp_next_hrtimer_event(basem, expires); return cmp_next_hrtimer_event(basem, expires);
} }
@ -1590,7 +1590,7 @@ static inline void __run_timers(struct timer_base *base)
if (!time_after_eq(jiffies, base->clk)) if (!time_after_eq(jiffies, base->clk))
return; return;
spin_lock_irq(&base->lock); raw_spin_lock_irq(&base->lock);
while (time_after_eq(jiffies, base->clk)) { while (time_after_eq(jiffies, base->clk)) {
@ -1601,7 +1601,7 @@ static inline void __run_timers(struct timer_base *base)
expire_timers(base, heads + levels); expire_timers(base, heads + levels);
} }
base->running_timer = NULL; base->running_timer = NULL;
spin_unlock_irq(&base->lock); raw_spin_unlock_irq(&base->lock);
} }
/* /*
@ -1786,16 +1786,16 @@ int timers_dead_cpu(unsigned int cpu)
* The caller is globally serialized and nobody else * The caller is globally serialized and nobody else
* takes two locks at once, deadlock is not possible. * takes two locks at once, deadlock is not possible.
*/ */
spin_lock_irq(&new_base->lock); raw_spin_lock_irq(&new_base->lock);
spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
BUG_ON(old_base->running_timer); BUG_ON(old_base->running_timer);
for (i = 0; i < WHEEL_SIZE; i++) for (i = 0; i < WHEEL_SIZE; i++)
migrate_timer_list(new_base, old_base->vectors + i); migrate_timer_list(new_base, old_base->vectors + i);
spin_unlock(&old_base->lock); raw_spin_unlock(&old_base->lock);
spin_unlock_irq(&new_base->lock); raw_spin_unlock_irq(&new_base->lock);
put_cpu_ptr(&timer_bases); put_cpu_ptr(&timer_bases);
} }
return 0; return 0;
@ -1811,7 +1811,7 @@ static void __init init_timer_cpu(int cpu)
for (i = 0; i < NR_BASES; i++) { for (i = 0; i < NR_BASES; i++) {
base = per_cpu_ptr(&timer_bases[i], cpu); base = per_cpu_ptr(&timer_bases[i], cpu);
base->cpu = cpu; base->cpu = cpu;
spin_lock_init(&base->lock); raw_spin_lock_init(&base->lock);
base->clk = jiffies; base->clk = jiffies;
} }
} }