From a6537ef3a4046d22c31f10d572bc1e2d88ab0913 Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Fri, 26 May 2017 19:16:07 +0200 Subject: [PATCH 162/352] Revert "timers: Don't wake ktimersoftd on every tick" This reverts commit 032f93cae150a ("timers: Don't wake ktimersoftd on every tick"). The problem is that the look ahead optimization from the tick timer interrupt context can race with the softirq thread expiring timer. As a consequence the temporary hlist heads which hold the to expire timers are overwritten and the timers which are already removed from the wheel bucket for expiry are now dangling w/o a list head. That means those timers never get expired. If one of those timers is canceled the removal operation will result in a hlist corruption. Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior --- kernel/time/timer.c | 95 ++++++++++++++++------------------------------------- 1 file changed, 28 insertions(+), 67 deletions(-) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index c1b88e0..f90e684 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -207,8 +207,6 @@ struct timer_base { bool must_forward_clk; DECLARE_BITMAP(pending_map, WHEEL_SIZE); struct hlist_head vectors[WHEEL_SIZE]; - struct hlist_head expired_lists[LVL_DEPTH]; - int expired_count; } ____cacheline_aligned; static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]); @@ -1366,8 +1364,7 @@ static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long), } } -static inline void __expire_timers(struct timer_base *base, - struct hlist_head *head) +static void expire_timers(struct timer_base *base, struct hlist_head *head) { while (!hlist_empty(head)) { struct timer_list *timer; @@ -1398,38 +1395,21 @@ static inline void __expire_timers(struct timer_base *base, } } -static void expire_timers(struct timer_base *base) -{ - struct hlist_head *head; - - while (base->expired_count--) { - head = base->expired_lists + base->expired_count; - __expire_timers(base, head); - } - base->expired_count = 0; -} - -static void __collect_expired_timers(struct timer_base *base) +static int __collect_expired_timers(struct timer_base *base, + struct hlist_head *heads) { unsigned long clk = base->clk; struct hlist_head *vec; - int i; + int i, levels = 0; unsigned int idx; - /* - * expire_timers() must be called at least once before we can - * collect more timers - */ - if (WARN_ON(base->expired_count)) - return; - for (i = 0; i < LVL_DEPTH; i++) { idx = (clk & LVL_MASK) + i * LVL_SIZE; if (__test_and_clear_bit(idx, base->pending_map)) { vec = base->vectors + idx; - hlist_move_list(vec, - &base->expired_lists[base->expired_count++]); + hlist_move_list(vec, heads++); + levels++; } /* Is it time to look at the next level? */ if (clk & LVL_CLK_MASK) @@ -1437,6 +1417,7 @@ static void __collect_expired_timers(struct timer_base *base) /* Shift clock for the next level granularity */ clk >>= LVL_CLK_SHIFT; } + return levels; } #ifdef CONFIG_NO_HZ_COMMON @@ -1635,7 +1616,8 @@ void timer_clear_idle(void) base->is_idle = false; } -static void collect_expired_timers(struct timer_base *base) +static int collect_expired_timers(struct timer_base *base, + struct hlist_head *heads) { /* * NOHZ optimization. After a long idle sleep we need to forward the @@ -1652,49 +1634,20 @@ static void collect_expired_timers(struct timer_base *base) if (time_after(next, jiffies)) { /* The call site will increment clock! */ base->clk = jiffies - 1; - return; + return 0; } base->clk = next; } - __collect_expired_timers(base); + return __collect_expired_timers(base, heads); } #else -static inline void collect_expired_timers(struct timer_base *base) +static inline int collect_expired_timers(struct timer_base *base, + struct hlist_head *heads) { - __collect_expired_timers(base); + return __collect_expired_timers(base, heads); } #endif -static int find_expired_timers(struct timer_base *base) -{ - const unsigned long int end_clk = jiffies; - - while (!base->expired_count && time_after_eq(end_clk, base->clk)) { - collect_expired_timers(base); - base->clk++; - } - - return base->expired_count; -} - -/* Called from CPU tick routine to quickly collect expired timers */ -static int tick_find_expired(struct timer_base *base) -{ - int count; - - raw_spin_lock(&base->lock); - - if (unlikely(time_after(jiffies, base->clk + HZ))) { - /* defer to ktimersoftd; don't spend too long in irq context */ - count = -1; - } else - count = find_expired_timers(base); - - raw_spin_unlock(&base->lock); - - return count; -} - /* * Called from the timer interrupt handler to charge one tick to the current * process. user_tick is 1 if the tick is user time, 0 for system. @@ -1721,6 +1674,12 @@ void update_process_times(int user_tick) */ static inline void __run_timers(struct timer_base *base) { + struct hlist_head heads[LVL_DEPTH]; + int levels; + + if (!time_after_eq(jiffies, base->clk)) + return; + raw_spin_lock_irq(&base->lock); /* @@ -1739,9 +1698,13 @@ static inline void __run_timers(struct timer_base *base) */ base->must_forward_clk = false; - while (find_expired_timers(base)) - expire_timers(base); + while (time_after_eq(jiffies, base->clk)) { + levels = collect_expired_timers(base, heads); + base->clk++; + while (levels--) + expire_timers(base, heads + levels); + } raw_spin_unlock_irq(&base->lock); wakeup_timer_waiters(base); } @@ -1767,12 +1730,12 @@ void run_local_timers(void) hrtimer_run_queues(); /* Raise the softirq only if required. */ - if (time_before(jiffies, base->clk) || !tick_find_expired(base)) { + if (time_before(jiffies, base->clk)) { if (!IS_ENABLED(CONFIG_NO_HZ_COMMON)) return; /* CPU is awake, so check the deferrable base. */ base++; - if (time_before(jiffies, base->clk) || !tick_find_expired(base)) + if (time_before(jiffies, base->clk)) return; } raise_softirq(TIMER_SOFTIRQ); @@ -1963,7 +1926,6 @@ int timers_dead_cpu(unsigned int cpu) forward_timer_base(new_base); BUG_ON(old_base->running_timer); - BUG_ON(old_base->expired_count); for (i = 0; i < WHEEL_SIZE; i++) migrate_timer_list(new_base, old_base->vectors + i); @@ -1990,7 +1952,6 @@ static void __init init_timer_cpu(int cpu) #ifdef CONFIG_PREEMPT_RT_FULL init_swait_queue_head(&base->wait_for_running_timer); #endif - base->expired_count = 0; } } -- 2.7.4