From 307fe90fb1d086dfcbcb01dfb37bf074256407e6 Mon Sep 17 00:00:00 2001 From: Luiz Capitulino Date: Fri, 27 May 2016 15:03:28 +0200 Subject: [PATCH 103/352] mm: perform lru_add_drain_all() remotely lru_add_drain_all() works by scheduling lru_add_drain_cpu() to run on all CPUs that have non-empty LRU pagevecs and then waiting for the scheduled work to complete. However, workqueue threads may never have the chance to run on a CPU that's running a SCHED_FIFO task. This causes lru_add_drain_all() to block forever. This commit solves this problem by changing lru_add_drain_all() to drain the LRU pagevecs of remote CPUs. This is done by grabbing swapvec_lock and calling lru_add_drain_cpu(). PS: This is based on an idea and initial implementation by Rik van Riel. Signed-off-by: Rik van Riel Signed-off-by: Luiz Capitulino Signed-off-by: Sebastian Andrzej Siewior --- mm/swap.c | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/mm/swap.c b/mm/swap.c index 796f366..e360916 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -597,9 +597,15 @@ void lru_add_drain_cpu(int cpu) unsigned long flags; /* No harm done if a racing interrupt already did this */ +#ifdef CONFIG_PREEMPT_RT_BASE + local_lock_irqsave_on(rotate_lock, flags, cpu); + pagevec_move_tail(pvec); + local_unlock_irqrestore_on(rotate_lock, flags, cpu); +#else local_lock_irqsave(rotate_lock, flags); pagevec_move_tail(pvec); local_unlock_irqrestore(rotate_lock, flags); +#endif } pvec = &per_cpu(lru_deactivate_file_pvecs, cpu); @@ -667,12 +673,15 @@ void lru_add_drain(void) local_unlock_cpu(swapvec_lock); } -static void lru_add_drain_per_cpu(struct work_struct *dummy) +#ifdef CONFIG_PREEMPT_RT_BASE +static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work) { - lru_add_drain(); + local_lock_on(swapvec_lock, cpu); + lru_add_drain_cpu(cpu); + local_unlock_on(swapvec_lock, cpu); } -static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); +#else /* * lru_add_drain_wq is used to do lru_add_drain_all() from a WQ_MEM_RECLAIM @@ -692,6 +701,22 @@ static int __init lru_init(void) } early_initcall(lru_init); +static void lru_add_drain_per_cpu(struct work_struct *dummy) +{ + lru_add_drain(); +} + +static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); +static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work) +{ + struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); + + INIT_WORK(work, lru_add_drain_per_cpu); + queue_work_on(cpu, lru_add_drain_wq, work); + cpumask_set_cpu(cpu, has_work); +} +#endif + void lru_add_drain_all(void) { static DEFINE_MUTEX(lock); @@ -703,21 +728,18 @@ void lru_add_drain_all(void) cpumask_clear(&has_work); for_each_online_cpu(cpu) { - struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); - if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) || pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) || pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) || pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) || - need_activate_page_drain(cpu)) { - INIT_WORK(work, lru_add_drain_per_cpu); - queue_work_on(cpu, lru_add_drain_wq, work); - cpumask_set_cpu(cpu, &has_work); - } + need_activate_page_drain(cpu)) + remote_lru_add_drain(cpu, &has_work); } +#ifndef CONFIG_PREEMPT_RT_BASE for_each_cpu(cpu, &has_work) flush_work(&per_cpu(lru_add_drain_work, cpu)); +#endif put_online_cpus(); mutex_unlock(&lock); -- 2.7.4