mirrors/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-23 08:35:19 -05:00
Code Activity

rcu: Conditionally compile RCU's eqs warnings

Browse source 
This commit applies some warning-omission micro-optimizations to RCU's
various extended-quiescent-state functions, which are on the kernel/user
hotpath for CONFIG_NO_HZ_FULL=y.

Reported-by: Rik van Riel <riel@redhat.com>
Reported by: Mike Galbraith <umgwanakikbuti@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
Paul E. McKenney 2015-05-05 23:04:22 -07:00
parent e72aeafc66
commit 1ce46ee597
2 changed files with 26 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
kernel/rcu/tree.c
View file

@ -620,7 +620,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting); trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
if (!user && !is_idle_task(current)) { if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
!user && !is_idle_task(current)) {
struct task_struct *idle __maybe_unused = struct task_struct *idle __maybe_unused =
idle_task(smp_processor_id()); idle_task(smp_processor_id());
@ -639,7 +640,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
smp_mb__before_atomic(); /* See above. */ smp_mb__before_atomic(); /* See above. */
atomic_inc(&rdtp->dynticks); atomic_inc(&rdtp->dynticks);
smp_mb__after_atomic(); /* Force ordering with next sojourn. */ smp_mb__after_atomic(); /* Force ordering with next sojourn. */
WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
atomic_read(&rdtp->dynticks) & 0x1);
rcu_dynticks_task_enter(); rcu_dynticks_task_enter();
/* /*
@ -665,7 +667,8 @@ static void rcu_eqs_enter(bool user)
rdtp = this_cpu_ptr(&rcu_dynticks); rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting; oldval = rdtp->dynticks_nesting;
WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
(oldval & DYNTICK_TASK_NEST_MASK) == 0);
if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) { if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) {
rdtp->dynticks_nesting = 0; rdtp->dynticks_nesting = 0;
rcu_eqs_enter_common(oldval, user); rcu_eqs_enter_common(oldval, user);
@ -738,7 +741,8 @@ void rcu_irq_exit(void)
rdtp = this_cpu_ptr(&rcu_dynticks); rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting; oldval = rdtp->dynticks_nesting;
rdtp->dynticks_nesting--; rdtp->dynticks_nesting--;
WARN_ON_ONCE(rdtp->dynticks_nesting < 0); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
rdtp->dynticks_nesting < 0);
if (rdtp->dynticks_nesting) if (rdtp->dynticks_nesting)
trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting); trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);
else else
@ -763,10 +767,12 @@ static void rcu_eqs_exit_common(long long oldval, int user)
atomic_inc(&rdtp->dynticks); atomic_inc(&rdtp->dynticks);
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
smp_mb__after_atomic(); /* See above. */ smp_mb__after_atomic(); /* See above. */
WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
!(atomic_read(&rdtp->dynticks) & 0x1));
rcu_cleanup_after_idle(); rcu_cleanup_after_idle();
trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting); trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
if (!user && !is_idle_task(current)) { if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
!user && !is_idle_task(current)) {
struct task_struct *idle __maybe_unused = struct task_struct *idle __maybe_unused =
idle_task(smp_processor_id()); idle_task(smp_processor_id());
@ -790,7 +796,7 @@ static void rcu_eqs_exit(bool user)
rdtp = this_cpu_ptr(&rcu_dynticks); rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting; oldval = rdtp->dynticks_nesting;
WARN_ON_ONCE(oldval < 0); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
if (oldval & DYNTICK_TASK_NEST_MASK) { if (oldval & DYNTICK_TASK_NEST_MASK) {
rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE; rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
} else { } else {
@ -863,7 +869,8 @@ void rcu_irq_enter(void)
rdtp = this_cpu_ptr(&rcu_dynticks); rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting; oldval = rdtp->dynticks_nesting;
rdtp->dynticks_nesting++; rdtp->dynticks_nesting++;
WARN_ON_ONCE(rdtp->dynticks_nesting == 0); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
rdtp->dynticks_nesting == 0);
if (oldval) if (oldval)
trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting); trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);
else else

11
lib/Kconfig.debug
View file

@ -1373,6 +1373,17 @@ config RCU_TRACE
Say Y here if you want to enable RCU tracing Say Y here if you want to enable RCU tracing
Say N if you are unsure. Say N if you are unsure.
config RCU_EQS_DEBUG
bool "Use this when adding any sort of NO_HZ support to your arch"
depends on DEBUG_KERNEL
help
This option provides consistency checks in RCU's handling of
NO_HZ. These checks have proven quite helpful in detecting
bugs in arch-specific NO_HZ code.
Say N here if you need ultimate kernel/user switch latencies
Say Y if you are unsure
endmenu # "RCU Debugging" endmenu # "RCU Debugging"
config DEBUG_BLOCK_EXT_DEVT config DEBUG_BLOCK_EXT_DEVT