lk/linux-kernel-test
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Revert "rcu: Decrease memory-barrier usage based on semi-formal proof"

Browse Source 
This reverts commit e59fb3120b.

This reversion was due to (extreme) boot-time slowdowns on SPARC seen by
Yinghai Lu and on x86 by Ingo
.
This is a non-trivial reversion due to intervening commits.

Conflicts:

	Documentation/RCU/trace.txt
	kernel/rcutree.c

Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Paul E. McKenney
2011-05-12 01:08:07 -07:00
committed by Ingo Molnar
parent 11c476f31a
commit 80d02085d9
5 changed files with 102 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
kernel/rcutree.c
View File

@@ -162,7 +162,7 @@ EXPORT_SYMBOL_GPL(rcu_note_context_switch);
#ifdef CONFIG_NO_HZ
DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
.dynticks_nesting = 1,
.dynticks = ATOMIC_INIT(1),
.dynticks = 1,
};
#endif /* #ifdef CONFIG_NO_HZ */
@@ -321,25 +321,13 @@ void rcu_enter_nohz(void)
unsigned long flags;
struct rcu_dynticks *rdtp;
smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
local_irq_save(flags);
rdtp = &__get_cpu_var(rcu_dynticks);
if (--rdtp->dynticks_nesting) {
local_irq_restore(flags);
return;
}
/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
smp_mb__before_atomic_inc(); /* See above. */
atomic_inc(&rdtp->dynticks);
smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */
WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
rdtp->dynticks++;
rdtp->dynticks_nesting--;
WARN_ON_ONCE(rdtp->dynticks & 0x1);
local_irq_restore(flags);
/* If the interrupt queued a callback, get out of dyntick mode. */
if (in_irq() &&
(__get_cpu_var(rcu_sched_data).nxtlist ||
__get_cpu_var(rcu_bh_data).nxtlist ||
rcu_preempt_needs_cpu(smp_processor_id())))
set_need_resched();
}
/*
@@ -355,16 +343,11 @@ void rcu_exit_nohz(void)
local_irq_save(flags);
rdtp = &__get_cpu_var(rcu_dynticks);
if (rdtp->dynticks_nesting++) {
local_irq_restore(flags);
return;
}
smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */
atomic_inc(&rdtp->dynticks);
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
smp_mb__after_atomic_inc(); /* See above. */
WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
rdtp->dynticks++;
rdtp->dynticks_nesting++;
WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
local_irq_restore(flags);
smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
}
/**
@@ -378,15 +361,11 @@ void rcu_nmi_enter(void)
{
struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
if (rdtp->dynticks_nmi_nesting == 0 &&
(atomic_read(&rdtp->dynticks) & 0x1))
if (rdtp->dynticks & 0x1)
return;
rdtp->dynticks_nmi_nesting++;
smp_mb__before_atomic_inc(); /* Force delay from prior write. */
atomic_inc(&rdtp->dynticks);
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
smp_mb__after_atomic_inc(); /* See above. */
WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
rdtp->dynticks_nmi++;
WARN_ON_ONCE(!(rdtp->dynticks_nmi & 0x1));
smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
}
/**
@@ -400,14 +379,11 @@ void rcu_nmi_exit(void)
{
struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
if (rdtp->dynticks_nmi_nesting == 0 ||
--rdtp->dynticks_nmi_nesting != 0)
if (rdtp->dynticks & 0x1)
return;
/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
smp_mb__before_atomic_inc(); /* See above. */
atomic_inc(&rdtp->dynticks);
smp_mb__after_atomic_inc(); /* Force delay to next write. */
WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
rdtp->dynticks_nmi++;
WARN_ON_ONCE(rdtp->dynticks_nmi & 0x1);
}
/**
@@ -418,7 +394,13 @@ void rcu_nmi_exit(void)
*/
void rcu_irq_enter(void)
{
rcu_exit_nohz();
struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
if (rdtp->dynticks_nesting++)
return;
rdtp->dynticks++;
WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
}
/**
@@ -430,7 +412,18 @@ void rcu_irq_enter(void)
*/
void rcu_irq_exit(void)
{
rcu_enter_nohz();
struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
if (--rdtp->dynticks_nesting)
return;
smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
rdtp->dynticks++;
WARN_ON_ONCE(rdtp->dynticks & 0x1);
/* If the interrupt queued a callback, get out of dyntick mode. */
if (__this_cpu_read(rcu_sched_data.nxtlist) ||
__this_cpu_read(rcu_bh_data.nxtlist))
set_need_resched();
}
#ifdef CONFIG_SMP
@@ -442,8 +435,19 @@ void rcu_irq_exit(void)
*/
static int dyntick_save_progress_counter(struct rcu_data *rdp)
{
rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
return 0;
int ret;
int snap;
int snap_nmi;
snap = rdp->dynticks->dynticks;
snap_nmi = rdp->dynticks->dynticks_nmi;
smp_mb(); /* Order sampling of snap with end of grace period. */
rdp->dynticks_snap = snap;
rdp->dynticks_nmi_snap = snap_nmi;
ret = ((snap & 0x1) == 0) && ((snap_nmi & 0x1) == 0);
if (ret)
rdp->dynticks_fqs++;
return ret;
}
/*
@@ -454,11 +458,16 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp)
*/
static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
{
unsigned long curr;
unsigned long snap;
long curr;
long curr_nmi;
long snap;
long snap_nmi;
curr = (unsigned long)atomic_add_return(0, &rdp->dynticks->dynticks);
snap = (unsigned long)rdp->dynticks_snap;
curr = rdp->dynticks->dynticks;
snap = rdp->dynticks_snap;
curr_nmi = rdp->dynticks->dynticks_nmi;
snap_nmi = rdp->dynticks_nmi_snap;
smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
/*
* If the CPU passed through or entered a dynticks idle phase with
@@ -468,7 +477,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
* read-side critical section that started before the beginning
* of the current RCU grace period.
*/
if ((curr & 0x1) == 0 || ULONG_CMP_GE(curr, snap + 2)) {
if ((curr != snap || (curr & 0x1) == 0) &&
(curr_nmi != snap_nmi || (curr_nmi & 0x1) == 0)) {
rdp->dynticks_fqs++;
return 1;
}
@@ -897,12 +907,6 @@ static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
unsigned long gp_duration;
WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
/*
* Ensure that all grace-period and pre-grace-period activity
* is seen before the assignment to rsp->completed.
*/
smp_mb(); /* See above block comment. */
gp_duration = jiffies - rsp->gp_start;
if (gp_duration > rsp->gp_max)
rsp->gp_max = gp_duration;
@@ -1450,11 +1454,25 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
*/
static void rcu_process_callbacks(void)
{
/*
* Memory references from any prior RCU read-side critical sections
* executed by the interrupted code must be seen before any RCU
* grace-period manipulations below.
*/
smp_mb(); /* See above block comment. */
__rcu_process_callbacks(&rcu_sched_state,
&__get_cpu_var(rcu_sched_data));
__rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
rcu_preempt_process_callbacks();
/*
* Memory references from any later RCU read-side critical sections
* executed by the interrupted code must be seen after any RCU
* grace-period manipulations above.
*/
smp_mb(); /* See above block comment. */
/* If we are last CPU on way to dyntick-idle mode, accelerate it. */
rcu_needs_cpu_flush();
}