b4ecc12699
Xiaohui Xin and some other folks at Intel have been looking into what's
behind the performance hit of paravirt_ops when running native.
It appears that the hit is entirely due to the paravirtualized
spinlocks introduced by:
| commit 8efcbab674
| Date: Mon Jul 7 12:07:51 2008 -0700
|
| paravirt: introduce a "lock-byte" spinlock implementation
The extra call/return in the spinlock path is somehow
causing an increase in the cycles/instruction of somewhere around 2-7%
(seems to vary quite a lot from test to test). The working theory is
that the CPU's pipeline is getting upset about the
call->call->locked-op->return->return, and seems to be failing to
speculate (though I haven't seen anything definitive about the precise
reasons). This doesn't entirely make sense, because the performance
hit is also visible on unlock and other operations which don't involve
locked instructions. But spinlock operations clearly swamp all the
other pvops operations, even though I can't imagine that they're
nearly as common (there's only a .05% increase in instructions
executed).
If I disable just the pv-spinlock calls, my tests show that pvops is
identical to non-pvops performance on native (my measurements show that
it is actually about .1% faster, but Xiaohui shows a .05% slowdown).
Summary of results, averaging 10 runs of the "mmperf" test, using a
no-pvops build as baseline:
nopv Pv-nospin Pv-spin
CPU cycles 100.00% 99.89% 102.18%
instructions 100.00% 100.10% 100.15%
CPI 100.00% 99.79% 102.03%
cache ref 100.00% 100.84% 100.28%
cache miss 100.00% 90.47% 88.56%
cache miss rate 100.00% 89.72% 88.31%
branches 100.00% 99.93% 100.04%
branch miss 100.00% 103.66% 107.72%
branch miss rt 100.00% 103.73% 107.67%
wallclock 100.00% 99.90% 102.20%
The clear effect here is that the 2% increase in CPI is
directly reflected in the final wallclock time.
(The other interesting effect is that the more ops are
out of line calls via pvops, the lower the cache access
and miss rates. Not too surprising, but it suggests that
the non-pvops kernel is over-inlined. On the flipside,
the branch misses go up correspondingly...)
So, what's the fix?
Paravirt patching turns all the pvops calls into direct calls, so
_spin_lock etc do end up having direct calls. For example, the compiler
generated code for paravirtualized _spin_lock is:
<_spin_lock+0>: mov %gs:0xb4c8,%rax
<_spin_lock+9>: incl 0xffffffffffffe044(%rax)
<_spin_lock+15>: callq *0xffffffff805a5b30
<_spin_lock+22>: retq
The indirect call will get patched to:
<_spin_lock+0>: mov %gs:0xb4c8,%rax
<_spin_lock+9>: incl 0xffffffffffffe044(%rax)
<_spin_lock+15>: callq <__ticket_spin_lock>
<_spin_lock+20>: nop; nop /* or whatever 2-byte nop */
<_spin_lock+22>: retq
One possibility is to inline _spin_lock, etc, when building an
optimised kernel (ie, when there's no spinlock/preempt
instrumentation/debugging enabled). That will remove the outer
call/return pair, returning the instruction stream to a single
call/return, which will presumably execute the same as the non-pvops
case. The downsides arel 1) it will replicate the
preempt_disable/enable code at eack lock/unlock callsite; this code is
fairly small, but not nothing; and 2) the spinlock definitions are
already a very heavily tangled mass of #ifdefs and other preprocessor
magic, and making any changes will be non-trivial.
The other obvious answer is to disable pv-spinlocks. Making them a
separate config option is fairly easy, and it would be trivial to
enable them only when Xen is enabled (as the only non-default user).
But it doesn't really address the common case of a distro build which
is going to have Xen support enabled, and leaves the open question of
whether the native performance cost of pv-spinlocks is worth the
performance improvement on a loaded Xen system (10% saving of overall
system CPU when guests block rather than spin). Still it is a
reasonable short-term workaround.
[ Impact: fix pvops performance regression when running native ]
Analysed-by: "Xin Xiaohui" <xiaohui.xin@intel.com>
Analysed-by: "Li Xin" <xin.li@intel.com>
Analysed-by: "Nakajima Jun" <jun.nakajima@intel.com>
Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Acked-by: H. Peter Anvin <hpa@zytor.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Xen-devel <xen-devel@lists.xensource.com>
LKML-Reference: <4A0B62F7.5030802@goop.org>
[ fixed the help text ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
106 lines
2.7 KiB
C
106 lines
2.7 KiB
C
#ifndef XEN_OPS_H
|
|
#define XEN_OPS_H
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/clocksource.h>
|
|
#include <linux/irqreturn.h>
|
|
#include <xen/xen-ops.h>
|
|
|
|
/* These are code, but not functions. Defined in entry.S */
|
|
extern const char xen_hypervisor_callback[];
|
|
extern const char xen_failsafe_callback[];
|
|
|
|
extern void *xen_initial_gdt;
|
|
|
|
struct trap_info;
|
|
void xen_copy_trap_info(struct trap_info *traps);
|
|
|
|
DECLARE_PER_CPU(struct vcpu_info, xen_vcpu_info);
|
|
DECLARE_PER_CPU(unsigned long, xen_cr3);
|
|
DECLARE_PER_CPU(unsigned long, xen_current_cr3);
|
|
|
|
extern struct start_info *xen_start_info;
|
|
extern struct shared_info xen_dummy_shared_info;
|
|
extern struct shared_info *HYPERVISOR_shared_info;
|
|
|
|
void xen_setup_mfn_list_list(void);
|
|
void xen_setup_shared_info(void);
|
|
void xen_setup_machphys_mapping(void);
|
|
pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn);
|
|
void xen_ident_map_ISA(void);
|
|
void xen_reserve_top(void);
|
|
|
|
void xen_leave_lazy(void);
|
|
void xen_post_allocator_init(void);
|
|
|
|
char * __init xen_memory_setup(void);
|
|
void __init xen_arch_setup(void);
|
|
void __init xen_init_IRQ(void);
|
|
void xen_enable_sysenter(void);
|
|
void xen_enable_syscall(void);
|
|
void xen_vcpu_restore(void);
|
|
|
|
void __init xen_build_dynamic_phys_to_machine(void);
|
|
|
|
void xen_init_irq_ops(void);
|
|
void xen_setup_timer(int cpu);
|
|
void xen_teardown_timer(int cpu);
|
|
cycle_t xen_clocksource_read(void);
|
|
void xen_setup_cpu_clockevents(void);
|
|
unsigned long xen_tsc_khz(void);
|
|
void __init xen_time_init(void);
|
|
unsigned long xen_get_wallclock(void);
|
|
int xen_set_wallclock(unsigned long time);
|
|
unsigned long long xen_sched_clock(void);
|
|
|
|
irqreturn_t xen_debug_interrupt(int irq, void *dev_id);
|
|
|
|
bool xen_vcpu_stolen(int vcpu);
|
|
|
|
void xen_setup_vcpu_info_placement(void);
|
|
|
|
#ifdef CONFIG_SMP
|
|
void xen_smp_init(void);
|
|
|
|
extern cpumask_var_t xen_cpu_initialized_map;
|
|
#else
|
|
static inline void xen_smp_init(void) {}
|
|
#endif
|
|
|
|
#ifdef CONFIG_PARAVIRT_SPINLOCKS
|
|
void __init xen_init_spinlocks(void);
|
|
__cpuinit void xen_init_lock_cpu(int cpu);
|
|
void xen_uninit_lock_cpu(int cpu);
|
|
#else
|
|
static inline void xen_init_spinlocks(void)
|
|
{
|
|
}
|
|
static inline void xen_init_lock_cpu(int cpu)
|
|
{
|
|
}
|
|
static inline void xen_uninit_lock_cpu(int cpu)
|
|
{
|
|
}
|
|
#endif
|
|
|
|
/* Declare an asm function, along with symbols needed to make it
|
|
inlineable */
|
|
#define DECL_ASM(ret, name, ...) \
|
|
ret name(__VA_ARGS__); \
|
|
extern char name##_end[]; \
|
|
extern char name##_reloc[] \
|
|
|
|
DECL_ASM(void, xen_irq_enable_direct, void);
|
|
DECL_ASM(void, xen_irq_disable_direct, void);
|
|
DECL_ASM(unsigned long, xen_save_fl_direct, void);
|
|
DECL_ASM(void, xen_restore_fl_direct, unsigned long);
|
|
|
|
/* These are not functions, and cannot be called normally */
|
|
void xen_iret(void);
|
|
void xen_sysexit(void);
|
|
void xen_sysret32(void);
|
|
void xen_sysret64(void);
|
|
void xen_adjust_exception_frame(void);
|
|
|
|
#endif /* XEN_OPS_H */
|