lk/linux-kernel-test
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branches 'x86/alternatives', 'x86/cleanups', 'x86/commandline', 'x86/crashdump', 'x86/debug', 'x86/defconfig', 'x86/doc', 'x86/exports', 'x86/fpu', 'x86/gart', 'x86/idle', 'x86/mm', 'x86/mtrr', 'x86/nmi-watchdog', 'x86/oprofile', 'x86/paravirt', 'x86/reboot', 'x86/sparse-fixes', 'x86/tsc', 'x86/urgent' and 'x86/vmalloc' into x86-v28-for-linus-phase1

Browse Source
This commit is contained in:
Ingo Molnar
2008-10-06 18:17:07 +02:00
parent b159d7a989 5bbd4c3724 175e438f7a 516cbf3730 af2d237bf5 9b1568458a 5b7e41ff37 1befdefcf4 a03352d2c1 7b22ff5344 2c7e9fd4c6 91030ca1e7 dd5523552c b3e15bdef6 20211e4d34 efd327a2d4 c7ffa6c262 e51a1ac2df 5df4551551 d99e90164e e621bd1895
commit e496e3d645
90 changed files with 1305 additions and 774 deletions
Download Patch File Download Diff File Expand all files Collapse all files

290
arch/x86/kernel/tsc.c
View File

@@ -104,7 +104,7 @@ __setup("notsc", notsc_setup);
/*
* Read TSC and the reference counters. Take care of SMI disturbance
*/
static u64 tsc_read_refs(u64 *pm, u64 *hpet)
static u64 tsc_read_refs(u64 *p, int hpet)
{
u64 t1, t2;
int i;
@@ -112,9 +112,9 @@ static u64 tsc_read_refs(u64 *pm, u64 *hpet)
for (i = 0; i < MAX_RETRIES; i++) {
t1 = get_cycles();
if (hpet)
*hpet = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
*p = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
else
*pm = acpi_pm_read_early();
*p = acpi_pm_read_early();
t2 = get_cycles();
if ((t2 - t1) < SMI_TRESHOLD)
return t2;
@@ -122,6 +122,52 @@ static u64 tsc_read_refs(u64 *pm, u64 *hpet)
return ULLONG_MAX;
}
/*
* Calculate the TSC frequency from HPET reference
*/
static unsigned long calc_hpet_ref(u64 deltatsc, u64 hpet1, u64 hpet2)
{
u64 tmp;
if (hpet2 < hpet1)
hpet2 += 0x100000000ULL;
hpet2 -= hpet1;
tmp = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
do_div(tmp, 1000000);
do_div(deltatsc, tmp);
return (unsigned long) deltatsc;
}
/*
* Calculate the TSC frequency from PMTimer reference
*/
static unsigned long calc_pmtimer_ref(u64 deltatsc, u64 pm1, u64 pm2)
{
u64 tmp;
if (!pm1 && !pm2)
return ULONG_MAX;
if (pm2 < pm1)
pm2 += (u64)ACPI_PM_OVRRUN;
pm2 -= pm1;
tmp = pm2 * 1000000000LL;
do_div(tmp, PMTMR_TICKS_PER_SEC);
do_div(deltatsc, tmp);
return (unsigned long) deltatsc;
}
#define CAL_MS 10
#define CAL_LATCH (CLOCK_TICK_RATE / (1000 / CAL_MS))
#define CAL_PIT_LOOPS 1000
#define CAL2_MS 50
#define CAL2_LATCH (CLOCK_TICK_RATE / (1000 / CAL2_MS))
#define CAL2_PIT_LOOPS 5000
/*
* Try to calibrate the TSC against the Programmable
* Interrupt Timer and return the frequency of the TSC
@@ -129,7 +175,7 @@ static u64 tsc_read_refs(u64 *pm, u64 *hpet)
*
* Return ULONG_MAX on failure to calibrate.
*/
static unsigned long pit_calibrate_tsc(void)
static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin)
{
u64 tsc, t1, t2, delta;
unsigned long tscmin, tscmax;
@@ -144,8 +190,8 @@ static unsigned long pit_calibrate_tsc(void)
* (LSB then MSB) to begin countdown.
*/
outb(0xb0, 0x43);
outb((CLOCK_TICK_RATE / (1000 / 50)) & 0xff, 0x42);
outb((CLOCK_TICK_RATE / (1000 / 50)) >> 8, 0x42);
outb(latch & 0xff, 0x42);
outb(latch >> 8, 0x42);
tsc = t1 = t2 = get_cycles();
@@ -166,31 +212,154 @@ static unsigned long pit_calibrate_tsc(void)
/*
* Sanity checks:
*
* If we were not able to read the PIT more than 5000
* If we were not able to read the PIT more than loopmin
* times, then we have been hit by a massive SMI
*
* If the maximum is 10 times larger than the minimum,
* then we got hit by an SMI as well.
*/
if (pitcnt < 5000 || tscmax > 10 * tscmin)
if (pitcnt < loopmin || tscmax > 10 * tscmin)
return ULONG_MAX;
/* Calculate the PIT value */
delta = t2 - t1;
do_div(delta, 50);
do_div(delta, ms);
return delta;
}
/*
* This reads the current MSB of the PIT counter, and
* checks if we are running on sufficiently fast and
* non-virtualized hardware.
*
* Our expectations are:
*
* - the PIT is running at roughly 1.19MHz
*
* - each IO is going to take about 1us on real hardware,
* but we allow it to be much faster (by a factor of 10) or
* _slightly_ slower (ie we allow up to a 2us read+counter
* update - anything else implies a unacceptably slow CPU
* or PIT for the fast calibration to work.
*
* - with 256 PIT ticks to read the value, we have 214us to
* see the same MSB (and overhead like doing a single TSC
* read per MSB value etc).
*
* - We're doing 2 reads per loop (LSB, MSB), and we expect
* them each to take about a microsecond on real hardware.
* So we expect a count value of around 100. But we'll be
* generous, and accept anything over 50.
*
* - if the PIT is stuck, and we see *many* more reads, we
* return early (and the next caller of pit_expect_msb()
* then consider it a failure when they don't see the
* next expected value).
*
* These expectations mean that we know that we have seen the
* transition from one expected value to another with a fairly
* high accuracy, and we didn't miss any events. We can thus
* use the TSC value at the transitions to calculate a pretty
* good value for the TSC frequencty.
*/
static inline int pit_expect_msb(unsigned char val)
{
int count = 0;
for (count = 0; count < 50000; count++) {
/* Ignore LSB */
inb(0x42);
if (inb(0x42) != val)
break;
}
return count > 50;
}
/*
* How many MSB values do we want to see? We aim for a
* 15ms calibration, which assuming a 2us counter read
* error should give us roughly 150 ppm precision for
* the calibration.
*/
#define QUICK_PIT_MS 15
#define QUICK_PIT_ITERATIONS (QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
static unsigned long quick_pit_calibrate(void)
{
/* Set the Gate high, disable speaker */
outb((inb(0x61) & ~0x02) | 0x01, 0x61);
/*
* Counter 2, mode 0 (one-shot), binary count
*
* NOTE! Mode 2 decrements by two (and then the
* output is flipped each time, giving the same
* final output frequency as a decrement-by-one),
* so mode 0 is much better when looking at the
* individual counts.
*/
outb(0xb0, 0x43);
/* Start at 0xffff */
outb(0xff, 0x42);
outb(0xff, 0x42);
if (pit_expect_msb(0xff)) {
int i;
u64 t1, t2, delta;
unsigned char expect = 0xfe;
t1 = get_cycles();
for (i = 0; i < QUICK_PIT_ITERATIONS; i++, expect--) {
if (!pit_expect_msb(expect))
goto failed;
}
t2 = get_cycles();
/*
* Make sure we can rely on the second TSC timestamp:
*/
if (!pit_expect_msb(expect))
goto failed;
/*
* Ok, if we get here, then we've seen the
* MSB of the PIT decrement QUICK_PIT_ITERATIONS
* times, and each MSB had many hits, so we never
* had any sudden jumps.
*
* As a result, we can depend on there not being
* any odd delays anywhere, and the TSC reads are
* reliable.
*
* kHz = ticks / time-in-seconds / 1000;
* kHz = (t2 - t1) / (QPI * 256 / PIT_TICK_RATE) / 1000
* kHz = ((t2 - t1) * PIT_TICK_RATE) / (QPI * 256 * 1000)
*/
delta = (t2 - t1)*PIT_TICK_RATE;
do_div(delta, QUICK_PIT_ITERATIONS*256*1000);
printk("Fast TSC calibration using PIT\n");
return delta;
}
failed:
return 0;
}
/**
* native_calibrate_tsc - calibrate the tsc on boot
*/
unsigned long native_calibrate_tsc(void)
{
u64 tsc1, tsc2, delta, pm1, pm2, hpet1, hpet2;
u64 tsc1, tsc2, delta, ref1, ref2;
unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
unsigned long flags;
int hpet = is_hpet_enabled(), i;
unsigned long flags, latch, ms, fast_calibrate;
int hpet = is_hpet_enabled(), i, loopmin;
local_irq_save(flags);
fast_calibrate = quick_pit_calibrate();
local_irq_restore(flags);
if (fast_calibrate)
return fast_calibrate;
/*
* Run 5 calibration loops to get the lowest frequency value
@@ -216,7 +385,13 @@ unsigned long native_calibrate_tsc(void)
* calibration delay loop as we have to wait for a certain
* amount of time anyway.
*/
for (i = 0; i < 5; i++) {
/* Preset PIT loop values */
latch = CAL_LATCH;
ms = CAL_MS;
loopmin = CAL_PIT_LOOPS;
for (i = 0; i < 3; i++) {
unsigned long tsc_pit_khz;
/*
@@ -226,16 +401,16 @@ unsigned long native_calibrate_tsc(void)
* read the end value.
*/
local_irq_save(flags);
tsc1 = tsc_read_refs(&pm1, hpet ? &hpet1 : NULL);
tsc_pit_khz = pit_calibrate_tsc();
tsc2 = tsc_read_refs(&pm2, hpet ? &hpet2 : NULL);
tsc1 = tsc_read_refs(&ref1, hpet);
tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin);
tsc2 = tsc_read_refs(&ref2, hpet);
local_irq_restore(flags);
/* Pick the lowest PIT TSC calibration so far */
tsc_pit_min = min(tsc_pit_min, tsc_pit_khz);
/* hpet or pmtimer available ? */
if (!hpet && !pm1 && !pm2)
if (!hpet && !ref1 && !ref2)
continue;
/* Check, whether the sampling was disturbed by an SMI */
@@ -243,23 +418,41 @@ unsigned long native_calibrate_tsc(void)
continue;
tsc2 = (tsc2 - tsc1) * 1000000LL;
if (hpet)
tsc2 = calc_hpet_ref(tsc2, ref1, ref2);
else
tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2);
if (hpet) {
if (hpet2 < hpet1)
hpet2 += 0x100000000ULL;
hpet2 -= hpet1;
tsc1 = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
do_div(tsc1, 1000000);
} else {
if (pm2 < pm1)
pm2 += (u64)ACPI_PM_OVRRUN;
pm2 -= pm1;
tsc1 = pm2 * 1000000000LL;
do_div(tsc1, PMTMR_TICKS_PER_SEC);
tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2);
/* Check the reference deviation */
delta = ((u64) tsc_pit_min) * 100;
do_div(delta, tsc_ref_min);
/*
* If both calibration results are inside a 10% window
* then we can be sure, that the calibration
* succeeded. We break out of the loop right away. We
* use the reference value, as it is more precise.
*/
if (delta >= 90 && delta <= 110) {
printk(KERN_INFO
"TSC: PIT calibration matches %s. %d loops\n",
hpet ? "HPET" : "PMTIMER", i + 1);
return tsc_ref_min;
}
do_div(tsc2, tsc1);
tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2);
/*
* Check whether PIT failed more than once. This
* happens in virtualized environments. We need to
* give the virtual PC a slightly longer timeframe for
* the HPET/PMTIMER to make the result precise.
*/
if (i == 1 && tsc_pit_min == ULONG_MAX) {
latch = CAL2_LATCH;
ms = CAL2_MS;
loopmin = CAL2_PIT_LOOPS;
}
}
/*
@@ -270,7 +463,7 @@ unsigned long native_calibrate_tsc(void)
printk(KERN_WARNING "TSC: Unable to calibrate against PIT\n");
/* We don't have an alternative source, disable TSC */
if (!hpet && !pm1 && !pm2) {
if (!hpet && !ref1 && !ref2) {
printk("TSC: No reference (HPET/PMTIMER) available\n");
return 0;
}
@@ -278,7 +471,7 @@ unsigned long native_calibrate_tsc(void)
/* The alternative source failed as well, disable TSC */
if (tsc_ref_min == ULONG_MAX) {
printk(KERN_WARNING "TSC: HPET/PMTIMER calibration "
"failed due to SMI disturbance.\n");
"failed.\n");
return 0;
}
@@ -290,44 +483,25 @@ unsigned long native_calibrate_tsc(void)
}
/* We don't have an alternative source, use the PIT calibration value */
if (!hpet && !pm1 && !pm2) {
if (!hpet && !ref1 && !ref2) {
printk(KERN_INFO "TSC: Using PIT calibration value\n");
return tsc_pit_min;
}
/* The alternative source failed, use the PIT calibration value */
if (tsc_ref_min == ULONG_MAX) {
printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed due "
"to SMI disturbance. Using PIT calibration\n");
printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed. "
"Using PIT calibration\n");
return tsc_pit_min;
}
/* Check the reference deviation */
delta = ((u64) tsc_pit_min) * 100;
do_div(delta, tsc_ref_min);
/*
* If both calibration results are inside a 5% window, the we
* use the lower frequency of those as it is probably the
* closest estimate.
*/
if (delta >= 95 && delta <= 105) {
printk(KERN_INFO "TSC: PIT calibration confirmed by %s.\n",
hpet ? "HPET" : "PMTIMER");
printk(KERN_INFO "TSC: using %s calibration value\n",
tsc_pit_min <= tsc_ref_min ? "PIT" :
hpet ? "HPET" : "PMTIMER");
return tsc_pit_min <= tsc_ref_min ? tsc_pit_min : tsc_ref_min;
}
printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n",
hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min);
/*
* The calibration values differ too much. In doubt, we use
* the PIT value as we know that there are PMTIMERs around
* running at double speed.
* running at double speed. At least we let the user know:
*/
printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n",
hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min);
printk(KERN_INFO "TSC: Using PIT calibration value\n");
return tsc_pit_min;
}