From ff1b15b646177c6cc465ac2dd0be6ae16e965654 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 09:27:19 -0300 Subject: [PATCH 01/43] x86: don't use size specifiers. Remove the "l" from inline asm at arch/x86/lib/delay_32.c. It is not needed. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/delay_32.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/lib/delay_32.c b/arch/x86/lib/delay_32.c index ef691316f8b6..54013f87d956 100644 --- a/arch/x86/lib/delay_32.c +++ b/arch/x86/lib/delay_32.c @@ -38,9 +38,9 @@ static void delay_loop(unsigned long loops) "1: jmp 2f \n" ".align 16 \n" - "2: decl %0 \n" + "2: dec %0 \n" " jnz 2b \n" - "3: decl %0 \n" + "3: dec %0 \n" : /* we don't need output */ :"a" (loops) From 0a4d8a472f645d99f86303db1462b64e371b090d Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 09:34:08 -0300 Subject: [PATCH 02/43] x86: provide delay loop for x86_64. This is for consistency with i386. We call use_tsc_delay() at tsc initialization for x86_64, so we'll be always using it. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/kernel/tsc.c | 1 + arch/x86/lib/delay_64.c | 44 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 3c36f92160c9..4a775d001957 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -513,6 +513,7 @@ void __init tsc_init(void) */ for_each_possible_cpu(cpu) set_cyc2ns_scale(cpu_khz, cpu); + use_tsc_delay(); if (tsc_disabled > 0) return; diff --git a/arch/x86/lib/delay_64.c b/arch/x86/lib/delay_64.c index 4c441be92641..d0326d07c845 100644 --- a/arch/x86/lib/delay_64.c +++ b/arch/x86/lib/delay_64.c @@ -22,13 +22,28 @@ #include #endif -int __devinit read_current_timer(unsigned long *timer_value) +/* simple loop based delay: */ +static void delay_loop(unsigned long loops) { - rdtscll(*timer_value); - return 0; + asm volatile( + " test %0,%0 \n" + " jz 3f \n" + " jmp 1f \n" + + ".align 16 \n" + "1: jmp 2f \n" + + ".align 16 \n" + "2: dec %0 \n" + " jnz 2b \n" + "3: dec %0 \n" + + : /* we don't need output */ + :"a" (loops) + ); } -void __delay(unsigned long loops) +static void delay_tsc(unsigned long loops) { unsigned bclock, now; int cpu; @@ -63,6 +78,27 @@ void __delay(unsigned long loops) } preempt_enable(); } + +static void (*delay_fn)(unsigned long) = delay_loop; + +void use_tsc_delay(void) +{ + delay_fn = delay_tsc; +} + +int __devinit read_current_timer(unsigned long *timer_value) +{ + if (delay_fn == delay_tsc) { + rdtscll(*timer_value); + return 0; + } + return -1; +} + +void __delay(unsigned long loops) +{ + delay_fn(loops); +} EXPORT_SYMBOL(__delay); inline void __const_udelay(unsigned long xloops) From a76febe975997b933b7285b6e20bb0a21c09d453 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 09:52:36 -0300 Subject: [PATCH 03/43] x86: use rdtscll in read_current_timer for i386. This way we achieve the same code for both arches. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/delay_32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/lib/delay_32.c b/arch/x86/lib/delay_32.c index 54013f87d956..bf6de05445ba 100644 --- a/arch/x86/lib/delay_32.c +++ b/arch/x86/lib/delay_32.c @@ -98,7 +98,7 @@ void use_tsc_delay(void) int __devinit read_current_timer(unsigned long *timer_val) { if (delay_fn == delay_tsc) { - rdtscl(*timer_val); + rdtscll(*timer_val); return 0; } return -1; From 7e58818d32c18197602d1869b22cfda99efd05fe Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 10:21:25 -0300 Subject: [PATCH 04/43] x86: explicitly use edx in const delay function. For x86_64, we can't just use %0, as it would generate a mul against rdx, which is not really what we want (note the ">> 32" in x86_64 version). Using a u64 variable with a shift in i386 generates bad code, so the solution is to explicitly use %%edx in inline assembly for both. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/delay_32.c | 2 +- arch/x86/lib/delay_64.c | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/x86/lib/delay_32.c b/arch/x86/lib/delay_32.c index bf6de05445ba..0b659a320b1e 100644 --- a/arch/x86/lib/delay_32.c +++ b/arch/x86/lib/delay_32.c @@ -114,7 +114,7 @@ inline void __const_udelay(unsigned long xloops) int d0; xloops *= 4; - __asm__("mull %0" + __asm__("mull %%edx" :"=d" (xloops), "=&a" (d0) :"1" (xloops), "0" (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); diff --git a/arch/x86/lib/delay_64.c b/arch/x86/lib/delay_64.c index d0326d07c845..ff3dfecdb6f9 100644 --- a/arch/x86/lib/delay_64.c +++ b/arch/x86/lib/delay_64.c @@ -103,9 +103,16 @@ EXPORT_SYMBOL(__delay); inline void __const_udelay(unsigned long xloops) { - __delay(((xloops * HZ * - cpu_data(raw_smp_processor_id()).loops_per_jiffy) >> 32) + 1); + int d0; + xloops *= 4; + __asm__("mull %%edx" + :"=d" (xloops), "=&a" (d0) + :"1" (xloops), "0" + (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); + + __delay(++xloops); } + EXPORT_SYMBOL(__const_udelay); void __udelay(unsigned long usecs) From f0fbf0abc093ec8bf64506eee4ede9e5daf40ffd Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Thu, 3 Jul 2008 12:35:41 -0300 Subject: [PATCH 05/43] x86: integrate delay functions. delay_32.c, delay_64.c are now equal, and are integrated into delay.c. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/Makefile | 2 +- arch/x86/lib/{delay_32.c => delay.c} | 11 ++- arch/x86/lib/delay_64.c | 128 --------------------------- 3 files changed, 6 insertions(+), 135 deletions(-) rename arch/x86/lib/{delay_32.c => delay.c} (98%) delete mode 100644 arch/x86/lib/delay_64.c diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 76f60f52a885..86960a6c41c0 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_SMP) := msr-on-cpu.o -lib-y := delay_$(BITS).o +lib-y := delay.o lib-y += usercopy_$(BITS).o getuser_$(BITS).o putuser_$(BITS).o lib-y += memcpy_$(BITS).o diff --git a/arch/x86/lib/delay_32.c b/arch/x86/lib/delay.c similarity index 98% rename from arch/x86/lib/delay_32.c rename to arch/x86/lib/delay.c index 0b659a320b1e..f4568605d7d5 100644 --- a/arch/x86/lib/delay_32.c +++ b/arch/x86/lib/delay.c @@ -29,7 +29,7 @@ /* simple loop based delay: */ static void delay_loop(unsigned long loops) { - __asm__ __volatile__( + asm volatile( " test %0,%0 \n" " jz 3f \n" " jmp 1f \n" @@ -108,31 +108,30 @@ void __delay(unsigned long loops) { delay_fn(loops); } +EXPORT_SYMBOL(__delay); inline void __const_udelay(unsigned long xloops) { int d0; xloops *= 4; - __asm__("mull %%edx" + asm("mull %%edx" :"=d" (xloops), "=&a" (d0) :"1" (xloops), "0" (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); __delay(++xloops); } +EXPORT_SYMBOL(__const_udelay); void __udelay(unsigned long usecs) { __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ } +EXPORT_SYMBOL(__udelay); void __ndelay(unsigned long nsecs) { __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ } - -EXPORT_SYMBOL(__delay); -EXPORT_SYMBOL(__const_udelay); -EXPORT_SYMBOL(__udelay); EXPORT_SYMBOL(__ndelay); diff --git a/arch/x86/lib/delay_64.c b/arch/x86/lib/delay_64.c deleted file mode 100644 index ff3dfecdb6f9..000000000000 --- a/arch/x86/lib/delay_64.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Precise Delay Loops for x86-64 - * - * Copyright (C) 1993 Linus Torvalds - * Copyright (C) 1997 Martin Mares - * - * The __delay function must _NOT_ be inlined as its execution time - * depends wildly on alignment on many x86 processors. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include - -#ifdef CONFIG_SMP -#include -#endif - -/* simple loop based delay: */ -static void delay_loop(unsigned long loops) -{ - asm volatile( - " test %0,%0 \n" - " jz 3f \n" - " jmp 1f \n" - - ".align 16 \n" - "1: jmp 2f \n" - - ".align 16 \n" - "2: dec %0 \n" - " jnz 2b \n" - "3: dec %0 \n" - - : /* we don't need output */ - :"a" (loops) - ); -} - -static void delay_tsc(unsigned long loops) -{ - unsigned bclock, now; - int cpu; - - preempt_disable(); - cpu = smp_processor_id(); - rdtscl(bclock); - for (;;) { - rdtscl(now); - if ((now - bclock) >= loops) - break; - - /* Allow RT tasks to run */ - preempt_enable(); - rep_nop(); - preempt_disable(); - - /* - * It is possible that we moved to another CPU, and - * since TSC's are per-cpu we need to calculate - * that. The delay must guarantee that we wait "at - * least" the amount of time. Being moved to another - * CPU could make the wait longer but we just need to - * make sure we waited long enough. Rebalance the - * counter for this CPU. - */ - if (unlikely(cpu != smp_processor_id())) { - loops -= (now - bclock); - cpu = smp_processor_id(); - rdtscl(bclock); - } - } - preempt_enable(); -} - -static void (*delay_fn)(unsigned long) = delay_loop; - -void use_tsc_delay(void) -{ - delay_fn = delay_tsc; -} - -int __devinit read_current_timer(unsigned long *timer_value) -{ - if (delay_fn == delay_tsc) { - rdtscll(*timer_value); - return 0; - } - return -1; -} - -void __delay(unsigned long loops) -{ - delay_fn(loops); -} -EXPORT_SYMBOL(__delay); - -inline void __const_udelay(unsigned long xloops) -{ - int d0; - xloops *= 4; - __asm__("mull %%edx" - :"=d" (xloops), "=&a" (d0) - :"1" (xloops), "0" - (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); - - __delay(++xloops); -} - -EXPORT_SYMBOL(__const_udelay); - -void __udelay(unsigned long usecs) -{ - __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ -} -EXPORT_SYMBOL(__udelay); - -void __ndelay(unsigned long nsecs) -{ - __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ -} -EXPORT_SYMBOL(__ndelay); From f8dd0d3c62164160c59034a96eb17d69ac8a0328 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Jul 2008 09:12:55 +0200 Subject: [PATCH 06/43] x86: delay lib unification build fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix: arch/x86/lib/delay.c:93:24: error: macro "use_tsc_delay" passed 1 arguments, but takes just 0 arch/x86/lib/delay.c:94: error: expected ‘=’, ‘,’, ‘;’, ‘asm’ or ‘__attribute__’ before ‘{’ token Signed-off-by: Ingo Molnar --- include/asm-x86/delay.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/asm-x86/delay.h b/include/asm-x86/delay.h index bb80880c834b..409a649204aa 100644 --- a/include/asm-x86/delay.h +++ b/include/asm-x86/delay.h @@ -26,10 +26,6 @@ extern void __delay(unsigned long loops); ((n) > 20000 ? __bad_ndelay() : __const_udelay((n) * 5ul)) : \ __ndelay(n)) -#ifdef CONFIG_X86_32 void use_tsc_delay(void); -#else -#define use_tsc_delay() {} -#endif #endif /* _ASM_X86_DELAY_H */ From edf10162b2c5ad78ada8e63e960f9d0949c6c219 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Fri, 13 Jun 2008 16:35:52 -0300 Subject: [PATCH 07/43] x86: don't clobber r8 nor use rcx. There's really no reason to clobber r8 or pass the address in rcx. We can safely use only two registers (which we already have to touch anyway) to do the job. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_64.S | 42 ++++++++++++++++++------------------ include/asm-x86/uaccess_64.h | 3 +-- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 5448876261f8..2b003d313480 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -36,10 +36,10 @@ .text ENTRY(__get_user_1) CFI_STARTPROC - GET_THREAD_INFO(%r8) - cmpq threadinfo_addr_limit(%r8),%rcx + GET_THREAD_INFO(%rdx) + cmpq threadinfo_addr_limit(%rdx),%rax jae bad_get_user -1: movzb (%rcx),%edx +1: movzb (%rax),%edx xorl %eax,%eax ret CFI_ENDPROC @@ -47,48 +47,48 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - GET_THREAD_INFO(%r8) - addq $1,%rcx + GET_THREAD_INFO(%rdx) + addq $1,%rax jc 20f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq threadinfo_addr_limit(%rdx),%rax jae 20f - decq %rcx -2: movzwl (%rcx),%edx + decq %rax +2: movzwl (%rax),%edx xorl %eax,%eax ret -20: decq %rcx +20: decq %rax jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - GET_THREAD_INFO(%r8) - addq $3,%rcx + GET_THREAD_INFO(%rdx) + addq $3,%rax jc 30f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq threadinfo_addr_limit(%rdx),%rax jae 30f - subq $3,%rcx -3: movl (%rcx),%edx + subq $3,%rax +3: movl (%rax),%edx xorl %eax,%eax ret -30: subq $3,%rcx +30: subq $3,%rax jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_4) ENTRY(__get_user_8) CFI_STARTPROC - GET_THREAD_INFO(%r8) - addq $7,%rcx + GET_THREAD_INFO(%rdx) + addq $7,%rax jc 40f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq threadinfo_addr_limit(%rdx),%rax jae 40f - subq $7,%rcx -4: movq (%rcx),%rdx + subq $7,%rax +4: movq (%rax),%rdx xorl %eax,%eax ret -40: subq $7,%rcx +40: subq $7,%rax jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_8) diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index b8a2f4339903..a2d49078e19c 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -90,8 +90,7 @@ extern int fixup_exception(struct pt_regs *regs); #define __get_user_x(size, ret, x, ptr) \ asm volatile("call __get_user_" #size \ : "=a" (ret),"=d" (x) \ - : "c" (ptr) \ - : "r8") + : "0" (ptr)) \ /* Careful: we have to cast the result to the type of the pointer * for sign reasons */ From 9aa038815b5756e20a00b8e1efd5740434b37aea Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Fri, 13 Jun 2008 22:41:51 -0300 Subject: [PATCH 08/43] x86: don't use word-size specifiers. Since the instructions refer to registers, they'll be able to figure it out. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_32.S | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/x86/lib/getuser_32.S b/arch/x86/lib/getuser_32.S index 6d84b53f12a2..8200fde55f57 100644 --- a/arch/x86/lib/getuser_32.S +++ b/arch/x86/lib/getuser_32.S @@ -29,44 +29,44 @@ ENTRY(__get_user_1) CFI_STARTPROC GET_THREAD_INFO(%edx) - cmpl TI_addr_limit(%edx),%eax + cmp TI_addr_limit(%edx),%eax jae bad_get_user -1: movzbl (%eax),%edx - xorl %eax,%eax +1: movzb (%eax),%edx + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - addl $1,%eax + add $1,%eax jc bad_get_user GET_THREAD_INFO(%edx) - cmpl TI_addr_limit(%edx),%eax + cmp TI_addr_limit(%edx),%eax jae bad_get_user 2: movzwl -1(%eax),%edx - xorl %eax,%eax + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - addl $3,%eax + add $3,%eax jc bad_get_user GET_THREAD_INFO(%edx) - cmpl TI_addr_limit(%edx),%eax + cmp TI_addr_limit(%edx),%eax jae bad_get_user -3: movl -3(%eax),%edx - xorl %eax,%eax +3: mov -3(%eax),%edx + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_4) bad_get_user: CFI_STARTPROC - xorl %edx,%edx - movl $-14,%eax + xor %edx,%edx + mov $-14,%eax ret CFI_ENDPROC END(bad_get_user) From 9262875395cf22b5a90dd8a640e1070cedf55d0e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 11:13:16 -0300 Subject: [PATCH 09/43] x86: adapt x86_64 getuser functions. Instead of doing a sub after the addition, use the offset directly at the memory operand of the mov instructions. This is the way i386 do. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_64.S | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 2b003d313480..df37d3a9ba2a 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -47,49 +47,40 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - GET_THREAD_INFO(%rdx) addq $1,%rax - jc 20f + jc bad_get_user + GET_THREAD_INFO(%rdx) cmpq threadinfo_addr_limit(%rdx),%rax - jae 20f - decq %rax -2: movzwl (%rax),%edx + jae bad_get_user +2: movzwl -1(%rax),%edx xorl %eax,%eax ret -20: decq %rax - jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - GET_THREAD_INFO(%rdx) addq $3,%rax - jc 30f + jc bad_get_user + GET_THREAD_INFO(%rdx) cmpq threadinfo_addr_limit(%rdx),%rax - jae 30f - subq $3,%rax -3: movl (%rax),%edx + jae bad_get_user +3: movl -3(%rax),%edx xorl %eax,%eax ret -30: subq $3,%rax - jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_4) ENTRY(__get_user_8) CFI_STARTPROC - GET_THREAD_INFO(%rdx) addq $7,%rax - jc 40f + jc bad_get_user + GET_THREAD_INFO(%rdx) cmpq threadinfo_addr_limit(%rdx),%rax - jae 40f - subq $7,%rax -4: movq (%rax),%rdx + jae bad_get_user +4: movq -7(%rax),%rdx xorl %eax,%eax ret -40: subq $7,%rax - jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_8) From 26ccb8a7183eed424ff9c874c83af20dafe7cdef Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 11:19:35 -0300 Subject: [PATCH 10/43] x86: rename threadinfo to TI. This is for consistency with i386. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/ia32/ia32entry.S | 25 ++++++++++++++----------- arch/x86/kernel/asm-offsets_64.c | 2 +- arch/x86/kernel/entry_64.S | 23 ++++++++++++----------- arch/x86/lib/copy_user_64.S | 4 ++-- arch/x86/lib/getuser_64.S | 8 ++++---- arch/x86/lib/putuser_64.S | 8 ++++---- 6 files changed, 37 insertions(+), 33 deletions(-) diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S index 24e4d4928d65..20371d0635e4 100644 --- a/arch/x86/ia32/ia32entry.S +++ b/arch/x86/ia32/ia32entry.S @@ -116,7 +116,7 @@ ENTRY(ia32_sysenter_target) pushfq CFI_ADJUST_CFA_OFFSET 8 /*CFI_REL_OFFSET rflags,0*/ - movl 8*3-THREAD_SIZE+threadinfo_sysenter_return(%rsp), %r10d + movl 8*3-THREAD_SIZE+TI_sysenter_return(%rsp), %r10d CFI_REGISTER rip,r10 pushq $__USER32_CS CFI_ADJUST_CFA_OFFSET 8 @@ -136,8 +136,9 @@ ENTRY(ia32_sysenter_target) .quad 1b,ia32_badarg .previous GET_THREAD_INFO(%r10) - orl $TS_COMPAT,threadinfo_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) + orl $TS_COMPAT,TI_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ + TI_flags(%r10) CFI_REMEMBER_STATE jnz sysenter_tracesys sysenter_do_call: @@ -149,9 +150,9 @@ sysenter_do_call: GET_THREAD_INFO(%r10) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - testl $_TIF_ALLWORK_MASK,threadinfo_flags(%r10) + testl $_TIF_ALLWORK_MASK,TI_flags(%r10) jnz int_ret_from_sys_call - andl $~TS_COMPAT,threadinfo_status(%r10) + andl $~TS_COMPAT,TI_status(%r10) /* clear IF, that popfq doesn't enable interrupts early */ andl $~0x200,EFLAGS-R11(%rsp) movl RIP-R11(%rsp),%edx /* User %eip */ @@ -240,8 +241,9 @@ ENTRY(ia32_cstar_target) .quad 1b,ia32_badarg .previous GET_THREAD_INFO(%r10) - orl $TS_COMPAT,threadinfo_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) + orl $TS_COMPAT,TI_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ + TI_flags(%r10) CFI_REMEMBER_STATE jnz cstar_tracesys cstar_do_call: @@ -253,9 +255,9 @@ cstar_do_call: GET_THREAD_INFO(%r10) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - testl $_TIF_ALLWORK_MASK,threadinfo_flags(%r10) + testl $_TIF_ALLWORK_MASK,TI_flags(%r10) jnz int_ret_from_sys_call - andl $~TS_COMPAT,threadinfo_status(%r10) + andl $~TS_COMPAT,TI_status(%r10) RESTORE_ARGS 1,-ARG_SKIP,1,1,1 movl RIP-ARGOFFSET(%rsp),%ecx CFI_REGISTER rip,rcx @@ -333,8 +335,9 @@ ENTRY(ia32_syscall) this could be a problem. */ SAVE_ARGS 0,0,1 GET_THREAD_INFO(%r10) - orl $TS_COMPAT,threadinfo_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) + orl $TS_COMPAT,TI_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ + TI_flags(%r10) jnz ia32_tracesys ia32_do_syscall: cmpl $(IA32_NR_syscalls-1),%eax diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index 3295e7c08fe7..bacf5deeec2d 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c @@ -34,7 +34,7 @@ int main(void) ENTRY(pid); BLANK(); #undef ENTRY -#define ENTRY(entry) DEFINE(threadinfo_ ## entry, offsetof(struct thread_info, entry)) +#define ENTRY(entry) DEFINE(TI_ ## entry, offsetof(struct thread_info, entry)) ENTRY(flags); ENTRY(addr_limit); ENTRY(preempt_count); diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 07d69f262337..466b9284ed2f 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -168,13 +168,13 @@ ENTRY(ret_from_fork) CFI_ADJUST_CFA_OFFSET -4 call schedule_tail GET_THREAD_INFO(%rcx) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx) jnz rff_trace rff_action: RESTORE_REST testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread? je int_ret_from_sys_call - testl $_TIF_IA32,threadinfo_flags(%rcx) + testl $_TIF_IA32,TI_flags(%rcx) jnz int_ret_from_sys_call RESTORE_TOP_OF_STACK %rdi,ARGOFFSET jmp ret_from_sys_call @@ -243,7 +243,8 @@ ENTRY(system_call_after_swapgs) movq %rcx,RIP-ARGOFFSET(%rsp) CFI_REL_OFFSET rip,RIP-ARGOFFSET GET_THREAD_INFO(%rcx) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ + TI_flags(%rcx) jnz tracesys cmpq $__NR_syscall_max,%rax ja badsys @@ -262,7 +263,7 @@ sysret_check: GET_THREAD_INFO(%rcx) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - movl threadinfo_flags(%rcx),%edx + movl TI_flags(%rcx),%edx andl %edi,%edx jnz sysret_careful CFI_REMEMBER_STATE @@ -347,10 +348,10 @@ int_ret_from_sys_call: int_with_check: LOCKDEP_SYS_EXIT_IRQ GET_THREAD_INFO(%rcx) - movl threadinfo_flags(%rcx),%edx + movl TI_flags(%rcx),%edx andl %edi,%edx jnz int_careful - andl $~TS_COMPAT,threadinfo_status(%rcx) + andl $~TS_COMPAT,TI_status(%rcx) jmp retint_swapgs /* Either reschedule or signal or syscall exit tracking needed. */ @@ -558,7 +559,7 @@ retint_with_reschedule: movl $_TIF_WORK_MASK,%edi retint_check: LOCKDEP_SYS_EXIT_IRQ - movl threadinfo_flags(%rcx),%edx + movl TI_flags(%rcx),%edx andl %edi,%edx CFI_REMEMBER_STATE jnz retint_careful @@ -654,9 +655,9 @@ retint_signal: /* Returning to kernel space. Check if we need preemption */ /* rcx: threadinfo. interrupts off. */ ENTRY(retint_kernel) - cmpl $0,threadinfo_preempt_count(%rcx) + cmpl $0,TI_preempt_count(%rcx) jnz retint_restore_args - bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx) + bt $TIF_NEED_RESCHED,TI_flags(%rcx) jnc retint_restore_args bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ jnc retint_restore_args @@ -819,7 +820,7 @@ paranoid_restore\trace: jmp irq_return paranoid_userspace\trace: GET_THREAD_INFO(%rcx) - movl threadinfo_flags(%rcx),%ebx + movl TI_flags(%rcx),%ebx andl $_TIF_WORK_MASK,%ebx jz paranoid_swapgs\trace movq %rsp,%rdi /* &pt_regs */ @@ -917,7 +918,7 @@ error_exit: testl %eax,%eax jne retint_kernel LOCKDEP_SYS_EXIT_IRQ - movl threadinfo_flags(%rcx),%edx + movl TI_flags(%rcx),%edx movl $_TIF_WORK_MASK,%edi andl %edi,%edx jnz retint_careful diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S index ee1c3f635157..7eaaf0123b4d 100644 --- a/arch/x86/lib/copy_user_64.S +++ b/arch/x86/lib/copy_user_64.S @@ -40,7 +40,7 @@ ENTRY(copy_to_user) movq %rdi,%rcx addq %rdx,%rcx jc bad_to_user - cmpq threadinfo_addr_limit(%rax),%rcx + cmpq TI_addr_limit(%rax),%rcx jae bad_to_user xorl %eax,%eax /* clear zero flag */ ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string @@ -65,7 +65,7 @@ ENTRY(copy_from_user) movq %rsi,%rcx addq %rdx,%rcx jc bad_from_user - cmpq threadinfo_addr_limit(%rax),%rcx + cmpq TI_addr_limit(%rax),%rcx jae bad_from_user movl $1,%ecx /* set zero flag */ ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index df37d3a9ba2a..0ec7890f9dcc 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -37,7 +37,7 @@ ENTRY(__get_user_1) CFI_STARTPROC GET_THREAD_INFO(%rdx) - cmpq threadinfo_addr_limit(%rdx),%rax + cmpq TI_addr_limit(%rdx),%rax jae bad_get_user 1: movzb (%rax),%edx xorl %eax,%eax @@ -50,7 +50,7 @@ ENTRY(__get_user_2) addq $1,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq threadinfo_addr_limit(%rdx),%rax + cmpq TI_addr_limit(%rdx),%rax jae bad_get_user 2: movzwl -1(%rax),%edx xorl %eax,%eax @@ -63,7 +63,7 @@ ENTRY(__get_user_4) addq $3,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq threadinfo_addr_limit(%rdx),%rax + cmpq TI_addr_limit(%rdx),%rax jae bad_get_user 3: movl -3(%rax),%edx xorl %eax,%eax @@ -76,7 +76,7 @@ ENTRY(__get_user_8) addq $7,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq threadinfo_addr_limit(%rdx),%rax + cmpq TI_addr_limit(%rdx),%rax jae bad_get_user 4: movq -7(%rax),%rdx xorl %eax,%eax diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index 4989f5a8fa9b..940796fa0d98 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -35,7 +35,7 @@ ENTRY(__put_user_1) CFI_STARTPROC GET_THREAD_INFO(%r8) - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%r8),%rcx jae bad_put_user 1: movb %dl,(%rcx) xorl %eax,%eax @@ -48,7 +48,7 @@ ENTRY(__put_user_2) GET_THREAD_INFO(%r8) addq $1,%rcx jc 20f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%r8),%rcx jae 20f decq %rcx 2: movw %dx,(%rcx) @@ -64,7 +64,7 @@ ENTRY(__put_user_4) GET_THREAD_INFO(%r8) addq $3,%rcx jc 30f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%r8),%rcx jae 30f subq $3,%rcx 3: movl %edx,(%rcx) @@ -80,7 +80,7 @@ ENTRY(__put_user_8) GET_THREAD_INFO(%r8) addq $7,%rcx jc 40f - cmpq threadinfo_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%r8),%rcx jae 40f subq $7,%rcx 4: movq %rdx,(%rcx) From ef8c1a2d0e990d0f4f15e1d45eeb262755e3d4c3 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 11:21:53 -0300 Subject: [PATCH 11/43] x86: don't use word-size specifiers on getuser_64. The instructions access registers, so the size is unambiguous. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_64.S | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 0ec7890f9dcc..6134752a75fd 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -37,57 +37,57 @@ ENTRY(__get_user_1) CFI_STARTPROC GET_THREAD_INFO(%rdx) - cmpq TI_addr_limit(%rdx),%rax + cmp TI_addr_limit(%rdx),%rax jae bad_get_user 1: movzb (%rax),%edx - xorl %eax,%eax + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - addq $1,%rax + add $1,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq TI_addr_limit(%rdx),%rax + cmp TI_addr_limit(%rdx),%rax jae bad_get_user 2: movzwl -1(%rax),%edx - xorl %eax,%eax + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - addq $3,%rax + add $3,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq TI_addr_limit(%rdx),%rax + cmp TI_addr_limit(%rdx),%rax jae bad_get_user -3: movl -3(%rax),%edx - xorl %eax,%eax +3: mov -3(%rax),%edx + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_4) ENTRY(__get_user_8) CFI_STARTPROC - addq $7,%rax + add $7,%rax jc bad_get_user GET_THREAD_INFO(%rdx) - cmpq TI_addr_limit(%rdx),%rax + cmp TI_addr_limit(%rdx),%rax jae bad_get_user 4: movq -7(%rax),%rdx - xorl %eax,%eax + xor %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_8) bad_get_user: CFI_STARTPROC - xorl %edx,%edx - movq $(-EFAULT),%rax + xor %edx,%edx + mov $(-EFAULT),%rax ret CFI_ENDPROC END(bad_get_user) From 40faf463e62de0b29722910eded7dd26cd8b684b Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 11:37:57 -0300 Subject: [PATCH 12/43] x86: introduce __ASM_REG macro. There are situations in which the architecture wants to use the register that represents its word-size, whatever it is. For those, introduce __ASM_REG in asm.h, along with the first users _ASM_AX and _ASM_DX. They have users waiting for it, namely the getuser functions. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_32.S | 25 +++++++++++++------------ arch/x86/lib/getuser_64.S | 36 ++++++++++++++++++------------------ include/asm-x86/asm.h | 3 +++ 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/arch/x86/lib/getuser_32.S b/arch/x86/lib/getuser_32.S index 8200fde55f57..2cc3ceee8f91 100644 --- a/arch/x86/lib/getuser_32.S +++ b/arch/x86/lib/getuser_32.S @@ -11,6 +11,7 @@ #include #include #include +#include /* @@ -28,10 +29,10 @@ .text ENTRY(__get_user_1) CFI_STARTPROC - GET_THREAD_INFO(%edx) - cmp TI_addr_limit(%edx),%eax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -1: movzb (%eax),%edx +1: movzb (%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -39,12 +40,12 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - add $1,%eax + add $1,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%edx) - cmp TI_addr_limit(%edx),%eax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -2: movzwl -1(%eax),%edx +2: movzwl -1(%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -52,12 +53,12 @@ ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - add $3,%eax + add $3,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%edx) - cmp TI_addr_limit(%edx),%eax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -3: mov -3(%eax),%edx +3: mov -3(%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -66,7 +67,7 @@ ENDPROC(__get_user_4) bad_get_user: CFI_STARTPROC xor %edx,%edx - mov $-14,%eax + mov $-14,%_ASM_AX ret CFI_ENDPROC END(bad_get_user) diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 6134752a75fd..63b0e5c1e582 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -13,14 +13,13 @@ /* * __get_user_X * - * Inputs: %rcx contains the address. + * Inputs: %rax contains the address. * The register is modified, but all changes are undone * before returning because the C code doesn't know about it. * * Outputs: %rax is error code (0 or -EFAULT) * %rdx contains zero-extended value * - * %r8 is destroyed. * * These functions should not modify any other registers, * as they get called from within inline assembly. @@ -32,14 +31,15 @@ #include #include #include +#include .text ENTRY(__get_user_1) CFI_STARTPROC - GET_THREAD_INFO(%rdx) - cmp TI_addr_limit(%rdx),%rax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -1: movzb (%rax),%edx +1: movzb (%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -47,12 +47,12 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - add $1,%rax + add $1,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%rdx) - cmp TI_addr_limit(%rdx),%rax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -2: movzwl -1(%rax),%edx +2: movzwl -1(%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -60,12 +60,12 @@ ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - add $3,%rax + add $3,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%rdx) - cmp TI_addr_limit(%rdx),%rax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -3: mov -3(%rax),%edx +3: mov -3(%_ASM_AX),%edx xor %eax,%eax ret CFI_ENDPROC @@ -73,12 +73,12 @@ ENDPROC(__get_user_4) ENTRY(__get_user_8) CFI_STARTPROC - add $7,%rax + add $7,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%rdx) - cmp TI_addr_limit(%rdx),%rax + GET_THREAD_INFO(%_ASM_DX) + cmp TI_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user -4: movq -7(%rax),%rdx +4: movq -7(%_ASM_AX),%_ASM_DX xor %eax,%eax ret CFI_ENDPROC @@ -87,7 +87,7 @@ ENDPROC(__get_user_8) bad_get_user: CFI_STARTPROC xor %edx,%edx - mov $(-EFAULT),%rax + mov $(-EFAULT),%_ASM_AX ret CFI_ENDPROC END(bad_get_user) diff --git a/include/asm-x86/asm.h b/include/asm-x86/asm.h index 70939820c55f..435402e623e1 100644 --- a/include/asm-x86/asm.h +++ b/include/asm-x86/asm.h @@ -14,6 +14,7 @@ #endif #define __ASM_SIZE(inst) __ASM_SEL(inst##l, inst##q) +#define __ASM_REG(reg) __ASM_SEL(e##reg, r##reg) #define _ASM_PTR __ASM_SEL(.long, .quad) #define _ASM_ALIGN __ASM_SEL(.balign 4, .balign 8) @@ -24,6 +25,8 @@ #define _ASM_ADD __ASM_SIZE(add) #define _ASM_SUB __ASM_SIZE(sub) #define _ASM_XADD __ASM_SIZE(xadd) +#define _ASM_AX __ASM_REG(ax) +#define _ASM_DX __ASM_REG(dx) /* Exception table entry */ # define _ASM_EXTABLE(from,to) \ From 87e2f1e7f6ab66306320403d4502d7938d3c703e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 12:02:44 -0300 Subject: [PATCH 13/43] x86: use _ASM_PTR instead of explicit word-size pointers. Switch .long and .quad with _ASM_PTR in getuser*.S. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/getuser_32.S | 6 +++--- arch/x86/lib/getuser_64.S | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/lib/getuser_32.S b/arch/x86/lib/getuser_32.S index 2cc3ceee8f91..2bb0a183e066 100644 --- a/arch/x86/lib/getuser_32.S +++ b/arch/x86/lib/getuser_32.S @@ -73,7 +73,7 @@ bad_get_user: END(bad_get_user) .section __ex_table,"a" - .long 1b,bad_get_user - .long 2b,bad_get_user - .long 3b,bad_get_user + _ASM_PTR 1b,bad_get_user + _ASM_PTR 2b,bad_get_user + _ASM_PTR 3b,bad_get_user .previous diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser_64.S index 63b0e5c1e582..e33388419b7b 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser_64.S @@ -93,8 +93,8 @@ bad_get_user: END(bad_get_user) .section __ex_table,"a" - .quad 1b,bad_get_user - .quad 2b,bad_get_user - .quad 3b,bad_get_user - .quad 4b,bad_get_user + _ASM_PTR 1b,bad_get_user + _ASM_PTR 2b,bad_get_user + _ASM_PTR 3b,bad_get_user + _ASM_PTR 4b,bad_get_user .previous From 6c2d458680d49d939ffd4b4cdc84d9e004d65910 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 12:05:11 -0300 Subject: [PATCH 14/43] x86: merge getuser asm functions. getuser_32.S and getuser_64.S are merged into getuser.S. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/Makefile | 2 +- arch/x86/lib/{getuser_64.S => getuser.S} | 14 +++-- arch/x86/lib/getuser_32.S | 79 ------------------------ include/asm-x86/asm.h | 4 +- 4 files changed, 13 insertions(+), 86 deletions(-) rename arch/x86/lib/{getuser_64.S => getuser.S} (89%) delete mode 100644 arch/x86/lib/getuser_32.S diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 86960a6c41c0..e92948203a5d 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -5,7 +5,7 @@ obj-$(CONFIG_SMP) := msr-on-cpu.o lib-y := delay.o -lib-y += usercopy_$(BITS).o getuser_$(BITS).o putuser_$(BITS).o +lib-y += usercopy_$(BITS).o getuser.o putuser_$(BITS).o lib-y += memcpy_$(BITS).o ifeq ($(CONFIG_X86_32),y) diff --git a/arch/x86/lib/getuser_64.S b/arch/x86/lib/getuser.S similarity index 89% rename from arch/x86/lib/getuser_64.S rename to arch/x86/lib/getuser.S index e33388419b7b..ad374003742f 100644 --- a/arch/x86/lib/getuser_64.S +++ b/arch/x86/lib/getuser.S @@ -3,6 +3,7 @@ * * (C) Copyright 1998 Linus Torvalds * (C) Copyright 2005 Andi Kleen + * (C) Copyright 2008 Glauber Costa * * These functions have a non-standard call interface * to make them more efficient, especially as they @@ -13,13 +14,13 @@ /* * __get_user_X * - * Inputs: %rax contains the address. + * Inputs: %[r|e]ax contains the address. * The register is modified, but all changes are undone * before returning because the C code doesn't know about it. * - * Outputs: %rax is error code (0 or -EFAULT) - * %rdx contains zero-extended value - * + * Outputs: %[r|e]ax is error code (0 or -EFAULT) + * %[r|e]dx contains zero-extended value + * * * These functions should not modify any other registers, * as they get called from within inline assembly. @@ -71,6 +72,7 @@ ENTRY(__get_user_4) CFI_ENDPROC ENDPROC(__get_user_4) +#ifdef CONFIG_X86_64 ENTRY(__get_user_8) CFI_STARTPROC add $7,%_ASM_AX @@ -83,6 +85,7 @@ ENTRY(__get_user_8) ret CFI_ENDPROC ENDPROC(__get_user_8) +#endif bad_get_user: CFI_STARTPROC @@ -96,5 +99,6 @@ END(bad_get_user) _ASM_PTR 1b,bad_get_user _ASM_PTR 2b,bad_get_user _ASM_PTR 3b,bad_get_user +#ifdef CONFIG_X86_64 _ASM_PTR 4b,bad_get_user -.previous +#endif diff --git a/arch/x86/lib/getuser_32.S b/arch/x86/lib/getuser_32.S deleted file mode 100644 index 2bb0a183e066..000000000000 --- a/arch/x86/lib/getuser_32.S +++ /dev/null @@ -1,79 +0,0 @@ -/* - * __get_user functions. - * - * (C) Copyright 1998 Linus Torvalds - * - * These functions have a non-standard call interface - * to make them more efficient, especially as they - * return an error value in addition to the "real" - * return value. - */ -#include -#include -#include -#include - - -/* - * __get_user_X - * - * Inputs: %eax contains the address - * - * Outputs: %eax is error code (0 or -EFAULT) - * %edx contains zero-extended value - * - * These functions should not modify any other registers, - * as they get called from within inline assembly. - */ - -.text -ENTRY(__get_user_1) - CFI_STARTPROC - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -1: movzb (%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_1) - -ENTRY(__get_user_2) - CFI_STARTPROC - add $1,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -2: movzwl -1(%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_2) - -ENTRY(__get_user_4) - CFI_STARTPROC - add $3,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -3: mov -3(%_ASM_AX),%edx - xor %eax,%eax - ret - CFI_ENDPROC -ENDPROC(__get_user_4) - -bad_get_user: - CFI_STARTPROC - xor %edx,%edx - mov $-14,%_ASM_AX - ret - CFI_ENDPROC -END(bad_get_user) - -.section __ex_table,"a" - _ASM_PTR 1b,bad_get_user - _ASM_PTR 2b,bad_get_user - _ASM_PTR 3b,bad_get_user -.previous diff --git a/include/asm-x86/asm.h b/include/asm-x86/asm.h index 435402e623e1..57750a95685d 100644 --- a/include/asm-x86/asm.h +++ b/include/asm-x86/asm.h @@ -3,8 +3,10 @@ #ifdef __ASSEMBLY__ # define __ASM_FORM(x) x +# define __ASM_EX_SEC .section __ex_table #else # define __ASM_FORM(x) " " #x " " +# define __ASM_EX_SEC " .section __ex_table,\"a\"\n" #endif #ifdef CONFIG_X86_32 @@ -30,7 +32,7 @@ /* Exception table entry */ # define _ASM_EXTABLE(from,to) \ - " .section __ex_table,\"a\"\n" \ + __ASM_EX_SEC \ _ASM_ALIGN "\n" \ _ASM_PTR #from "," #to "\n" \ " .previous\n" From 268cf048c890d10bd3a86bd87922ed8a722d502f Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 12:40:55 -0300 Subject: [PATCH 15/43] x86: don't save ebx in putuser_32.S. Clobber it in the inline asm macros, and let the compiler do this for us. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_32.S | 13 ++----------- include/asm-x86/uaccess_32.h | 10 +++++----- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/arch/x86/lib/putuser_32.S b/arch/x86/lib/putuser_32.S index f58fba109d18..5b2a926f0e28 100644 --- a/arch/x86/lib/putuser_32.S +++ b/arch/x86/lib/putuser_32.S @@ -26,14 +26,8 @@ */ #define ENTER CFI_STARTPROC ; \ - pushl %ebx ; \ - CFI_ADJUST_CFA_OFFSET 4 ; \ - CFI_REL_OFFSET ebx, 0 ; \ GET_THREAD_INFO(%ebx) -#define EXIT popl %ebx ; \ - CFI_ADJUST_CFA_OFFSET -4 ; \ - CFI_RESTORE ebx ; \ - ret ; \ +#define EXIT ret ; \ CFI_ENDPROC .text @@ -81,10 +75,7 @@ ENTRY(__put_user_8) ENDPROC(__put_user_8) bad_put_user: - CFI_STARTPROC simple - CFI_DEF_CFA esp, 2*4 - CFI_OFFSET eip, -1*4 - CFI_OFFSET ebx, -2*4 + CFI_STARTPROC movl $-14,%eax EXIT END(bad_put_user) diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 8e7595c1f34e..0ecfe47ad60d 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -188,23 +188,23 @@ extern void __put_user_8(void); #define __put_user_1(x, ptr) \ asm volatile("call __put_user_1" : "=a" (__ret_pu) \ - : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") #define __put_user_2(x, ptr) \ asm volatile("call __put_user_2" : "=a" (__ret_pu) \ - : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") #define __put_user_4(x, ptr) \ asm volatile("call __put_user_4" : "=a" (__ret_pu) \ - : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") #define __put_user_8(x, ptr) \ asm volatile("call __put_user_8" : "=a" (__ret_pu) \ - : "A" ((typeof(*(ptr)))(x)), "c" (ptr)) + : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") #define __put_user_X(x, ptr) \ asm volatile("call __put_user_X" : "=a" (__ret_pu) \ - : "c" (ptr)) + : "c" (ptr): "ebx") /** * put_user: - Write a simple value into user space. From 70706e432ee5618abf59381101d8dea7b8d97a7d Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 15:02:31 -0300 Subject: [PATCH 16/43] x86: user put_user_x instead of all variants. Follow the pattern, and define a single put_user_x, instead of defining macros for all available sizes. Exception is put_user_8, since the "A" constraint does not give us enough power to specify which register (a or d) to use in the 32-bit common case. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_32.h | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 0ecfe47ad60d..f8abc12a77c2 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -186,25 +186,14 @@ extern void __put_user_2(void); extern void __put_user_4(void); extern void __put_user_8(void); -#define __put_user_1(x, ptr) \ - asm volatile("call __put_user_1" : "=a" (__ret_pu) \ - : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") - -#define __put_user_2(x, ptr) \ - asm volatile("call __put_user_2" : "=a" (__ret_pu) \ - : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") - -#define __put_user_4(x, ptr) \ - asm volatile("call __put_user_4" : "=a" (__ret_pu) \ - : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") +#define __put_user_x(size, x, ptr) \ + asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ + :"0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") #define __put_user_8(x, ptr) \ asm volatile("call __put_user_8" : "=a" (__ret_pu) \ : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") -#define __put_user_X(x, ptr) \ - asm volatile("call __put_user_X" : "=a" (__ret_pu) \ - : "c" (ptr): "ebx") /** * put_user: - Write a simple value into user space. @@ -232,19 +221,19 @@ extern void __put_user_8(void); __pu_val = x; \ switch (sizeof(*(ptr))) { \ case 1: \ - __put_user_1(__pu_val, ptr); \ + __put_user_x(1, __pu_val, ptr); \ break; \ case 2: \ - __put_user_2(__pu_val, ptr); \ + __put_user_x(2, __pu_val, ptr); \ break; \ case 4: \ - __put_user_4(__pu_val, ptr); \ + __put_user_x(4, __pu_val, ptr); \ break; \ case 8: \ __put_user_8(__pu_val, ptr); \ break; \ default: \ - __put_user_X(__pu_val, ptr); \ + __put_user_x(X, __pu_val, ptr); \ break; \ } \ __ret_pu; \ From 770546b99fb99e71a3aa4181980d42664f9c18bd Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 15:03:40 -0300 Subject: [PATCH 17/43] x86: clobber rbx in putuser_64.S. Instead of clobbering r8, clobber rbx, which is the i386 way. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_64.S | 18 +++++++++--------- include/asm-x86/uaccess_64.h | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index 940796fa0d98..07028851064c 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -18,7 +18,7 @@ * * Outputs: %rax is error code (0 or -EFAULT) * - * %r8 is destroyed. + * %rbx is destroyed. * * These functions should not modify any other registers, * as they get called from within inline assembly. @@ -34,8 +34,8 @@ .text ENTRY(__put_user_1) CFI_STARTPROC - GET_THREAD_INFO(%r8) - cmpq TI_addr_limit(%r8),%rcx + GET_THREAD_INFO(%rbx) + cmpq TI_addr_limit(%rbx),%rcx jae bad_put_user 1: movb %dl,(%rcx) xorl %eax,%eax @@ -45,10 +45,10 @@ ENDPROC(__put_user_1) ENTRY(__put_user_2) CFI_STARTPROC - GET_THREAD_INFO(%r8) + GET_THREAD_INFO(%rbx) addq $1,%rcx jc 20f - cmpq TI_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%rbx),%rcx jae 20f decq %rcx 2: movw %dx,(%rcx) @@ -61,10 +61,10 @@ ENDPROC(__put_user_2) ENTRY(__put_user_4) CFI_STARTPROC - GET_THREAD_INFO(%r8) + GET_THREAD_INFO(%rbx) addq $3,%rcx jc 30f - cmpq TI_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%rbx),%rcx jae 30f subq $3,%rcx 3: movl %edx,(%rcx) @@ -77,10 +77,10 @@ ENDPROC(__put_user_4) ENTRY(__put_user_8) CFI_STARTPROC - GET_THREAD_INFO(%r8) + GET_THREAD_INFO(%rbx) addq $7,%rcx jc 40f - cmpq TI_addr_limit(%r8),%rcx + cmpq TI_addr_limit(%rbx),%rcx jae 40f subq $7,%rcx 4: movq %rdx,(%rcx) diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index a2d49078e19c..21fda9ebee13 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -131,7 +131,7 @@ extern void __put_user_bad(void); asm volatile("call __put_user_" #size \ :"=a" (ret) \ :"c" (ptr),"d" (x) \ - :"r8") + :"ebx") #define put_user(x, ptr) \ __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) From 0ada3164031162b4e1b7ff6b36ba8cc80ff7fe96 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 16:44:39 -0300 Subject: [PATCH 18/43] x86: pass argument to putuser_64 functions in ax register. This is consistent with i386 usage. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_64.S | 8 ++++---- include/asm-x86/uaccess_64.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index 07028851064c..ce5fcd5d8c16 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -37,7 +37,7 @@ ENTRY(__put_user_1) GET_THREAD_INFO(%rbx) cmpq TI_addr_limit(%rbx),%rcx jae bad_put_user -1: movb %dl,(%rcx) +1: movb %al,(%rcx) xorl %eax,%eax ret CFI_ENDPROC @@ -51,7 +51,7 @@ ENTRY(__put_user_2) cmpq TI_addr_limit(%rbx),%rcx jae 20f decq %rcx -2: movw %dx,(%rcx) +2: movw %ax,(%rcx) xorl %eax,%eax ret 20: decq %rcx @@ -67,7 +67,7 @@ ENTRY(__put_user_4) cmpq TI_addr_limit(%rbx),%rcx jae 30f subq $3,%rcx -3: movl %edx,(%rcx) +3: movl %eax,(%rcx) xorl %eax,%eax ret 30: subq $3,%rcx @@ -83,7 +83,7 @@ ENTRY(__put_user_8) cmpq TI_addr_limit(%rbx),%rcx jae 40f subq $7,%rcx -4: movq %rdx,(%rcx) +4: movq %rax,(%rcx) xorl %eax,%eax ret 40: subq $7,%rcx diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 21fda9ebee13..f822a36cc1c0 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -130,7 +130,7 @@ extern void __put_user_bad(void); #define __put_user_x(size, ret, x, ptr) \ asm volatile("call __put_user_" #size \ :"=a" (ret) \ - :"c" (ptr),"d" (x) \ + :"c" (ptr),"a" (x) \ :"ebx") #define put_user(x, ptr) \ From 663aa96df32af9c4141ef3179282f95c7537643a Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 16:51:59 -0300 Subject: [PATCH 19/43] x86: change testing logic in putuser_64.S. Instead of operating over a register we need to put back into normal state afterwards (the memory position), just sub from rbx, which is trashed anyway. We can save a few instructions. Also, this is the i386 way. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_64.S | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index ce5fcd5d8c16..a96bd8a5298e 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -46,48 +46,39 @@ ENDPROC(__put_user_1) ENTRY(__put_user_2) CFI_STARTPROC GET_THREAD_INFO(%rbx) - addq $1,%rcx - jc 20f - cmpq TI_addr_limit(%rbx),%rcx - jae 20f - decq %rcx + mov TI_addr_limit(%rbx),%rbx + sub $1, %rbx + cmpq %rbx ,%rcx + jae bad_put_user 2: movw %ax,(%rcx) xorl %eax,%eax ret -20: decq %rcx - jmp bad_put_user CFI_ENDPROC ENDPROC(__put_user_2) ENTRY(__put_user_4) CFI_STARTPROC GET_THREAD_INFO(%rbx) - addq $3,%rcx - jc 30f - cmpq TI_addr_limit(%rbx),%rcx - jae 30f - subq $3,%rcx + mov TI_addr_limit(%rbx),%rbx + sub $3, %rbx + cmp %rbx, %rcx + jae bad_put_user 3: movl %eax,(%rcx) xorl %eax,%eax ret -30: subq $3,%rcx - jmp bad_put_user CFI_ENDPROC ENDPROC(__put_user_4) ENTRY(__put_user_8) CFI_STARTPROC GET_THREAD_INFO(%rbx) - addq $7,%rcx - jc 40f - cmpq TI_addr_limit(%rbx),%rcx - jae 40f - subq $7,%rcx + mov TI_addr_limit(%rbx),%rbx + sub $7, %rbx + cmp %rbx, %rcx + jae bad_put_user 4: movq %rax,(%rcx) xorl %eax,%eax ret -40: subq $7,%rcx - jmp bad_put_user CFI_ENDPROC ENDPROC(__put_user_8) From 766ed4282114eab616741107745b0dd11075e496 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 16:56:30 -0300 Subject: [PATCH 20/43] x86: replace function headers by macros. In putuser_64.S, do it the i386 way, and replace the code in beginning and end of functions with macros, since it's always the same thing. Save lines. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_64.S | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index a96bd8a5298e..6d7513bf885e 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -31,62 +31,58 @@ #include #include +#define ENTER CFI_STARTPROC ; \ + GET_THREAD_INFO(%rbx) +#define EXIT ret ; \ + CFI_ENDPROC + .text ENTRY(__put_user_1) - CFI_STARTPROC - GET_THREAD_INFO(%rbx) + ENTER cmpq TI_addr_limit(%rbx),%rcx jae bad_put_user 1: movb %al,(%rcx) xorl %eax,%eax - ret - CFI_ENDPROC + EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) - CFI_STARTPROC - GET_THREAD_INFO(%rbx) + ENTER mov TI_addr_limit(%rbx),%rbx sub $1, %rbx cmpq %rbx ,%rcx jae bad_put_user 2: movw %ax,(%rcx) xorl %eax,%eax - ret - CFI_ENDPROC + EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) - CFI_STARTPROC - GET_THREAD_INFO(%rbx) + ENTER mov TI_addr_limit(%rbx),%rbx sub $3, %rbx cmp %rbx, %rcx jae bad_put_user 3: movl %eax,(%rcx) xorl %eax,%eax - ret - CFI_ENDPROC + EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) - CFI_STARTPROC - GET_THREAD_INFO(%rbx) + ENTER mov TI_addr_limit(%rbx),%rbx sub $7, %rbx cmp %rbx, %rcx jae bad_put_user 4: movq %rax,(%rcx) xorl %eax,%eax - ret - CFI_ENDPROC + EXIT ENDPROC(__put_user_8) bad_put_user: CFI_STARTPROC movq $(-EFAULT),%rax - ret - CFI_ENDPROC + EXIT END(bad_put_user) .section __ex_table,"a" From efea505d83873cfc8a7cdbb8a2a11d2c67467843 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 16:59:05 -0300 Subject: [PATCH 21/43] x86: don't use word-size specifiers in putuser files. Remove them where unambiguous. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_32.S | 28 ++++++++++++++-------------- arch/x86/lib/putuser_64.S | 14 +++++++------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/arch/x86/lib/putuser_32.S b/arch/x86/lib/putuser_32.S index 5b2a926f0e28..b67a37cab1b0 100644 --- a/arch/x86/lib/putuser_32.S +++ b/arch/x86/lib/putuser_32.S @@ -33,44 +33,44 @@ .text ENTRY(__put_user_1) ENTER - cmpl TI_addr_limit(%ebx),%ecx + cmp TI_addr_limit(%ebx),%ecx jae bad_put_user 1: movb %al,(%ecx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) ENTER - movl TI_addr_limit(%ebx),%ebx - subl $1,%ebx - cmpl %ebx,%ecx + mov TI_addr_limit(%ebx),%ebx + sub $1,%ebx + cmp %ebx,%ecx jae bad_put_user 2: movw %ax,(%ecx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) ENTER - movl TI_addr_limit(%ebx),%ebx - subl $3,%ebx - cmpl %ebx,%ecx + mov TI_addr_limit(%ebx),%ebx + sub $3,%ebx + cmp %ebx,%ecx jae bad_put_user 3: movl %eax,(%ecx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) ENTER - movl TI_addr_limit(%ebx),%ebx - subl $7,%ebx - cmpl %ebx,%ecx + mov TI_addr_limit(%ebx),%ebx + sub $7,%ebx + cmp %ebx,%ecx jae bad_put_user 4: movl %eax,(%ecx) 5: movl %edx,4(%ecx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_8) diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index 6d7513bf885e..c18fc0f5256c 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -39,10 +39,10 @@ .text ENTRY(__put_user_1) ENTER - cmpq TI_addr_limit(%rbx),%rcx + cmp TI_addr_limit(%rbx),%rcx jae bad_put_user 1: movb %al,(%rcx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_1) @@ -50,10 +50,10 @@ ENTRY(__put_user_2) ENTER mov TI_addr_limit(%rbx),%rbx sub $1, %rbx - cmpq %rbx ,%rcx + cmp %rbx ,%rcx jae bad_put_user 2: movw %ax,(%rcx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_2) @@ -64,7 +64,7 @@ ENTRY(__put_user_4) cmp %rbx, %rcx jae bad_put_user 3: movl %eax,(%rcx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_4) @@ -75,13 +75,13 @@ ENTRY(__put_user_8) cmp %rbx, %rcx jae bad_put_user 4: movq %rax,(%rcx) - xorl %eax,%eax + xor %eax,%eax EXIT ENDPROC(__put_user_8) bad_put_user: CFI_STARTPROC - movq $(-EFAULT),%rax + mov $(-EFAULT),%rax EXIT END(bad_put_user) From 2528de431ddb200653d1dc6ca90074bad9520f09 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 17:36:31 -0300 Subject: [PATCH 22/43] x86: use macros from asm.h. In putuser_32.S and putuser_64.S, replace things like .quad, .long, and explicit references to [r|e]ax for the apropriate macros in asm/asm.h. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/putuser_32.S | 43 ++++++++++++++++++++------------------- arch/x86/lib/putuser_64.S | 41 +++++++++++++++++++------------------ include/asm-x86/asm.h | 2 ++ 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/arch/x86/lib/putuser_32.S b/arch/x86/lib/putuser_32.S index b67a37cab1b0..e7eda34feb34 100644 --- a/arch/x86/lib/putuser_32.S +++ b/arch/x86/lib/putuser_32.S @@ -11,6 +11,7 @@ #include #include #include +#include /* @@ -26,50 +27,50 @@ */ #define ENTER CFI_STARTPROC ; \ - GET_THREAD_INFO(%ebx) + GET_THREAD_INFO(%_ASM_BX) #define EXIT ret ; \ CFI_ENDPROC .text ENTRY(__put_user_1) ENTER - cmp TI_addr_limit(%ebx),%ecx + cmp TI_addr_limit(%_ASM_BX),%_ASM_CX jae bad_put_user -1: movb %al,(%ecx) +1: movb %al,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) ENTER - mov TI_addr_limit(%ebx),%ebx - sub $1,%ebx - cmp %ebx,%ecx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $1,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX jae bad_put_user -2: movw %ax,(%ecx) +2: movw %ax,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) ENTER - mov TI_addr_limit(%ebx),%ebx - sub $3,%ebx - cmp %ebx,%ecx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $3,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX jae bad_put_user -3: movl %eax,(%ecx) +3: movl %eax,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) ENTER - mov TI_addr_limit(%ebx),%ebx - sub $7,%ebx - cmp %ebx,%ecx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $7,%_ASM_BX + cmp %_ASM_BX,%_ASM_CX jae bad_put_user -4: movl %eax,(%ecx) -5: movl %edx,4(%ecx) +4: movl %_ASM_AX,(%_ASM_CX) +5: movl %edx,4(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_8) @@ -81,9 +82,9 @@ bad_put_user: END(bad_put_user) .section __ex_table,"a" - .long 1b,bad_put_user - .long 2b,bad_put_user - .long 3b,bad_put_user - .long 4b,bad_put_user - .long 5b,bad_put_user + _ASM_PTR 1b,bad_put_user + _ASM_PTR 2b,bad_put_user + _ASM_PTR 3b,bad_put_user + _ASM_PTR 4b,bad_put_user + _ASM_PTR 5b,bad_put_user .previous diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S index c18fc0f5256c..d496cc8e7308 100644 --- a/arch/x86/lib/putuser_64.S +++ b/arch/x86/lib/putuser_64.S @@ -30,64 +30,65 @@ #include #include #include +#include #define ENTER CFI_STARTPROC ; \ - GET_THREAD_INFO(%rbx) + GET_THREAD_INFO(%_ASM_BX) #define EXIT ret ; \ CFI_ENDPROC .text ENTRY(__put_user_1) ENTER - cmp TI_addr_limit(%rbx),%rcx + cmp TI_addr_limit(%_ASM_BX),%_ASM_CX jae bad_put_user -1: movb %al,(%rcx) +1: movb %al,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) ENTER - mov TI_addr_limit(%rbx),%rbx - sub $1, %rbx - cmp %rbx ,%rcx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $1, %_ASM_BX + cmp %_ASM_BX ,%_ASM_CX jae bad_put_user -2: movw %ax,(%rcx) +2: movw %ax,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) ENTER - mov TI_addr_limit(%rbx),%rbx - sub $3, %rbx - cmp %rbx, %rcx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $3, %_ASM_BX + cmp %_ASM_BX, %_ASM_CX jae bad_put_user -3: movl %eax,(%rcx) +3: movl %eax,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) ENTER - mov TI_addr_limit(%rbx),%rbx - sub $7, %rbx - cmp %rbx, %rcx + mov TI_addr_limit(%_ASM_BX),%_ASM_BX + sub $7, %_ASM_BX + cmp %_ASM_BX, %_ASM_CX jae bad_put_user -4: movq %rax,(%rcx) +4: movq %_ASM_AX,(%_ASM_CX) xor %eax,%eax EXIT ENDPROC(__put_user_8) bad_put_user: CFI_STARTPROC - mov $(-EFAULT),%rax + mov $(-EFAULT),%eax EXIT END(bad_put_user) .section __ex_table,"a" - .quad 1b,bad_put_user - .quad 2b,bad_put_user - .quad 3b,bad_put_user - .quad 4b,bad_put_user + _ASM_PTR 1b,bad_put_user + _ASM_PTR 2b,bad_put_user + _ASM_PTR 3b,bad_put_user + _ASM_PTR 4b,bad_put_user .previous diff --git a/include/asm-x86/asm.h b/include/asm-x86/asm.h index 57750a95685d..97220321f39d 100644 --- a/include/asm-x86/asm.h +++ b/include/asm-x86/asm.h @@ -28,6 +28,8 @@ #define _ASM_SUB __ASM_SIZE(sub) #define _ASM_XADD __ASM_SIZE(xadd) #define _ASM_AX __ASM_REG(ax) +#define _ASM_BX __ASM_REG(bx) +#define _ASM_CX __ASM_REG(cx) #define _ASM_DX __ASM_REG(dx) /* Exception table entry */ From 5cbbc3b1eb37bdc72eefd2de03b39f5e784400c2 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 24 Jun 2008 17:40:14 -0300 Subject: [PATCH 23/43] x86: merge putuser asm functions. putuser_32.S and putuser_64.S are merged into putuser.S. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/lib/Makefile | 2 +- arch/x86/lib/{putuser_32.S => putuser.S} | 11 ++- arch/x86/lib/putuser_64.S | 94 ------------------------ 3 files changed, 10 insertions(+), 97 deletions(-) rename arch/x86/lib/{putuser_32.S => putuser.S} (89%) delete mode 100644 arch/x86/lib/putuser_64.S diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index e92948203a5d..83226e0a7ce4 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -5,7 +5,7 @@ obj-$(CONFIG_SMP) := msr-on-cpu.o lib-y := delay.o -lib-y += usercopy_$(BITS).o getuser.o putuser_$(BITS).o +lib-y += usercopy_$(BITS).o getuser.o putuser.o lib-y += memcpy_$(BITS).o ifeq ($(CONFIG_X86_32),y) diff --git a/arch/x86/lib/putuser_32.S b/arch/x86/lib/putuser.S similarity index 89% rename from arch/x86/lib/putuser_32.S rename to arch/x86/lib/putuser.S index e7eda34feb34..36b0d15ae6e9 100644 --- a/arch/x86/lib/putuser_32.S +++ b/arch/x86/lib/putuser.S @@ -2,6 +2,8 @@ * __put_user functions. * * (C) Copyright 2005 Linus Torvalds + * (C) Copyright 2005 Andi Kleen + * (C) Copyright 2008 Glauber Costa * * These functions have a non-standard call interface * to make them more efficient, especially as they @@ -11,6 +13,7 @@ #include #include #include +#include #include @@ -69,15 +72,17 @@ ENTRY(__put_user_8) sub $7,%_ASM_BX cmp %_ASM_BX,%_ASM_CX jae bad_put_user -4: movl %_ASM_AX,(%_ASM_CX) +4: mov %_ASM_AX,(%_ASM_CX) +#ifdef CONFIG_X86_32 5: movl %edx,4(%_ASM_CX) +#endif xor %eax,%eax EXIT ENDPROC(__put_user_8) bad_put_user: CFI_STARTPROC - movl $-14,%eax + movl $-EFAULT,%eax EXIT END(bad_put_user) @@ -86,5 +91,7 @@ END(bad_put_user) _ASM_PTR 2b,bad_put_user _ASM_PTR 3b,bad_put_user _ASM_PTR 4b,bad_put_user +#ifdef CONFIG_X86_32 _ASM_PTR 5b,bad_put_user +#endif .previous diff --git a/arch/x86/lib/putuser_64.S b/arch/x86/lib/putuser_64.S deleted file mode 100644 index d496cc8e7308..000000000000 --- a/arch/x86/lib/putuser_64.S +++ /dev/null @@ -1,94 +0,0 @@ -/* - * __put_user functions. - * - * (C) Copyright 1998 Linus Torvalds - * (C) Copyright 2005 Andi Kleen - * - * These functions have a non-standard call interface - * to make them more efficient, especially as they - * return an error value in addition to the "real" - * return value. - */ - -/* - * __put_user_X - * - * Inputs: %rcx contains the address - * %rdx contains new value - * - * Outputs: %rax is error code (0 or -EFAULT) - * - * %rbx is destroyed. - * - * These functions should not modify any other registers, - * as they get called from within inline assembly. - */ - -#include -#include -#include -#include -#include -#include -#include - -#define ENTER CFI_STARTPROC ; \ - GET_THREAD_INFO(%_ASM_BX) -#define EXIT ret ; \ - CFI_ENDPROC - - .text -ENTRY(__put_user_1) - ENTER - cmp TI_addr_limit(%_ASM_BX),%_ASM_CX - jae bad_put_user -1: movb %al,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_1) - -ENTRY(__put_user_2) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $1, %_ASM_BX - cmp %_ASM_BX ,%_ASM_CX - jae bad_put_user -2: movw %ax,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_2) - -ENTRY(__put_user_4) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $3, %_ASM_BX - cmp %_ASM_BX, %_ASM_CX - jae bad_put_user -3: movl %eax,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_4) - -ENTRY(__put_user_8) - ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $7, %_ASM_BX - cmp %_ASM_BX, %_ASM_CX - jae bad_put_user -4: movq %_ASM_AX,(%_ASM_CX) - xor %eax,%eax - EXIT -ENDPROC(__put_user_8) - -bad_put_user: - CFI_STARTPROC - mov $(-EFAULT),%eax - EXIT -END(bad_put_user) - -.section __ex_table,"a" - _ASM_PTR 1b,bad_put_user - _ASM_PTR 2b,bad_put_user - _ASM_PTR 3b,bad_put_user - _ASM_PTR 4b,bad_put_user -.previous From 53938a68a2f971058bc53aaa8c70f9f24f684cdc Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 10:14:13 -0300 Subject: [PATCH 24/43] x86: commonize __range_not_ok. For i386, __range_not_ok is a better name than __range_ok, since it returns 0 when it is in fact okay. Other than that, both versions does not need the word size specifiers, and we remove them. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_32.h | 6 +++--- include/asm-x86/uaccess_64.h | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index f8abc12a77c2..8d3f02d3562c 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -56,11 +56,11 @@ extern struct movsl_mask { * * This needs 33-bit arithmetic. We have a carry... */ -#define __range_ok(addr, size) \ +#define __range_not_ok(addr, size) \ ({ \ unsigned long flag, roksum; \ __chk_user_ptr(addr); \ - asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \ + asm("add %3,%1 ; sbb %0,%0; cmp %1,%4; sbb $0,%0" \ :"=&r" (flag), "=r" (roksum) \ :"1" (addr), "g" ((int)(size)), \ "rm" (current_thread_info()->addr_limit.seg)); \ @@ -86,7 +86,7 @@ extern struct movsl_mask { * checks that the pointer is in the user space range - after calling * this function, memory access functions may still return -EFAULT. */ -#define access_ok(type, addr, size) (likely(__range_ok(addr, size) == 0)) +#define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0)) /* * The exception table consists of pairs of addresses: the first is the diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index f822a36cc1c0..012cba3972d7 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -41,8 +41,7 @@ ({ \ unsigned long flag, roksum; \ __chk_user_ptr(addr); \ - asm("# range_ok\n\r" \ - "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ + asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \ : "=&r" (flag), "=r" (roksum) \ : "1" (addr), "g" ((long)(size)), \ "g" (current_thread_info()->addr_limit.seg)); \ From c28b95d9bb7da0c3be22826c56a05899e21e5ece Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Mon, 30 Jun 2008 17:07:51 -0300 Subject: [PATCH 25/43] x86: change asm constraint. Our integration efforts broke a build with this function being used with i386. Reason is "g" can put the operand in an imm32, which according to The Book (tm), is invalid as the second operand. This is actually a bug in x86_64 too, since the x86_64 instruction set reference does not list it as valid. We probably didn't trigger this before due to the ammount of registers available for 64-bit platforms. But that's just my guess. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 012cba3972d7..dc6dde058381 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -44,7 +44,7 @@ asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \ : "=&r" (flag), "=r" (roksum) \ : "1" (addr), "g" ((long)(size)), \ - "g" (current_thread_info()->addr_limit.seg)); \ + "rm" (current_thread_info()->addr_limit.seg)); \ flag; \ }) From 8b0a8aaf05325a1a96f53b45708f77599da35161 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Mon, 30 Jun 2008 17:34:39 -0300 Subject: [PATCH 26/43] x86: introduce likely in macro. Put the likely hint in access_ok. Just for bisectability. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index dc6dde058381..0077dbe9359e 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -48,7 +48,7 @@ flag; \ }) -#define access_ok(type, addr, size) (__range_not_ok(addr, size) == 0) +#define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0)) /* * The exception table consists of pairs of addresses: the first is the From 1dc186e82c1ab476ef83080adca43a70969b01cd Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Mon, 30 Jun 2008 17:37:08 -0300 Subject: [PATCH 27/43] x86: use long instead of int. Do not refer to the processor word-size with int, as it won't work with x86_64. Use long instead. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_32.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 8d3f02d3562c..eed8d9422b4d 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -62,7 +62,7 @@ extern struct movsl_mask { __chk_user_ptr(addr); \ asm("add %3,%1 ; sbb %0,%0; cmp %1,%4; sbb $0,%0" \ :"=&r" (flag), "=r" (roksum) \ - :"1" (addr), "g" ((int)(size)), \ + :"1" (addr), "g" ((long)(size)), \ "rm" (current_thread_info()->addr_limit.seg)); \ flag; \ }) From be9d06bfd48934fbd56ccb7476eabccfa31b4afe Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Fri, 13 Jun 2008 21:01:46 -0300 Subject: [PATCH 28/43] x86: use something common for both architectures. Using explicit hexa (0xFFFFFFUL) introduces an unnecessary difference between i386 and x86_64 because of the size of their long. Use -1UL instead. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_32.h | 2 +- include/asm-x86/uaccess_64.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index eed8d9422b4d..2676b48ac0fa 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -25,7 +25,7 @@ #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) -#define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFFUL) +#define KERNEL_DS MAKE_MM_SEG(-1UL) #define USER_DS MAKE_MM_SEG(PAGE_OFFSET) #define get_ds() (KERNEL_DS) diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 0077dbe9359e..3a81775136c8 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -22,7 +22,7 @@ #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) -#define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFFFFFFFFFFUL) +#define KERNEL_DS MAKE_MM_SEG(-1UL) #define USER_DS MAKE_MM_SEG(PAGE_OFFSET) #define get_ds() (KERNEL_DS) From ca23386216b9d4fc3bb211101205077d2b2916ae Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Fri, 13 Jun 2008 14:39:25 -0300 Subject: [PATCH 29/43] x86: merge common parts of uaccess. Common parts of uaccess_32.h and uaccess_64.h are put in uaccess.h. Bits in uaccess_32.h and uaccess_64.h that come to this file are equal except for comments and whitespaces differences. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 124 +++++++++++++++++++++++++++++++++++ include/asm-x86/uaccess_32.h | 110 ------------------------------- include/asm-x86/uaccess_64.h | 83 ----------------------- 3 files changed, 124 insertions(+), 193 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index 9fefd2947e78..2fc30c2a8a98 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -1,5 +1,129 @@ +#ifndef _ASM_UACCES_H_ +#define _ASM_UACCES_H_ +/* + * User space memory access functions + */ +#include +#include +#include +#include +#include +#include +#include + +#define VERIFY_READ 0 +#define VERIFY_WRITE 1 + +/* + * The fs value determines whether argument validity checking should be + * performed or not. If get_fs() == USER_DS, checking is performed, with + * get_fs() == KERNEL_DS, checking is bypassed. + * + * For historical reasons, these macros are grossly misnamed. + */ + +#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) + +#define KERNEL_DS MAKE_MM_SEG(-1UL) +#define USER_DS MAKE_MM_SEG(PAGE_OFFSET) + +#define get_ds() (KERNEL_DS) +#define get_fs() (current_thread_info()->addr_limit) +#define set_fs(x) (current_thread_info()->addr_limit = (x)) + +#define segment_eq(a, b) ((a).seg == (b).seg) + +/* + * Test whether a block of memory is a valid user space address. + * Returns 0 if the range is valid, nonzero otherwise. + * + * This is equivalent to the following test: + * (u33)addr + (u33)size >= (u33)current->addr_limit.seg (u65 for x86_64) + * + * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry... + */ + +#define __range_not_ok(addr, size) \ +({ \ + unsigned long flag, roksum; \ + __chk_user_ptr(addr); \ + asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \ + : "=&r" (flag), "=r" (roksum) \ + : "1" (addr), "g" ((long)(size)), \ + "rm" (current_thread_info()->addr_limit.seg)); \ + flag; \ +}) + +/** + * access_ok: - Checks if a user space pointer is valid + * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that + * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe + * to write to a block, it is always safe to read from it. + * @addr: User space pointer to start of block to check + * @size: Size of block to check + * + * Context: User context only. This function may sleep. + * + * Checks if a pointer to a block of memory in user space is valid. + * + * Returns true (nonzero) if the memory block may be valid, false (zero) + * if it is definitely invalid. + * + * Note that, depending on architecture, this function probably just + * checks that the pointer is in the user space range - after calling + * this function, memory access functions may still return -EFAULT. + */ +#define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0)) + +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + unsigned long insn, fixup; +}; + +extern int fixup_exception(struct pt_regs *regs); + +/* + * These are the main single-value transfer routines. They automatically + * use the right size if we just have the right pointer type. + * + * This gets kind of ugly. We want to return _two_ values in "get_user()" + * and yet we don't want to do any pointers, because that is too much + * of a performance impact. Thus we have a few rather ugly macros here, + * and hide all the ugliness from the user. + * + * The "__xxx" versions of the user access functions are versions that + * do not verify the address space, that must have been done previously + * with a separate "access_ok()" call (this is used when we do multiple + * accesses to the same area of user memory). + */ + +extern int __get_user_1(void); +extern int __get_user_2(void); +extern int __get_user_4(void); +extern int __get_user_8(void); +extern int __get_user_bad(void); + +#define __get_user_x(size, ret, x, ptr) \ + asm volatile("call __get_user_" #size \ + : "=a" (ret),"=d" (x) \ + : "0" (ptr)) \ + #ifdef CONFIG_X86_32 # include "uaccess_32.h" #else # include "uaccess_64.h" #endif + +#endif diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 2676b48ac0fa..92ad19e70989 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -11,29 +11,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - */ - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - - -#define KERNEL_DS MAKE_MM_SEG(-1UL) -#define USER_DS MAKE_MM_SEG(PAGE_OFFSET) - -#define get_ds() (KERNEL_DS) -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) - -#define segment_eq(a, b) ((a).seg == (b).seg) - /* * movsl can be slow when source and dest are not both 8-byte aligned */ @@ -47,91 +24,6 @@ extern struct movsl_mask { ((unsigned long __force)(addr) < \ (current_thread_info()->addr_limit.seg)) -/* - * Test whether a block of memory is a valid user space address. - * Returns 0 if the range is valid, nonzero otherwise. - * - * This is equivalent to the following test: - * (u33)addr + (u33)size >= (u33)current->addr_limit.seg - * - * This needs 33-bit arithmetic. We have a carry... - */ -#define __range_not_ok(addr, size) \ -({ \ - unsigned long flag, roksum; \ - __chk_user_ptr(addr); \ - asm("add %3,%1 ; sbb %0,%0; cmp %1,%4; sbb $0,%0" \ - :"=&r" (flag), "=r" (roksum) \ - :"1" (addr), "g" ((long)(size)), \ - "rm" (current_thread_info()->addr_limit.seg)); \ - flag; \ -}) - -/** - * access_ok: - Checks if a user space pointer is valid - * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that - * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe - * to write to a block, it is always safe to read from it. - * @addr: User space pointer to start of block to check - * @size: Size of block to check - * - * Context: User context only. This function may sleep. - * - * Checks if a pointer to a block of memory in user space is valid. - * - * Returns true (nonzero) if the memory block may be valid, false (zero) - * if it is definitely invalid. - * - * Note that, depending on architecture, this function probably just - * checks that the pointer is in the user space range - after calling - * this function, memory access functions may still return -EFAULT. - */ -#define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0)) - -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn, fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); - -/* - * These are the main single-value transfer routines. They automatically - * use the right size if we just have the right pointer type. - * - * This gets kind of ugly. We want to return _two_ values in "get_user()" - * and yet we don't want to do any pointers, because that is too much - * of a performance impact. Thus we have a few rather ugly macros here, - * and hide all the ugliness from the user. - * - * The "__xxx" versions of the user access functions are versions that - * do not verify the address space, that must have been done previously - * with a separate "access_ok()" call (this is used when we do multiple - * accesses to the same area of user memory). - */ - -extern void __get_user_1(void); -extern void __get_user_2(void); -extern void __get_user_4(void); - -#define __get_user_x(size, ret, x, ptr) \ - asm volatile("call __get_user_" #size \ - :"=a" (ret),"=d" (x) \ - :"0" (ptr)) - - /* Careful: we have to cast the result to the type of the pointer * for sign reasons */ @@ -386,8 +278,6 @@ struct __large_struct { unsigned long buf[100]; }; __gu_err; \ }) -extern long __get_user_bad(void); - #define __get_user_size(x, ptr, size, retval, errret) \ do { \ retval = 0; \ diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 3a81775136c8..243dbb467f3a 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -9,88 +9,11 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - */ - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - -#define KERNEL_DS MAKE_MM_SEG(-1UL) -#define USER_DS MAKE_MM_SEG(PAGE_OFFSET) - -#define get_ds() (KERNEL_DS) -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) - -#define segment_eq(a, b) ((a).seg == (b).seg) - #define __addr_ok(addr) (!((unsigned long)(addr) & \ (current_thread_info()->addr_limit.seg))) -/* - * Uhhuh, this needs 65-bit arithmetic. We have a carry.. - */ -#define __range_not_ok(addr, size) \ -({ \ - unsigned long flag, roksum; \ - __chk_user_ptr(addr); \ - asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \ - : "=&r" (flag), "=r" (roksum) \ - : "1" (addr), "g" ((long)(size)), \ - "rm" (current_thread_info()->addr_limit.seg)); \ - flag; \ -}) - -#define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0)) - -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn, fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); - #define ARCH_HAS_SEARCH_EXTABLE -/* - * These are the main single-value transfer routines. They automatically - * use the right size if we just have the right pointer type. - * - * This gets kind of ugly. We want to return _two_ values in "get_user()" - * and yet we don't want to do any pointers, because that is too much - * of a performance impact. Thus we have a few rather ugly macros here, - * and hide all the ugliness from the user. - * - * The "__xxx" versions of the user access functions are versions that - * do not verify the address space, that must have been done previously - * with a separate "access_ok()" call (this is used when we do multiple - * accesses to the same area of user memory). - */ - -#define __get_user_x(size, ret, x, ptr) \ - asm volatile("call __get_user_" #size \ - : "=a" (ret),"=d" (x) \ - : "0" (ptr)) \ - /* Careful: we have to cast the result to the type of the pointer * for sign reasons */ @@ -226,12 +149,6 @@ struct __large_struct { unsigned long buf[100]; }; __gu_err; \ }) -extern int __get_user_1(void); -extern int __get_user_2(void); -extern int __get_user_4(void); -extern int __get_user_8(void); -extern int __get_user_bad(void); - #define __get_user_size(x, ptr, size, retval) \ do { \ retval = 0; \ From 865e5b76505cdade261773bde32f785b3ce579f1 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 11:05:11 -0300 Subject: [PATCH 30/43] x86: merge getuser. Merge versions of getuser from uaccess_32.h and uaccess_64.h into uaccess.h. There is a part which is 64-bit only (for now), and for that, we use a __get_user_8 macro. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 55 ++++++++++++++++++++++++++++++++++++ include/asm-x86/uaccess_32.h | 43 ---------------------------- include/asm-x86/uaccess_64.h | 29 ------------------- 3 files changed, 55 insertions(+), 72 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index 2fc30c2a8a98..8cb580c0b411 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -120,6 +120,61 @@ extern int __get_user_bad(void); : "=a" (ret),"=d" (x) \ : "0" (ptr)) \ +/* Careful: we have to cast the result to the type of the pointer + * for sign reasons */ + +/** + * get_user: - Get a simple variable from user space. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ +#ifdef CONFIG_X86_32 +#define __get_user_8(__ret_gu, __val_gu, ptr) \ + __get_user_x(X, __ret_gu, __val_gu, ptr) +#else +#define __get_user_8(__ret_gu, __val_gu, ptr) \ + __get_user_x(8, __ret_gu, __val_gu, ptr) +#endif + +#define get_user(x, ptr) \ +({ \ + int __ret_gu; \ + unsigned long __val_gu; \ + __chk_user_ptr(ptr); \ + switch (sizeof(*(ptr))) { \ + case 1: \ + __get_user_x(1, __ret_gu, __val_gu, ptr); \ + break; \ + case 2: \ + __get_user_x(2, __ret_gu, __val_gu, ptr); \ + break; \ + case 4: \ + __get_user_x(4, __ret_gu, __val_gu, ptr); \ + break; \ + case 8: \ + __get_user_8(__ret_gu, __val_gu, ptr); \ + break; \ + default: \ + __get_user_x(X, __ret_gu, __val_gu, ptr); \ + break; \ + } \ + (x) = (__typeof__(*(ptr)))__val_gu; \ + __ret_gu; \ +}) + + #ifdef CONFIG_X86_32 # include "uaccess_32.h" #else diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 92ad19e70989..3cc323694438 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -24,49 +24,6 @@ extern struct movsl_mask { ((unsigned long __force)(addr) < \ (current_thread_info()->addr_limit.seg)) -/* Careful: we have to cast the result to the type of the pointer - * for sign reasons */ - -/** - * get_user: - Get a simple variable from user space. - * @x: Variable to store result. - * @ptr: Source address, in user space. - * - * Context: User context only. This function may sleep. - * - * This macro copies a single simple variable from user space to kernel - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. - * - * @ptr must have pointer-to-simple-variable type, and the result of - * dereferencing @ptr must be assignable to @x without a cast. - * - * Returns zero on success, or -EFAULT on error. - * On error, the variable @x is set to zero. - */ -#define get_user(x, ptr) \ -({ \ - int __ret_gu; \ - unsigned long __val_gu; \ - __chk_user_ptr(ptr); \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __get_user_x(1, __ret_gu, __val_gu, ptr); \ - break; \ - case 2: \ - __get_user_x(2, __ret_gu, __val_gu, ptr); \ - break; \ - case 4: \ - __get_user_x(4, __ret_gu, __val_gu, ptr); \ - break; \ - default: \ - __get_user_x(X, __ret_gu, __val_gu, ptr); \ - break; \ - } \ - (x) = (__typeof__(*(ptr)))__val_gu; \ - __ret_gu; \ -}) - extern void __put_user_bad(void); /* diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 243dbb467f3a..4a44b906e4c3 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -14,35 +14,6 @@ #define ARCH_HAS_SEARCH_EXTABLE -/* Careful: we have to cast the result to the type of the pointer - * for sign reasons */ - -#define get_user(x, ptr) \ -({ \ - unsigned long __val_gu; \ - int __ret_gu; \ - __chk_user_ptr(ptr); \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __get_user_x(1, __ret_gu, __val_gu, ptr); \ - break; \ - case 2: \ - __get_user_x(2, __ret_gu, __val_gu, ptr); \ - break; \ - case 4: \ - __get_user_x(4, __ret_gu, __val_gu, ptr); \ - break; \ - case 8: \ - __get_user_x(8, __ret_gu, __val_gu, ptr); \ - break; \ - default: \ - __get_user_bad(); \ - break; \ - } \ - (x) = (__force typeof(*(ptr)))__val_gu; \ - __ret_gu; \ -}) - extern void __put_user_1(void); extern void __put_user_2(void); extern void __put_user_4(void); From 002ca1690c3d0a495e6aedd608281aeb01ce6385 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 11:08:51 -0300 Subject: [PATCH 31/43] x86: move __addr_ok to uaccess.h. Take it out of uaccess_32.h. Since it seems that no users of the x86_64 exists, we simply pick the i386 version. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 4 ++++ include/asm-x86/uaccess_32.h | 4 ---- include/asm-x86/uaccess_64.h | 3 --- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index 8cb580c0b411..8a1e45fdc980 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -33,6 +33,10 @@ #define segment_eq(a, b) ((a).seg == (b).seg) +#define __addr_ok(addr) \ + ((unsigned long __force)(addr) < \ + (current_thread_info()->addr_limit.seg)) + /* * Test whether a block of memory is a valid user space address. * Returns 0 if the range is valid, nonzero otherwise. diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 3cc323694438..87b1aede9d4b 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -20,10 +20,6 @@ extern struct movsl_mask { } ____cacheline_aligned_in_smp movsl_mask; #endif -#define __addr_ok(addr) \ - ((unsigned long __force)(addr) < \ - (current_thread_info()->addr_limit.seg)) - extern void __put_user_bad(void); /* diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 4a44b906e4c3..81308763a4c1 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -9,9 +9,6 @@ #include #include -#define __addr_ok(addr) (!((unsigned long)(addr) & \ - (current_thread_info()->addr_limit.seg))) - #define ARCH_HAS_SEARCH_EXTABLE extern void __put_user_1(void); From 5e322163b19735fbef3e294c297d38e0d2ba8f7e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 11:35:06 -0300 Subject: [PATCH 32/43] x86: use k modifier for 4-byte access. Do it in a separate patch for bisectability. Goal is to have put_user_size integrated. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_32.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 87b1aede9d4b..4c47a5ba65e3 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -180,7 +180,7 @@ do { \ __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ break; \ case 4: \ - __put_user_asm(x, ptr, retval, "l", "", "ir", errret); \ + __put_user_asm(x, ptr, retval, "l", "k", "ir", errret);\ break; \ case 8: \ __put_user_u64((__typeof__(*ptr))(x), ptr, retval); \ From 293e6a2524ae775fef41d21d17af288d4b2ee91e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 11:40:42 -0300 Subject: [PATCH 33/43] x86: mark x86_64 as having a working WP. Select X86_WP_WORKS_OK for x86_64 too. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- arch/x86/Kconfig.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index 3d22bb8175b4..abff1b84ed5b 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -344,7 +344,7 @@ config X86_F00F_BUG config X86_WP_WORKS_OK def_bool y - depends on X86_32 && !M386 + depends on !M386 config X86_INVLPG def_bool y From d42e6af613375be7a9a431628ecd742e87230554 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 12:00:44 -0300 Subject: [PATCH 34/43] x86: don't always use EFAULT on __put_user_size. Let the user of the macro specify the desired return. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_64.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 81308763a4c1..6532d638d5a3 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -37,7 +37,7 @@ extern void __put_user_bad(void); #define __put_user_nocheck(x, ptr, size) \ ({ \ int __pu_err; \ - __put_user_size((x), (ptr), (size), __pu_err); \ + __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ __pu_err; \ }) @@ -65,22 +65,22 @@ extern void __put_user_bad(void); __pu_err; \ }) -#define __put_user_size(x, ptr, size, retval) \ +#define __put_user_size(x, ptr, size, retval, errret) \ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: \ - __put_user_asm(x, ptr, retval, "b", "b", "iq", -EFAULT);\ + __put_user_asm(x, ptr, retval, "b", "b", "iq", errret);\ break; \ case 2: \ - __put_user_asm(x, ptr, retval, "w", "w", "ir", -EFAULT);\ + __put_user_asm(x, ptr, retval, "w", "w", "ir", errret);\ break; \ case 4: \ - __put_user_asm(x, ptr, retval, "l", "k", "ir", -EFAULT);\ + __put_user_asm(x, ptr, retval, "l", "k", "ir", errret);\ break; \ case 8: \ - __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT); \ + __put_user_asm(x, ptr, retval, "q", "", "Zr", errret); \ break; \ default: \ __put_user_bad(); \ From dc70ddf4098de043ac52f623c7573a11f2ae3d09 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 11:48:29 -0300 Subject: [PATCH 35/43] x86: merge __put_user_asm and its user. Move both __put_user_asm and __put_user_size to uaccess.h. i386 already had a special function for 64-bit access, so for x86_64, we just define a macro with the same name. Note that for X86_64, CONFIG_X86_WP_WORKS_OK will always be defined, so the #else part will never be even compiled in. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 84 ++++++++++++++++++++++++++++++++++++ include/asm-x86/uaccess_32.h | 77 --------------------------------- include/asm-x86/uaccess_64.h | 51 ---------------------- 3 files changed, 84 insertions(+), 128 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index 8a1e45fdc980..bcda5d075921 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -178,6 +178,90 @@ extern int __get_user_bad(void); __ret_gu; \ }) +#ifdef CONFIG_X86_32 +#define __put_user_u64(x, addr, err) \ + asm volatile("1: movl %%eax,0(%2)\n" \ + "2: movl %%edx,4(%2)\n" \ + "3:\n" \ + ".section .fixup,\"ax\"\n" \ + "4: movl %3,%0\n" \ + " jmp 3b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ + : "=r" (err) \ + : "A" (x), "r" (addr), "i" (-EFAULT), "0" (err)) +#else +#define __put_user_u64(x, ptr, retval) \ + __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT) +#endif + +#ifdef CONFIG_X86_WP_WORKS_OK + +#define __put_user_size(x, ptr, size, retval, errret) \ +do { \ + retval = 0; \ + __chk_user_ptr(ptr); \ + switch (size) { \ + case 1: \ + __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ + break; \ + case 2: \ + __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ + break; \ + case 4: \ + __put_user_asm(x, ptr, retval, "l", "k", "ir", errret);\ + break; \ + case 8: \ + __put_user_u64((__typeof__(*ptr))(x), ptr, retval); \ + break; \ + default: \ + __put_user_bad(); \ + } \ +} while (0) + +#else + +#define __put_user_size(x, ptr, size, retval, errret) \ +do { \ + __typeof__(*(ptr))__pus_tmp = x; \ + retval = 0; \ + \ + if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \ + retval = errret; \ +} while (0) + +#endif + +#define __put_user_nocheck(x, ptr, size) \ +({ \ + long __pu_err; \ + __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ + __pu_err; \ +}) + + + +/* FIXME: this hack is definitely wrong -AK */ +struct __large_struct { unsigned long buf[100]; }; +#define __m(x) (*(struct __large_struct __user *)(x)) + +/* + * Tell gcc we read from memory instead of writing: this is because + * we do not write to any memory gcc knows about, so there are no + * aliasing issues. + */ +#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ + asm volatile("1: mov"itype" %"rtype"1,%2\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: mov %3,%0\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r"(err) \ + : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) + #ifdef CONFIG_X86_32 # include "uaccess_32.h" diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 4c47a5ba65e3..fab755781b9b 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -145,83 +145,6 @@ extern void __put_user_8(void); #define __put_user(x, ptr) \ __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) -#define __put_user_nocheck(x, ptr, size) \ -({ \ - long __pu_err; \ - __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ - __pu_err; \ -}) - - -#define __put_user_u64(x, addr, err) \ - asm volatile("1: movl %%eax,0(%2)\n" \ - "2: movl %%edx,4(%2)\n" \ - "3:\n" \ - ".section .fixup,\"ax\"\n" \ - "4: movl %3,%0\n" \ - " jmp 3b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 4b) \ - _ASM_EXTABLE(2b, 4b) \ - : "=r" (err) \ - : "A" (x), "r" (addr), "i" (-EFAULT), "0" (err)) - -#ifdef CONFIG_X86_WP_WORKS_OK - -#define __put_user_size(x, ptr, size, retval, errret) \ -do { \ - retval = 0; \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ - break; \ - case 2: \ - __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ - break; \ - case 4: \ - __put_user_asm(x, ptr, retval, "l", "k", "ir", errret);\ - break; \ - case 8: \ - __put_user_u64((__typeof__(*ptr))(x), ptr, retval); \ - break; \ - default: \ - __put_user_bad(); \ - } \ -} while (0) - -#else - -#define __put_user_size(x, ptr, size, retval, errret) \ -do { \ - __typeof__(*(ptr))__pus_tmp = x; \ - retval = 0; \ - \ - if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \ - retval = errret; \ -} while (0) - -#endif -struct __large_struct { unsigned long buf[100]; }; -#define __m(x) (*(struct __large_struct __user *)(x)) - -/* - * Tell gcc we read from memory instead of writing: this is because - * we do not write to any memory gcc knows about, so there are no - * aliasing issues. - */ -#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ - asm volatile("1: mov"itype" %"rtype"1,%2\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r"(err) \ - : ltype (x), "m" (__m(addr)), "i" (errret), "0" (err)) - - #define __get_user_nocheck(x, ptr, size) \ ({ \ long __gu_err; \ diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 6532d638d5a3..42c01aa70e72 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -34,14 +34,6 @@ extern void __put_user_bad(void); #define __get_user_unaligned __get_user #define __put_user_unaligned __put_user -#define __put_user_nocheck(x, ptr, size) \ -({ \ - int __pu_err; \ - __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ - __pu_err; \ -}) - - #define __put_user_check(x, ptr, size) \ ({ \ int __pu_err; \ @@ -65,49 +57,6 @@ extern void __put_user_bad(void); __pu_err; \ }) -#define __put_user_size(x, ptr, size, retval, errret) \ -do { \ - retval = 0; \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __put_user_asm(x, ptr, retval, "b", "b", "iq", errret);\ - break; \ - case 2: \ - __put_user_asm(x, ptr, retval, "w", "w", "ir", errret);\ - break; \ - case 4: \ - __put_user_asm(x, ptr, retval, "l", "k", "ir", errret);\ - break; \ - case 8: \ - __put_user_asm(x, ptr, retval, "q", "", "Zr", errret); \ - break; \ - default: \ - __put_user_bad(); \ - } \ -} while (0) - -/* FIXME: this hack is definitely wrong -AK */ -struct __large_struct { unsigned long buf[100]; }; -#define __m(x) (*(struct __large_struct __user *)(x)) - -/* - * Tell gcc we read from memory instead of writing: this is because - * we do not write to any memory gcc knows about, so there are no - * aliasing issues. - */ -#define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - asm volatile("1: mov"itype" %"rtype"1,%2\n" \ - "2:\n" \ - ".section .fixup, \"ax\"\n" \ - "3: mov %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r"(err) \ - : ltype (x), "m" (__m(addr)), "i" (errno), "0" (err)) - - #define __get_user_nocheck(x, ptr, size) \ ({ \ int __gu_err; \ From 6fd4725d75c5049641640ce23fa5896501dfbd42 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 11:57:33 -0300 Subject: [PATCH 36/43] x86: don't always use EFAULT on __get_user_size. Let the user of the macro specify the desired return. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_64.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 42c01aa70e72..e0875d7ae898 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -61,27 +61,27 @@ extern void __put_user_bad(void); ({ \ int __gu_err; \ unsigned long __gu_val; \ - __get_user_size(__gu_val, (ptr), (size), __gu_err); \ + __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);\ (x) = (__force typeof(*(ptr)))__gu_val; \ __gu_err; \ }) -#define __get_user_size(x, ptr, size, retval) \ +#define __get_user_size(x, ptr, size, retval, errret) \ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: \ - __get_user_asm(x, ptr, retval, "b", "b", "=q", -EFAULT);\ + __get_user_asm(x, ptr, retval, "b", "b", "=q", errret);\ break; \ case 2: \ - __get_user_asm(x, ptr, retval, "w", "w", "=r", -EFAULT);\ + __get_user_asm(x, ptr, retval, "w", "w", "=r", errret);\ break; \ case 4: \ - __get_user_asm(x, ptr, retval, "l", "k", "=r", -EFAULT);\ + __get_user_asm(x, ptr, retval, "l", "k", "=r", errret);\ break; \ case 8: \ - __get_user_asm(x, ptr, retval, "q", "", "=r", -EFAULT); \ + __get_user_asm(x, ptr, retval, "q", "", "=r", errret); \ break; \ default: \ (x) = __get_user_bad(); \ From 3f168221167ca7a844fde3bf58e1c7ca0bf9741e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 12:48:47 -0300 Subject: [PATCH 37/43] x86: merge __get_user_asm and its users. Move __get_user_asm and __get_user_size and __get_user_nocheck to uaccess.h. This requires us to define a macro at __get_user_size for the 64-bit access case. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 50 +++++++++++++++++++++++++++++++++++- include/asm-x86/uaccess_32.h | 41 ----------------------------- include/asm-x86/uaccess_64.h | 43 ------------------------------- 3 files changed, 49 insertions(+), 85 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index bcda5d075921..4353b2267a02 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -233,6 +233,47 @@ do { \ #endif +#ifdef CONFIG_X86_32 +#define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad() +#else +#define __get_user_asm_u64(x, ptr, retval, errret) \ + __get_user_asm(x, ptr, retval, "q", "", "=r", errret) +#endif + +#define __get_user_size(x, ptr, size, retval, errret) \ +do { \ + retval = 0; \ + __chk_user_ptr(ptr); \ + switch (size) { \ + case 1: \ + __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ + break; \ + case 2: \ + __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ + break; \ + case 4: \ + __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \ + break; \ + case 8: \ + __get_user_asm_u64(x, ptr, retval, errret); \ + break; \ + default: \ + (x) = __get_user_bad(); \ + } \ +} while (0) + +#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ + asm volatile("1: mov"itype" %2,%"rtype"1\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: mov %3,%0\n" \ + " xor"itype" %"rtype"1,%"rtype"1\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r" (err), ltype(x) \ + : "m" (__m(addr)), "i" (errret), "0" (err)) + #define __put_user_nocheck(x, ptr, size) \ ({ \ long __pu_err; \ @@ -240,7 +281,14 @@ do { \ __pu_err; \ }) - +#define __get_user_nocheck(x, ptr, size) \ +({ \ + long __gu_err; \ + unsigned long __gu_val; \ + __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ + (x) = (__force __typeof__(*(ptr)))__gu_val; \ + __gu_err; \ +}) /* FIXME: this hack is definitely wrong -AK */ struct __large_struct { unsigned long buf[100]; }; diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index fab755781b9b..ebfe6b23ba57 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -145,47 +145,6 @@ extern void __put_user_8(void); #define __put_user(x, ptr) \ __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) -#define __get_user_nocheck(x, ptr, size) \ -({ \ - long __gu_err; \ - unsigned long __gu_val; \ - __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ - (x) = (__typeof__(*(ptr)))__gu_val; \ - __gu_err; \ -}) - -#define __get_user_size(x, ptr, size, retval, errret) \ -do { \ - retval = 0; \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ - break; \ - case 2: \ - __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ - break; \ - case 4: \ - __get_user_asm(x, ptr, retval, "l", "", "=r", errret); \ - break; \ - default: \ - (x) = __get_user_bad(); \ - } \ -} while (0) - -#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ - asm volatile("1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl %3,%0\n" \ - " xor"itype" %"rtype"1,%"rtype"1\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r" (err), ltype (x) \ - : "m" (__m(addr)), "i" (errret), "0" (err)) - - unsigned long __must_check __copy_to_user_ll (void __user *to, const void *from, unsigned long n); unsigned long __must_check __copy_from_user_ll diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index e0875d7ae898..42a9769687e9 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -57,49 +57,6 @@ extern void __put_user_bad(void); __pu_err; \ }) -#define __get_user_nocheck(x, ptr, size) \ -({ \ - int __gu_err; \ - unsigned long __gu_val; \ - __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);\ - (x) = (__force typeof(*(ptr)))__gu_val; \ - __gu_err; \ -}) - -#define __get_user_size(x, ptr, size, retval, errret) \ -do { \ - retval = 0; \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __get_user_asm(x, ptr, retval, "b", "b", "=q", errret);\ - break; \ - case 2: \ - __get_user_asm(x, ptr, retval, "w", "w", "=r", errret);\ - break; \ - case 4: \ - __get_user_asm(x, ptr, retval, "l", "k", "=r", errret);\ - break; \ - case 8: \ - __get_user_asm(x, ptr, retval, "q", "", "=r", errret); \ - break; \ - default: \ - (x) = __get_user_bad(); \ - } \ -} while (0) - -#define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - asm volatile("1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup, \"ax\"\n" \ - "3: mov %3,%0\n" \ - " xor"itype" %"rtype"1,%"rtype"1\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r" (err), ltype (x) \ - : "m" (__m(addr)), "i"(errno), "0"(err)) - /* * Copy To/From Userspace */ From 64a978c1e3b2c0e2b663caf54719319919c5c17f Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 12:59:37 -0300 Subject: [PATCH 38/43] x86: be more explicit in __put_user_x. For both __put_user_x and __put_user_8 macros, pass the error variable explicitly. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_32.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index ebfe6b23ba57..2c9067324d2c 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -31,11 +31,11 @@ extern void __put_user_2(void); extern void __put_user_4(void); extern void __put_user_8(void); -#define __put_user_x(size, x, ptr) \ +#define __put_user_x(size, x, ptr, __ret_pu) \ asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ :"0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") -#define __put_user_8(x, ptr) \ +#define __put_user_8(x, ptr, __ret_pu) \ asm volatile("call __put_user_8" : "=a" (__ret_pu) \ : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") @@ -66,19 +66,19 @@ extern void __put_user_8(void); __pu_val = x; \ switch (sizeof(*(ptr))) { \ case 1: \ - __put_user_x(1, __pu_val, ptr); \ + __put_user_x(1, __pu_val, ptr, __ret_pu); \ break; \ case 2: \ - __put_user_x(2, __pu_val, ptr); \ + __put_user_x(2, __pu_val, ptr, __ret_pu); \ break; \ case 4: \ - __put_user_x(4, __pu_val, ptr); \ + __put_user_x(4, __pu_val, ptr, __ret_pu); \ break; \ case 8: \ - __put_user_8(__pu_val, ptr); \ + __put_user_8(__pu_val, ptr, __ret_pu); \ break; \ default: \ - __put_user_x(X, __pu_val, ptr); \ + __put_user_x(X, __pu_val, ptr, __ret_pu); \ break; \ } \ __ret_pu; \ From ec840956d269e2beefe3a1c0fd7c1c70a199087f Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 13:00:48 -0300 Subject: [PATCH 39/43] x86: turn __put_user_check directly into put_user. We also check user pointer in x86_64 put_user, the way i386 does. In a separate patch for bisecting purposes. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess_64.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 42a9769687e9..9139854d37b0 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -23,9 +23,6 @@ extern void __put_user_bad(void); :"c" (ptr),"a" (x) \ :"ebx") -#define put_user(x, ptr) \ - __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - #define __get_user(x, ptr) \ __get_user_nocheck((x), (ptr), sizeof(*(ptr))) #define __put_user(x, ptr) \ @@ -34,11 +31,12 @@ extern void __put_user_bad(void); #define __get_user_unaligned __get_user #define __put_user_unaligned __put_user -#define __put_user_check(x, ptr, size) \ +#define put_user(x, ptr) \ ({ \ int __pu_err; \ typeof(*(ptr)) __user *__pu_addr = (ptr); \ - switch (size) { \ + __chk_user_ptr(ptr); \ + switch (sizeof(*(ptr))) { \ case 1: \ __put_user_x(1, __pu_err, x, __pu_addr); \ break; \ From e30a44fdbf11c7ca3a0096d71790f176a4a09e03 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 13:17:43 -0300 Subject: [PATCH 40/43] x86: merge put_user. Move both versions, which are highly similar, to uaccess.h. Note that, for x86_64, X86_WP_WORKS_OK is always defined. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 74 +++++++++++++++++++++++++++++++++ include/asm-x86/uaccess_32.h | 79 ------------------------------------ include/asm-x86/uaccess_64.h | 36 ---------------- 3 files changed, 74 insertions(+), 115 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index 4353b2267a02..3a9092af5af0 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -178,6 +178,12 @@ extern int __get_user_bad(void); __ret_gu; \ }) +#define __put_user_x(size, x, ptr, __ret_pu) \ + asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ + :"0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") + + + #ifdef CONFIG_X86_32 #define __put_user_u64(x, addr, err) \ asm volatile("1: movl %%eax,0(%2)\n" \ @@ -191,13 +197,71 @@ extern int __get_user_bad(void); _ASM_EXTABLE(2b, 4b) \ : "=r" (err) \ : "A" (x), "r" (addr), "i" (-EFAULT), "0" (err)) + +#define __put_user_x8(x, ptr, __ret_pu) \ + asm volatile("call __put_user_8" : "=a" (__ret_pu) \ + : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") #else #define __put_user_u64(x, ptr, retval) \ __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT) +#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu) #endif +extern void __put_user_bad(void); + +/* + * Strange magic calling convention: pointer in %ecx, + * value in %eax(:%edx), return value in %eax. clobbers %rbx + */ +extern void __put_user_1(void); +extern void __put_user_2(void); +extern void __put_user_4(void); +extern void __put_user_8(void); + #ifdef CONFIG_X86_WP_WORKS_OK +/** + * put_user: - Write a simple value into user space. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Returns zero on success, or -EFAULT on error. + */ +#define put_user(x, ptr) \ +({ \ + int __ret_pu; \ + __typeof__(*(ptr)) __pu_val; \ + __chk_user_ptr(ptr); \ + __pu_val = x; \ + switch (sizeof(*(ptr))) { \ + case 1: \ + __put_user_x(1, __pu_val, ptr, __ret_pu); \ + break; \ + case 2: \ + __put_user_x(2, __pu_val, ptr, __ret_pu); \ + break; \ + case 4: \ + __put_user_x(4, __pu_val, ptr, __ret_pu); \ + break; \ + case 8: \ + __put_user_x8(__pu_val, ptr, __ret_pu); \ + break; \ + default: \ + __put_user_x(X, __pu_val, ptr, __ret_pu); \ + break; \ + } \ + __ret_pu; \ +}) + #define __put_user_size(x, ptr, size, retval, errret) \ do { \ retval = 0; \ @@ -231,6 +295,16 @@ do { \ retval = errret; \ } while (0) +#define put_user(x, ptr) \ +({ \ + int __ret_pu; \ + __typeof__(*(ptr))__pus_tmp = x; \ + __ret_pu = 0; \ + if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \ + sizeof(*(ptr))) != 0)) \ + __ret_pu = -EFAULT; \ + __ret_pu; \ +}) #endif #ifdef CONFIG_X86_32 diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 2c9067324d2c..e5c0437f509b 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -20,85 +20,6 @@ extern struct movsl_mask { } ____cacheline_aligned_in_smp movsl_mask; #endif -extern void __put_user_bad(void); - -/* - * Strange magic calling convention: pointer in %ecx, - * value in %eax(:%edx), return value in %eax, no clobbers. - */ -extern void __put_user_1(void); -extern void __put_user_2(void); -extern void __put_user_4(void); -extern void __put_user_8(void); - -#define __put_user_x(size, x, ptr, __ret_pu) \ - asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ - :"0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") - -#define __put_user_8(x, ptr, __ret_pu) \ - asm volatile("call __put_user_8" : "=a" (__ret_pu) \ - : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") - - -/** - * put_user: - Write a simple value into user space. - * @x: Value to copy to user space. - * @ptr: Destination address, in user space. - * - * Context: User context only. This function may sleep. - * - * This macro copies a single simple value from kernel space to user - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. - * - * @ptr must have pointer-to-simple-variable type, and @x must be assignable - * to the result of dereferencing @ptr. - * - * Returns zero on success, or -EFAULT on error. - */ -#ifdef CONFIG_X86_WP_WORKS_OK - -#define put_user(x, ptr) \ -({ \ - int __ret_pu; \ - __typeof__(*(ptr)) __pu_val; \ - __chk_user_ptr(ptr); \ - __pu_val = x; \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __put_user_x(1, __pu_val, ptr, __ret_pu); \ - break; \ - case 2: \ - __put_user_x(2, __pu_val, ptr, __ret_pu); \ - break; \ - case 4: \ - __put_user_x(4, __pu_val, ptr, __ret_pu); \ - break; \ - case 8: \ - __put_user_8(__pu_val, ptr, __ret_pu); \ - break; \ - default: \ - __put_user_x(X, __pu_val, ptr, __ret_pu); \ - break; \ - } \ - __ret_pu; \ -}) - -#else -#define put_user(x, ptr) \ -({ \ - int __ret_pu; \ - __typeof__(*(ptr))__pus_tmp = x; \ - __ret_pu = 0; \ - if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \ - sizeof(*(ptr))) != 0)) \ - __ret_pu = -EFAULT; \ - __ret_pu; \ -}) - - -#endif - /** * __get_user: - Get a simple variable from user space, with less checking. * @x: Variable to store result. diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 9139854d37b0..2e75a5d1a17a 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -11,18 +11,6 @@ #define ARCH_HAS_SEARCH_EXTABLE -extern void __put_user_1(void); -extern void __put_user_2(void); -extern void __put_user_4(void); -extern void __put_user_8(void); -extern void __put_user_bad(void); - -#define __put_user_x(size, ret, x, ptr) \ - asm volatile("call __put_user_" #size \ - :"=a" (ret) \ - :"c" (ptr),"a" (x) \ - :"ebx") - #define __get_user(x, ptr) \ __get_user_nocheck((x), (ptr), sizeof(*(ptr))) #define __put_user(x, ptr) \ @@ -31,30 +19,6 @@ extern void __put_user_bad(void); #define __get_user_unaligned __get_user #define __put_user_unaligned __put_user -#define put_user(x, ptr) \ -({ \ - int __pu_err; \ - typeof(*(ptr)) __user *__pu_addr = (ptr); \ - __chk_user_ptr(ptr); \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __put_user_x(1, __pu_err, x, __pu_addr); \ - break; \ - case 2: \ - __put_user_x(2, __pu_err, x, __pu_addr); \ - break; \ - case 4: \ - __put_user_x(4, __pu_err, x, __pu_addr); \ - break; \ - case 8: \ - __put_user_x(8, __pu_err, x, __pu_addr); \ - break; \ - default: \ - __put_user_bad(); \ - } \ - __pu_err; \ -}) - /* * Copy To/From Userspace */ From 8cb834e99f44bd56409b794504ae2b170675fc92 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 14:43:30 -0300 Subject: [PATCH 41/43] x86: move __get_user and __put_user into uaccess.h. We also carry the unaligned version with us. Only x86_64 uses it, but there's no problem in defining it. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 47 ++++++++++++++++++++++++++++++++++++ include/asm-x86/uaccess_32.h | 46 ----------------------------------- include/asm-x86/uaccess_64.h | 8 ------ 3 files changed, 47 insertions(+), 54 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index 3a9092af5af0..7c7b46af6368 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -383,7 +383,54 @@ struct __large_struct { unsigned long buf[100]; }; _ASM_EXTABLE(1b, 3b) \ : "=r"(err) \ : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) +/** + * __get_user: - Get a simple variable from user space, with less checking. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr))) +/** + * __put_user: - Write a simple value into user space, with less checking. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + */ + +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) + +#define __get_user_unaligned __get_user +#define __put_user_unaligned __put_user #ifdef CONFIG_X86_32 # include "uaccess_32.h" diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index e5c0437f509b..d3b5bf88ea86 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -20,52 +20,6 @@ extern struct movsl_mask { } ____cacheline_aligned_in_smp movsl_mask; #endif -/** - * __get_user: - Get a simple variable from user space, with less checking. - * @x: Variable to store result. - * @ptr: Source address, in user space. - * - * Context: User context only. This function may sleep. - * - * This macro copies a single simple variable from user space to kernel - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. - * - * @ptr must have pointer-to-simple-variable type, and the result of - * dereferencing @ptr must be assignable to @x without a cast. - * - * Caller must check the pointer with access_ok() before calling this - * function. - * - * Returns zero on success, or -EFAULT on error. - * On error, the variable @x is set to zero. - */ -#define __get_user(x, ptr) \ - __get_user_nocheck((x), (ptr), sizeof(*(ptr))) - - -/** - * __put_user: - Write a simple value into user space, with less checking. - * @x: Value to copy to user space. - * @ptr: Destination address, in user space. - * - * Context: User context only. This function may sleep. - * - * This macro copies a single simple value from kernel space to user - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. - * - * @ptr must have pointer-to-simple-variable type, and @x must be assignable - * to the result of dereferencing @ptr. - * - * Caller must check the pointer with access_ok() before calling this - * function. - * - * Returns zero on success, or -EFAULT on error. - */ -#define __put_user(x, ptr) \ - __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - unsigned long __must_check __copy_to_user_ll (void __user *to, const void *from, unsigned long n); unsigned long __must_check __copy_from_user_ll diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index 2e75a5d1a17a..b5bacd613686 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -11,14 +11,6 @@ #define ARCH_HAS_SEARCH_EXTABLE -#define __get_user(x, ptr) \ - __get_user_nocheck((x), (ptr), sizeof(*(ptr))) -#define __put_user(x, ptr) \ - __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - -#define __get_user_unaligned __get_user -#define __put_user_unaligned __put_user - /* * Copy To/From Userspace */ From 8bc7de0c5dc0a5d3bcdc04bac6de0c799f91c5e4 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 14:53:41 -0300 Subject: [PATCH 42/43] x86: put movsl_mask into uaccess.h. x86_64 does not need it, but it won't have X86_INTEL_USERCOPY defined either. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 9 +++++++++ include/asm-x86/uaccess_32.h | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index 7c7b46af6368..0c4ab788d017 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -432,6 +432,15 @@ struct __large_struct { unsigned long buf[100]; }; #define __get_user_unaligned __get_user #define __put_user_unaligned __put_user +/* + * movsl can be slow when source and dest are not both 8-byte aligned + */ +#ifdef CONFIG_X86_INTEL_USERCOPY +extern struct movsl_mask { + int mask; +} ____cacheline_aligned_in_smp movsl_mask; +#endif + #ifdef CONFIG_X86_32 # include "uaccess_32.h" #else diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index d3b5bf88ea86..3467749c6beb 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -11,15 +11,6 @@ #include #include -/* - * movsl can be slow when source and dest are not both 8-byte aligned - */ -#ifdef CONFIG_X86_INTEL_USERCOPY -extern struct movsl_mask { - int mask; -} ____cacheline_aligned_in_smp movsl_mask; -#endif - unsigned long __must_check __copy_to_user_ll (void __user *to, const void *from, unsigned long n); unsigned long __must_check __copy_from_user_ll From 22cac1670786108ccd4caa0656c39fa4ba69fa7d Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Wed, 25 Jun 2008 14:56:53 -0300 Subject: [PATCH 43/43] x86: define architectural characteristics in uaccess.h. Remove them from the arch-specific file. Signed-off-by: Glauber Costa Signed-off-by: H. Peter Anvin Signed-off-by: Ingo Molnar --- include/asm-x86/uaccess.h | 3 +++ include/asm-x86/uaccess_32.h | 2 -- include/asm-x86/uaccess_64.h | 3 --- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h index 0c4ab788d017..f6fa4d841bbc 100644 --- a/include/asm-x86/uaccess.h +++ b/include/asm-x86/uaccess.h @@ -441,9 +441,12 @@ extern struct movsl_mask { } ____cacheline_aligned_in_smp movsl_mask; #endif +#define ARCH_HAS_NOCACHE_UACCESS 1 + #ifdef CONFIG_X86_32 # include "uaccess_32.h" #else +# define ARCH_HAS_SEARCH_EXTABLE # include "uaccess_64.h" #endif diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h index 3467749c6beb..6fdef39a0bcb 100644 --- a/include/asm-x86/uaccess_32.h +++ b/include/asm-x86/uaccess_32.h @@ -156,8 +156,6 @@ __copy_from_user(void *to, const void __user *from, unsigned long n) return __copy_from_user_ll(to, from, n); } -#define ARCH_HAS_NOCACHE_UACCESS - static __always_inline unsigned long __copy_from_user_nocache(void *to, const void __user *from, unsigned long n) { diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h index b5bacd613686..4e3ec004e14a 100644 --- a/include/asm-x86/uaccess_64.h +++ b/include/asm-x86/uaccess_64.h @@ -9,8 +9,6 @@ #include #include -#define ARCH_HAS_SEARCH_EXTABLE - /* * Copy To/From Userspace */ @@ -180,7 +178,6 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) return copy_user_generic((__force void *)dst, src, size); } -#define ARCH_HAS_NOCACHE_UACCESS 1 extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest);