x86/asm: Move 'status' from thread_struct to thread_info

(cherry picked from commit 37a8f7c38339b22b69876d6f5a0ab851565284e3)

The TS_COMPAT bit is very hot and is accessed from code paths that mostly
also touch thread_info::flags.  Move it into struct thread_info to improve
cache locality.

The only reason it was in thread_struct is that there was a brief period
during which arch-specific fields were not allowed in struct thread_info.

Linus suggested further changing:

  ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

to:

  if (unlikely(ti->status & (TS_COMPAT|TS_I386_REGS_POKED)))
          ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);

on the theory that frequently dirtying the cacheline even in pure 64-bit
code that never needs to modify status hurts performance.  That could be a
reasonable followup patch, but I suspect it matters less on top of this
patch.

Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Kernel Hardening <kernel-hardening@lists.openwall.com>
Link: https://lkml.kernel.org/r/03148bcc1b217100e6e8ecf6a5468c45cf4304b6.1517164461.git.luto@kernel.org
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index bdd9cc5..bd1d102 100644 (file)
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -201,7 +201,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
         * special case only applies after poking regs and before the
         * very next return to user mode.
         */
-       current->thread.status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
+       ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
 #endif
 
        user_enter_irqoff();
@@ -299,7 +299,7 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
        unsigned int nr = (unsigned int)regs->orig_ax;
 
 #ifdef CONFIG_IA32_EMULATION
-       current->thread.status |= TS_COMPAT;
+       ti->status |= TS_COMPAT;
 #endif
 
        if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 353f038..cb866ae 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -391,8 +391,6 @@ struct thread_struct {
        unsigned short          gsindex;
 #endif
 
-       u32                     status;         /* thread synchronous flags */
-
 #ifdef CONFIG_X86_64
        unsigned long           fsbase;
        unsigned long           gsbase;

diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h
index e3c95e8..03eedc2 100644 (file)
--- a/arch/x86/include/asm/syscall.h
+++ b/arch/x86/include/asm/syscall.h
@@ -60,7 +60,7 @@ static inline long syscall_get_error(struct task_struct *task,
         * TS_COMPAT is set for 32-bit syscall entries and then
         * remains set until we return to user mode.
         */
-       if (task->thread.status & (TS_COMPAT|TS_I386_REGS_POKED))
+       if (task->thread_info.status & (TS_COMPAT|TS_I386_REGS_POKED))
                /*
                 * Sign-extend the value so (int)-EFOO becomes (long)-EFOO
                 * and will match correctly in comparisons.
@@ -116,7 +116,7 @@ static inline void syscall_get_arguments(struct task_struct *task,
                                         unsigned long *args)
 {
 # ifdef CONFIG_IA32_EMULATION
-       if (task->thread.status & TS_COMPAT)
+       if (task->thread_info.status & TS_COMPAT)
                switch (i) {
                case 0:
                        if (!n--) break;
@@ -177,7 +177,7 @@ static inline void syscall_set_arguments(struct task_struct *task,
                                         const unsigned long *args)
 {
 # ifdef CONFIG_IA32_EMULATION
-       if (task->thread.status & TS_COMPAT)
+       if (task->thread_info.status & TS_COMPAT)
                switch (i) {
                case 0:
                        if (!n--) break;

diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index bdf9c4c..89978b9 100644 (file)
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -54,6 +54,7 @@ struct task_struct;
 
 struct thread_info {
        unsigned long           flags;          /* low level flags */
+       u32                     status;         /* thread synchronous flags */
 };
 
 #define INIT_THREAD_INFO(tsk)                  \
@@ -213,7 +214,7 @@ static inline int arch_within_stack_frames(const void * const stack,
 #define in_ia32_syscall() true
 #else
 #define in_ia32_syscall() (IS_ENABLED(CONFIG_IA32_EMULATION) && \
-                          current->thread.status & TS_COMPAT)
+                          current_thread_info()->status & TS_COMPAT)
 #endif
 
 /*

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 0887d2a..dffe81d 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -538,7 +538,7 @@ void set_personality_ia32(bool x32)
                current->personality &= ~READ_IMPLIES_EXEC;
                /* in_compat_syscall() uses the presence of the x32
                   syscall bit flag to determine compat status */
-               current->thread.status &= ~TS_COMPAT;
+               current_thread_info()->status &= ~TS_COMPAT;
        } else {
                set_thread_flag(TIF_IA32);
                clear_thread_flag(TIF_X32);
@@ -546,7 +546,7 @@ void set_personality_ia32(bool x32)
                        current->mm->context.ia32_compat = TIF_IA32;
                current->personality |= force_personality32;
                /* Prepare the first "return" to user space */
-               current->thread.status |= TS_COMPAT;
+               current_thread_info()->status |= TS_COMPAT;
        }
 }
 EXPORT_SYMBOL_GPL(set_personality_ia32);

diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index 0e63c02..e497d37 100644 (file)
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -934,7 +934,7 @@ static int putreg32(struct task_struct *child, unsigned regno, u32 value)
                 */
                regs->orig_ax = value;
                if (syscall_get_nr(child, regs) >= 0)
-                       child->thread.status |= TS_I386_REGS_POKED;
+                       child->thread_info.status |= TS_I386_REGS_POKED;
                break;
 
        case offsetof(struct user32, regs.eflags):

diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index 763af1d..b1a5d25 100644 (file)
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -785,7 +785,7 @@ static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs)
         * than the tracee.
         */
 #ifdef CONFIG_IA32_EMULATION
-       if (current->thread.status & (TS_COMPAT|TS_I386_REGS_POKED))
+       if (current_thread_info()->status & (TS_COMPAT|TS_I386_REGS_POKED))
                return __NR_ia32_restart_syscall;
 #endif
 #ifdef CONFIG_X86_X32_ABI