1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_CONTEXT_TRACKING_H 3 #define _LINUX_CONTEXT_TRACKING_H 4 5 #include <linux/sched.h> 6 #include <linux/vtime.h> 7 #include <linux/context_tracking_state.h> 8 #include <asm/ptrace.h> 9 10 11 #ifdef CONFIG_CONTEXT_TRACKING 12 extern void context_tracking_cpu_set(int cpu); 13 14 /* Called with interrupts disabled. */ 15 extern void __context_tracking_enter(enum ctx_state state); 16 extern void __context_tracking_exit(enum ctx_state state); 17 18 extern void context_tracking_enter(enum ctx_state state); 19 extern void context_tracking_exit(enum ctx_state state); 20 extern void context_tracking_user_enter(void); 21 extern void context_tracking_user_exit(void); 22 23 static inline void user_enter(void) 24 { 25 if (context_tracking_enabled()) 26 context_tracking_enter(CONTEXT_USER); 27 28 } 29 static inline void user_exit(void) 30 { 31 if (context_tracking_enabled()) 32 context_tracking_exit(CONTEXT_USER); 33 } 34 35 /* Called with interrupts disabled. */ 36 static __always_inline void user_enter_irqoff(void) 37 { 38 if (context_tracking_enabled()) 39 __context_tracking_enter(CONTEXT_USER); 40 41 } 42 static __always_inline void user_exit_irqoff(void) 43 { 44 if (context_tracking_enabled()) 45 __context_tracking_exit(CONTEXT_USER); 46 } 47 48 static inline enum ctx_state exception_enter(void) 49 { 50 enum ctx_state prev_ctx; 51 52 if (!context_tracking_enabled()) 53 return 0; 54 55 prev_ctx = this_cpu_read(context_tracking.state); 56 if (prev_ctx != CONTEXT_KERNEL) 57 context_tracking_exit(prev_ctx); 58 59 return prev_ctx; 60 } 61 62 static inline void exception_exit(enum ctx_state prev_ctx) 63 { 64 if (context_tracking_enabled()) { 65 if (prev_ctx != CONTEXT_KERNEL) 66 context_tracking_enter(prev_ctx); 67 } 68 } 69 70 71 /** 72 * ct_state() - return the current context tracking state if known 73 * 74 * Returns the current cpu's context tracking state if context tracking 75 * is enabled. If context tracking is disabled, returns 76 * CONTEXT_DISABLED. This should be used primarily for debugging. 77 */ 78 static __always_inline enum ctx_state ct_state(void) 79 { 80 return context_tracking_enabled() ? 81 this_cpu_read(context_tracking.state) : CONTEXT_DISABLED; 82 } 83 #else 84 static inline void user_enter(void) { } 85 static inline void user_exit(void) { } 86 static inline void user_enter_irqoff(void) { } 87 static inline void user_exit_irqoff(void) { } 88 static inline enum ctx_state exception_enter(void) { return 0; } 89 static inline void exception_exit(enum ctx_state prev_ctx) { } 90 static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; } 91 #endif /* !CONFIG_CONTEXT_TRACKING */ 92 93 #define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) 94 95 #ifdef CONFIG_CONTEXT_TRACKING_FORCE 96 extern void context_tracking_init(void); 97 #else 98 static inline void context_tracking_init(void) { } 99 #endif /* CONFIG_CONTEXT_TRACKING_FORCE */ 100 101 102 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 103 /* must be called with irqs disabled */ 104 static __always_inline void guest_enter_irqoff(void) 105 { 106 instrumentation_begin(); 107 if (vtime_accounting_enabled_this_cpu()) 108 vtime_guest_enter(current); 109 else 110 current->flags |= PF_VCPU; 111 instrumentation_end(); 112 113 if (context_tracking_enabled()) 114 __context_tracking_enter(CONTEXT_GUEST); 115 116 /* KVM does not hold any references to rcu protected data when it 117 * switches CPU into a guest mode. In fact switching to a guest mode 118 * is very similar to exiting to userspace from rcu point of view. In 119 * addition CPU may stay in a guest mode for quite a long time (up to 120 * one time slice). Lets treat guest mode as quiescent state, just like 121 * we do with user-mode execution. 122 */ 123 if (!context_tracking_enabled_this_cpu()) { 124 instrumentation_begin(); 125 rcu_virt_note_context_switch(smp_processor_id()); 126 instrumentation_end(); 127 } 128 } 129 130 static __always_inline void guest_exit_irqoff(void) 131 { 132 if (context_tracking_enabled()) 133 __context_tracking_exit(CONTEXT_GUEST); 134 135 instrumentation_begin(); 136 if (vtime_accounting_enabled_this_cpu()) 137 vtime_guest_exit(current); 138 else 139 current->flags &= ~PF_VCPU; 140 instrumentation_end(); 141 } 142 143 #else 144 static __always_inline void guest_enter_irqoff(void) 145 { 146 /* 147 * This is running in ioctl context so its safe 148 * to assume that it's the stime pending cputime 149 * to flush. 150 */ 151 instrumentation_begin(); 152 vtime_account_kernel(current); 153 current->flags |= PF_VCPU; 154 rcu_virt_note_context_switch(smp_processor_id()); 155 instrumentation_end(); 156 } 157 158 static __always_inline void guest_exit_irqoff(void) 159 { 160 instrumentation_begin(); 161 /* Flush the guest cputime we spent on the guest */ 162 vtime_account_kernel(current); 163 current->flags &= ~PF_VCPU; 164 instrumentation_end(); 165 } 166 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */ 167 168 static inline void guest_exit(void) 169 { 170 unsigned long flags; 171 172 local_irq_save(flags); 173 guest_exit_irqoff(); 174 local_irq_restore(flags); 175 } 176 177 #endif 178