1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shadow Call Stack support. 4 * 5 * Copyright (C) 2019 Google LLC 6 */ 7 8 #include <linux/kasan.h> 9 #include <linux/mm.h> 10 #include <linux/scs.h> 11 #include <linux/slab.h> 12 #include <linux/vmstat.h> 13 #include <asm/scs.h> 14 15 static struct kmem_cache *scs_cache; 16 17 static void __scs_account(void *s, int account) 18 { 19 struct page *scs_page = virt_to_page(s); 20 21 mod_zone_page_state(page_zone(scs_page), NR_KERNEL_SCS_KB, 22 account * (SCS_SIZE / SZ_1K)); 23 } 24 25 static void *scs_alloc(int node) 26 { 27 void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node); 28 29 if (!s) 30 return NULL; 31 32 *__scs_magic(s) = SCS_END_MAGIC; 33 34 /* 35 * Poison the allocation to catch unintentional accesses to 36 * the shadow stack when KASAN is enabled. 37 */ 38 kasan_poison_object_data(scs_cache, s); 39 __scs_account(s, 1); 40 return s; 41 } 42 43 static void scs_free(void *s) 44 { 45 __scs_account(s, -1); 46 kasan_unpoison_object_data(scs_cache, s); 47 kmem_cache_free(scs_cache, s); 48 } 49 50 void __init scs_init(void) 51 { 52 scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL); 53 } 54 55 int scs_prepare(struct task_struct *tsk, int node) 56 { 57 void *s = scs_alloc(node); 58 59 if (!s) 60 return -ENOMEM; 61 62 task_scs(tsk) = task_scs_sp(tsk) = s; 63 return 0; 64 } 65 66 static void scs_check_usage(struct task_struct *tsk) 67 { 68 static unsigned long highest; 69 70 unsigned long *p, prev, curr = highest, used = 0; 71 72 if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE)) 73 return; 74 75 for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { 76 if (!READ_ONCE_NOCHECK(*p)) 77 break; 78 used++; 79 } 80 81 while (used > curr) { 82 prev = cmpxchg_relaxed(&highest, curr, used); 83 84 if (prev == curr) { 85 pr_info("%s (%d): highest shadow stack usage: %lu bytes\n", 86 tsk->comm, task_pid_nr(tsk), used); 87 break; 88 } 89 90 curr = prev; 91 } 92 } 93 94 void scs_release(struct task_struct *tsk) 95 { 96 void *s = task_scs(tsk); 97 98 if (!s) 99 return; 100 101 WARN(scs_corrupted(tsk), "corrupted shadow stack detected when freeing task\n"); 102 scs_check_usage(tsk); 103 scs_free(s); 104 } 105