1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * c 2001 PPC 64 Team, IBM Corp 4 */ 5 6 #include <linux/smp.h> 7 #include <linux/export.h> 8 #include <linux/memblock.h> 9 #include <linux/sched/task.h> 10 #include <linux/numa.h> 11 12 #include <asm/lppaca.h> 13 #include <asm/paca.h> 14 #include <asm/sections.h> 15 #include <asm/pgtable.h> 16 #include <asm/kexec.h> 17 18 #include "setup.h" 19 20 #ifndef CONFIG_SMP 21 #define boot_cpuid 0 22 #endif 23 24 static void *__init alloc_paca_data(unsigned long size, unsigned long align, 25 unsigned long limit, int cpu) 26 { 27 void *ptr; 28 int nid; 29 30 /* 31 * boot_cpuid paca is allocated very early before cpu_to_node is up. 32 * Set bottom-up mode, because the boot CPU should be on node-0, 33 * which will put its paca in the right place. 34 */ 35 if (cpu == boot_cpuid) { 36 nid = NUMA_NO_NODE; 37 memblock_set_bottom_up(true); 38 } else { 39 nid = early_cpu_to_node(cpu); 40 } 41 42 ptr = memblock_alloc_try_nid(size, align, MEMBLOCK_LOW_LIMIT, 43 limit, nid); 44 if (!ptr) 45 panic("cannot allocate paca data"); 46 47 if (cpu == boot_cpuid) 48 memblock_set_bottom_up(false); 49 50 return ptr; 51 } 52 53 #ifdef CONFIG_PPC_PSERIES 54 55 #define LPPACA_SIZE 0x400 56 57 /* 58 * See asm/lppaca.h for more detail. 59 * 60 * lppaca structures must must be 1kB in size, L1 cache line aligned, 61 * and not cross 4kB boundary. A 1kB size and 1kB alignment will satisfy 62 * these requirements. 63 */ 64 static inline void init_lppaca(struct lppaca *lppaca) 65 { 66 BUILD_BUG_ON(sizeof(struct lppaca) != 640); 67 68 *lppaca = (struct lppaca) { 69 .desc = cpu_to_be32(0xd397d781), /* "LpPa" */ 70 .size = cpu_to_be16(LPPACA_SIZE), 71 .fpregs_in_use = 1, 72 .slb_count = cpu_to_be16(64), 73 .vmxregs_in_use = 0, 74 .page_ins = 0, }; 75 }; 76 77 static struct lppaca * __init new_lppaca(int cpu, unsigned long limit) 78 { 79 struct lppaca *lp; 80 81 BUILD_BUG_ON(sizeof(struct lppaca) > LPPACA_SIZE); 82 83 if (early_cpu_has_feature(CPU_FTR_HVMODE)) 84 return NULL; 85 86 lp = alloc_paca_data(LPPACA_SIZE, 0x400, limit, cpu); 87 init_lppaca(lp); 88 89 return lp; 90 } 91 #endif /* CONFIG_PPC_PSERIES */ 92 93 #ifdef CONFIG_PPC_BOOK3S_64 94 95 /* 96 * 3 persistent SLBs are allocated here. The buffer will be zero 97 * initially, hence will all be invaild until we actually write them. 98 * 99 * If you make the number of persistent SLB entries dynamic, please also 100 * update PR KVM to flush and restore them accordingly. 101 */ 102 static struct slb_shadow * __init new_slb_shadow(int cpu, unsigned long limit) 103 { 104 struct slb_shadow *s; 105 106 if (cpu != boot_cpuid) { 107 /* 108 * Boot CPU comes here before early_radix_enabled 109 * is parsed (e.g., for disable_radix). So allocate 110 * always and this will be fixed up in free_unused_pacas. 111 */ 112 if (early_radix_enabled()) 113 return NULL; 114 } 115 116 s = alloc_paca_data(sizeof(*s), L1_CACHE_BYTES, limit, cpu); 117 118 s->persistent = cpu_to_be32(SLB_NUM_BOLTED); 119 s->buffer_length = cpu_to_be32(sizeof(*s)); 120 121 return s; 122 } 123 124 #endif /* CONFIG_PPC_BOOK3S_64 */ 125 126 /* The Paca is an array with one entry per processor. Each contains an 127 * lppaca, which contains the information shared between the 128 * hypervisor and Linux. 129 * On systems with hardware multi-threading, there are two threads 130 * per processor. The Paca array must contain an entry for each thread. 131 * The VPD Areas will give a max logical processors = 2 * max physical 132 * processors. The processor VPD array needs one entry per physical 133 * processor (not thread). 134 */ 135 struct paca_struct **paca_ptrs __read_mostly; 136 EXPORT_SYMBOL(paca_ptrs); 137 138 void __init initialise_paca(struct paca_struct *new_paca, int cpu) 139 { 140 #ifdef CONFIG_PPC_PSERIES 141 new_paca->lppaca_ptr = NULL; 142 #endif 143 #ifdef CONFIG_PPC_BOOK3E 144 new_paca->kernel_pgd = swapper_pg_dir; 145 #endif 146 new_paca->lock_token = 0x8000; 147 new_paca->paca_index = cpu; 148 new_paca->kernel_toc = kernel_toc_addr(); 149 new_paca->kernelbase = (unsigned long) _stext; 150 /* Only set MSR:IR/DR when MMU is initialized */ 151 new_paca->kernel_msr = MSR_KERNEL & ~(MSR_IR | MSR_DR); 152 new_paca->hw_cpu_id = 0xffff; 153 new_paca->kexec_state = KEXEC_STATE_NONE; 154 new_paca->__current = &init_task; 155 new_paca->data_offset = 0xfeeeeeeeeeeeeeeeULL; 156 #ifdef CONFIG_PPC_BOOK3S_64 157 new_paca->slb_shadow_ptr = NULL; 158 #endif 159 160 #ifdef CONFIG_PPC_BOOK3E 161 /* For now -- if we have threads this will be adjusted later */ 162 new_paca->tcd_ptr = &new_paca->tcd; 163 #endif 164 } 165 166 /* Put the paca pointer into r13 and SPRG_PACA */ 167 void setup_paca(struct paca_struct *new_paca) 168 { 169 /* Setup r13 */ 170 local_paca = new_paca; 171 172 #ifdef CONFIG_PPC_BOOK3E 173 /* On Book3E, initialize the TLB miss exception frames */ 174 mtspr(SPRN_SPRG_TLB_EXFRAME, local_paca->extlb); 175 #else 176 /* In HV mode, we setup both HPACA and PACA to avoid problems 177 * if we do a GET_PACA() before the feature fixups have been 178 * applied 179 */ 180 if (early_cpu_has_feature(CPU_FTR_HVMODE)) 181 mtspr(SPRN_SPRG_HPACA, local_paca); 182 #endif 183 mtspr(SPRN_SPRG_PACA, local_paca); 184 185 } 186 187 static int __initdata paca_nr_cpu_ids; 188 static int __initdata paca_ptrs_size; 189 static int __initdata paca_struct_size; 190 191 void __init allocate_paca_ptrs(void) 192 { 193 paca_nr_cpu_ids = nr_cpu_ids; 194 195 paca_ptrs_size = sizeof(struct paca_struct *) * nr_cpu_ids; 196 paca_ptrs = memblock_alloc_raw(paca_ptrs_size, SMP_CACHE_BYTES); 197 if (!paca_ptrs) 198 panic("Failed to allocate %d bytes for paca pointers\n", 199 paca_ptrs_size); 200 201 memset(paca_ptrs, 0x88, paca_ptrs_size); 202 } 203 204 void __init allocate_paca(int cpu) 205 { 206 u64 limit; 207 struct paca_struct *paca; 208 209 BUG_ON(cpu >= paca_nr_cpu_ids); 210 211 #ifdef CONFIG_PPC_BOOK3S_64 212 /* 213 * We access pacas in real mode, and cannot take SLB faults 214 * on them when in virtual mode, so allocate them accordingly. 215 */ 216 limit = min(ppc64_bolted_size(), ppc64_rma_size); 217 #else 218 limit = ppc64_rma_size; 219 #endif 220 221 paca = alloc_paca_data(sizeof(struct paca_struct), L1_CACHE_BYTES, 222 limit, cpu); 223 paca_ptrs[cpu] = paca; 224 225 initialise_paca(paca, cpu); 226 #ifdef CONFIG_PPC_PSERIES 227 paca->lppaca_ptr = new_lppaca(cpu, limit); 228 #endif 229 #ifdef CONFIG_PPC_BOOK3S_64 230 paca->slb_shadow_ptr = new_slb_shadow(cpu, limit); 231 #endif 232 paca_struct_size += sizeof(struct paca_struct); 233 } 234 235 void __init free_unused_pacas(void) 236 { 237 int new_ptrs_size; 238 239 new_ptrs_size = sizeof(struct paca_struct *) * nr_cpu_ids; 240 if (new_ptrs_size < paca_ptrs_size) 241 memblock_free(__pa(paca_ptrs) + new_ptrs_size, 242 paca_ptrs_size - new_ptrs_size); 243 244 paca_nr_cpu_ids = nr_cpu_ids; 245 paca_ptrs_size = new_ptrs_size; 246 247 #ifdef CONFIG_PPC_BOOK3S_64 248 if (early_radix_enabled()) { 249 /* Ugly fixup, see new_slb_shadow() */ 250 memblock_free(__pa(paca_ptrs[boot_cpuid]->slb_shadow_ptr), 251 sizeof(struct slb_shadow)); 252 paca_ptrs[boot_cpuid]->slb_shadow_ptr = NULL; 253 } 254 #endif 255 256 printk(KERN_DEBUG "Allocated %u bytes for %u pacas\n", 257 paca_ptrs_size + paca_struct_size, nr_cpu_ids); 258 } 259 260 void copy_mm_to_paca(struct mm_struct *mm) 261 { 262 #ifdef CONFIG_PPC_BOOK3S 263 mm_context_t *context = &mm->context; 264 265 get_paca()->mm_ctx_id = context->id; 266 #ifdef CONFIG_PPC_MM_SLICES 267 VM_BUG_ON(!mm_ctx_slb_addr_limit(context)); 268 get_paca()->mm_ctx_slb_addr_limit = mm_ctx_slb_addr_limit(context); 269 memcpy(&get_paca()->mm_ctx_low_slices_psize, mm_ctx_low_slices(context), 270 LOW_SLICE_ARRAY_SZ); 271 memcpy(&get_paca()->mm_ctx_high_slices_psize, mm_ctx_high_slices(context), 272 TASK_SLICE_ARRAY_SZ(context)); 273 #else /* CONFIG_PPC_MM_SLICES */ 274 get_paca()->mm_ctx_user_psize = context->user_psize; 275 get_paca()->mm_ctx_sllp = context->sllp; 276 #endif 277 #else /* !CONFIG_PPC_BOOK3S */ 278 return; 279 #endif 280 } 281