xref: /linux-6.15/arch/arm64/kernel/vdso.c (revision ee3cda8e) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VDSO implementations.
4  *
5  * Copyright (C) 2012 ARM Limited
6  *
7  * Author: Will Deacon <[email protected]>
8  */
9 
10 #include <linux/cache.h>
11 #include <linux/clocksource.h>
12 #include <linux/elf.h>
13 #include <linux/err.h>
14 #include <linux/errno.h>
15 #include <linux/gfp.h>
16 #include <linux/kernel.h>
17 #include <linux/mm.h>
18 #include <linux/sched.h>
19 #include <linux/signal.h>
20 #include <linux/slab.h>
21 #include <linux/time_namespace.h>
22 #include <linux/timekeeper_internal.h>
23 #include <linux/vmalloc.h>
24 #include <vdso/datapage.h>
25 #include <vdso/helpers.h>
26 #include <vdso/vsyscall.h>
27 
28 #include <asm/cacheflush.h>
29 #include <asm/signal32.h>
30 #include <asm/vdso.h>
31 
32 extern char vdso_start[], vdso_end[];
33 #ifdef CONFIG_COMPAT_VDSO
34 extern char vdso32_start[], vdso32_end[];
35 #endif /* CONFIG_COMPAT_VDSO */
36 
37 enum vdso_abi {
38 	VDSO_ABI_AA64,
39 #ifdef CONFIG_COMPAT_VDSO
40 	VDSO_ABI_AA32,
41 #endif /* CONFIG_COMPAT_VDSO */
42 };
43 
44 enum vvar_pages {
45 	VVAR_DATA_PAGE_OFFSET,
46 	VVAR_TIMENS_PAGE_OFFSET,
47 	VVAR_NR_PAGES,
48 };
49 
50 struct vdso_abi_info {
51 	const char *name;
52 	const char *vdso_code_start;
53 	const char *vdso_code_end;
54 	unsigned long vdso_pages;
55 	/* Data Mapping */
56 	struct vm_special_mapping *dm;
57 	/* Code Mapping */
58 	struct vm_special_mapping *cm;
59 };
60 
61 static struct vdso_abi_info vdso_info[] __ro_after_init = {
62 	[VDSO_ABI_AA64] = {
63 		.name = "vdso",
64 		.vdso_code_start = vdso_start,
65 		.vdso_code_end = vdso_end,
66 	},
67 #ifdef CONFIG_COMPAT_VDSO
68 	[VDSO_ABI_AA32] = {
69 		.name = "vdso32",
70 		.vdso_code_start = vdso32_start,
71 		.vdso_code_end = vdso32_end,
72 	},
73 #endif /* CONFIG_COMPAT_VDSO */
74 };
75 
76 /*
77  * The vDSO data page.
78  */
79 static union {
80 	struct vdso_data	data[CS_BASES];
81 	u8			page[PAGE_SIZE];
82 } vdso_data_store __page_aligned_data;
83 struct vdso_data *vdso_data = vdso_data_store.data;
84 
85 static int __vdso_remap(enum vdso_abi abi,
86 			const struct vm_special_mapping *sm,
87 			struct vm_area_struct *new_vma)
88 {
89 	unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
90 	unsigned long vdso_size = vdso_info[abi].vdso_code_end -
91 				  vdso_info[abi].vdso_code_start;
92 
93 	if (vdso_size != new_size)
94 		return -EINVAL;
95 
96 	current->mm->context.vdso = (void *)new_vma->vm_start;
97 
98 	return 0;
99 }
100 
101 static int __vdso_init(enum vdso_abi abi)
102 {
103 	int i;
104 	struct page **vdso_pagelist;
105 	unsigned long pfn;
106 
107 	if (memcmp(vdso_info[abi].vdso_code_start, "\177ELF", 4)) {
108 		pr_err("vDSO is not a valid ELF object!\n");
109 		return -EINVAL;
110 	}
111 
112 	vdso_info[abi].vdso_pages = (
113 			vdso_info[abi].vdso_code_end -
114 			vdso_info[abi].vdso_code_start) >>
115 			PAGE_SHIFT;
116 
117 	vdso_pagelist = kcalloc(vdso_info[abi].vdso_pages,
118 				sizeof(struct page *),
119 				GFP_KERNEL);
120 	if (vdso_pagelist == NULL)
121 		return -ENOMEM;
122 
123 	/* Grab the vDSO code pages. */
124 	pfn = sym_to_pfn(vdso_info[abi].vdso_code_start);
125 
126 	for (i = 0; i < vdso_info[abi].vdso_pages; i++)
127 		vdso_pagelist[i] = pfn_to_page(pfn + i);
128 
129 	vdso_info[abi].cm->pages = vdso_pagelist;
130 
131 	return 0;
132 }
133 
134 #ifdef CONFIG_TIME_NS
135 struct vdso_data *arch_get_vdso_data(void *vvar_page)
136 {
137 	return (struct vdso_data *)(vvar_page);
138 }
139 
140 /*
141  * The vvar mapping contains data for a specific time namespace, so when a task
142  * changes namespace we must unmap its vvar data for the old namespace.
143  * Subsequent faults will map in data for the new namespace.
144  *
145  * For more details see timens_setup_vdso_data().
146  */
147 int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
148 {
149 	struct mm_struct *mm = task->mm;
150 	struct vm_area_struct *vma;
151 
152 	mmap_read_lock(mm);
153 
154 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
155 		unsigned long size = vma->vm_end - vma->vm_start;
156 
157 		if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm))
158 			zap_page_range(vma, vma->vm_start, size);
159 #ifdef CONFIG_COMPAT_VDSO
160 		if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA32].dm))
161 			zap_page_range(vma, vma->vm_start, size);
162 #endif
163 	}
164 
165 	mmap_read_unlock(mm);
166 	return 0;
167 }
168 
169 static struct page *find_timens_vvar_page(struct vm_area_struct *vma)
170 {
171 	if (likely(vma->vm_mm == current->mm))
172 		return current->nsproxy->time_ns->vvar_page;
173 
174 	/*
175 	 * VM_PFNMAP | VM_IO protect .fault() handler from being called
176 	 * through interfaces like /proc/$pid/mem or
177 	 * process_vm_{readv,writev}() as long as there's no .access()
178 	 * in special_mapping_vmops.
179 	 * For more details check_vma_flags() and __access_remote_vm()
180 	 */
181 	WARN(1, "vvar_page accessed remotely");
182 
183 	return NULL;
184 }
185 #else
186 static struct page *find_timens_vvar_page(struct vm_area_struct *vma)
187 {
188 	return NULL;
189 }
190 #endif
191 
192 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
193 			     struct vm_area_struct *vma, struct vm_fault *vmf)
194 {
195 	struct page *timens_page = find_timens_vvar_page(vma);
196 	unsigned long pfn;
197 
198 	switch (vmf->pgoff) {
199 	case VVAR_DATA_PAGE_OFFSET:
200 		if (timens_page)
201 			pfn = page_to_pfn(timens_page);
202 		else
203 			pfn = sym_to_pfn(vdso_data);
204 		break;
205 #ifdef CONFIG_TIME_NS
206 	case VVAR_TIMENS_PAGE_OFFSET:
207 		/*
208 		 * If a task belongs to a time namespace then a namespace
209 		 * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
210 		 * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
211 		 * offset.
212 		 * See also the comment near timens_setup_vdso_data().
213 		 */
214 		if (!timens_page)
215 			return VM_FAULT_SIGBUS;
216 		pfn = sym_to_pfn(vdso_data);
217 		break;
218 #endif /* CONFIG_TIME_NS */
219 	default:
220 		return VM_FAULT_SIGBUS;
221 	}
222 
223 	return vmf_insert_pfn(vma, vmf->address, pfn);
224 }
225 
226 static int __setup_additional_pages(enum vdso_abi abi,
227 				    struct mm_struct *mm,
228 				    struct linux_binprm *bprm,
229 				    int uses_interp)
230 {
231 	unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
232 	unsigned long gp_flags = 0;
233 	void *ret;
234 
235 	BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES);
236 
237 	vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
238 	/* Be sure to map the data page */
239 	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE;
240 
241 	vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
242 	if (IS_ERR_VALUE(vdso_base)) {
243 		ret = ERR_PTR(vdso_base);
244 		goto up_fail;
245 	}
246 
247 	ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
248 				       VM_READ|VM_MAYREAD|VM_PFNMAP,
249 				       vdso_info[abi].dm);
250 	if (IS_ERR(ret))
251 		goto up_fail;
252 
253 	if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && system_supports_bti())
254 		gp_flags = VM_ARM64_BTI;
255 
256 	vdso_base += VVAR_NR_PAGES * PAGE_SIZE;
257 	mm->context.vdso = (void *)vdso_base;
258 	ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
259 				       VM_READ|VM_EXEC|gp_flags|
260 				       VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
261 				       vdso_info[abi].cm);
262 	if (IS_ERR(ret))
263 		goto up_fail;
264 
265 	return 0;
266 
267 up_fail:
268 	mm->context.vdso = NULL;
269 	return PTR_ERR(ret);
270 }
271 
272 #ifdef CONFIG_COMPAT
273 /*
274  * Create and map the vectors page for AArch32 tasks.
275  */
276 #ifdef CONFIG_COMPAT_VDSO
277 static int aarch32_vdso_mremap(const struct vm_special_mapping *sm,
278 		struct vm_area_struct *new_vma)
279 {
280 	return __vdso_remap(VDSO_ABI_AA32, sm, new_vma);
281 }
282 #endif /* CONFIG_COMPAT_VDSO */
283 
284 enum aarch32_map {
285 	AA32_MAP_VECTORS, /* kuser helpers */
286 #ifdef CONFIG_COMPAT_VDSO
287 	AA32_MAP_VVAR,
288 	AA32_MAP_VDSO,
289 #endif
290 	AA32_MAP_SIGPAGE
291 };
292 
293 static struct page *aarch32_vectors_page __ro_after_init;
294 static struct page *aarch32_sig_page __ro_after_init;
295 
296 static struct vm_special_mapping aarch32_vdso_maps[] = {
297 	[AA32_MAP_VECTORS] = {
298 		.name	= "[vectors]", /* ABI */
299 		.pages	= &aarch32_vectors_page,
300 	},
301 #ifdef CONFIG_COMPAT_VDSO
302 	[AA32_MAP_VVAR] = {
303 		.name = "[vvar]",
304 		.fault = vvar_fault,
305 	},
306 	[AA32_MAP_VDSO] = {
307 		.name = "[vdso]",
308 		.mremap = aarch32_vdso_mremap,
309 	},
310 #endif /* CONFIG_COMPAT_VDSO */
311 	[AA32_MAP_SIGPAGE] = {
312 		.name	= "[sigpage]", /* ABI */
313 		.pages	= &aarch32_sig_page,
314 	},
315 };
316 
317 static int aarch32_alloc_kuser_vdso_page(void)
318 {
319 	extern char __kuser_helper_start[], __kuser_helper_end[];
320 	int kuser_sz = __kuser_helper_end - __kuser_helper_start;
321 	unsigned long vdso_page;
322 
323 	if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
324 		return 0;
325 
326 	vdso_page = get_zeroed_page(GFP_ATOMIC);
327 	if (!vdso_page)
328 		return -ENOMEM;
329 
330 	memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start,
331 	       kuser_sz);
332 	aarch32_vectors_page = virt_to_page(vdso_page);
333 	flush_dcache_page(aarch32_vectors_page);
334 	return 0;
335 }
336 
337 static int aarch32_alloc_sigpage(void)
338 {
339 	extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
340 	int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;
341 	unsigned long sigpage;
342 
343 	sigpage = get_zeroed_page(GFP_ATOMIC);
344 	if (!sigpage)
345 		return -ENOMEM;
346 
347 	memcpy((void *)sigpage, __aarch32_sigret_code_start, sigret_sz);
348 	aarch32_sig_page = virt_to_page(sigpage);
349 	flush_dcache_page(aarch32_sig_page);
350 	return 0;
351 }
352 
353 #ifdef CONFIG_COMPAT_VDSO
354 static int __aarch32_alloc_vdso_pages(void)
355 {
356 	vdso_info[VDSO_ABI_AA32].dm = &aarch32_vdso_maps[AA32_MAP_VVAR];
357 	vdso_info[VDSO_ABI_AA32].cm = &aarch32_vdso_maps[AA32_MAP_VDSO];
358 
359 	return __vdso_init(VDSO_ABI_AA32);
360 }
361 #endif /* CONFIG_COMPAT_VDSO */
362 
363 static int __init aarch32_alloc_vdso_pages(void)
364 {
365 	int ret;
366 
367 #ifdef CONFIG_COMPAT_VDSO
368 	ret = __aarch32_alloc_vdso_pages();
369 	if (ret)
370 		return ret;
371 #endif
372 
373 	ret = aarch32_alloc_sigpage();
374 	if (ret)
375 		return ret;
376 
377 	return aarch32_alloc_kuser_vdso_page();
378 }
379 arch_initcall(aarch32_alloc_vdso_pages);
380 
381 static int aarch32_kuser_helpers_setup(struct mm_struct *mm)
382 {
383 	void *ret;
384 
385 	if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
386 		return 0;
387 
388 	/*
389 	 * Avoid VM_MAYWRITE for compatibility with arch/arm/, where it's
390 	 * not safe to CoW the page containing the CPU exception vectors.
391 	 */
392 	ret = _install_special_mapping(mm, AARCH32_VECTORS_BASE, PAGE_SIZE,
393 				       VM_READ | VM_EXEC |
394 				       VM_MAYREAD | VM_MAYEXEC,
395 				       &aarch32_vdso_maps[AA32_MAP_VECTORS]);
396 
397 	return PTR_ERR_OR_ZERO(ret);
398 }
399 
400 static int aarch32_sigreturn_setup(struct mm_struct *mm)
401 {
402 	unsigned long addr;
403 	void *ret;
404 
405 	addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);
406 	if (IS_ERR_VALUE(addr)) {
407 		ret = ERR_PTR(addr);
408 		goto out;
409 	}
410 
411 	/*
412 	 * VM_MAYWRITE is required to allow gdb to Copy-on-Write and
413 	 * set breakpoints.
414 	 */
415 	ret = _install_special_mapping(mm, addr, PAGE_SIZE,
416 				       VM_READ | VM_EXEC | VM_MAYREAD |
417 				       VM_MAYWRITE | VM_MAYEXEC,
418 				       &aarch32_vdso_maps[AA32_MAP_SIGPAGE]);
419 	if (IS_ERR(ret))
420 		goto out;
421 
422 	mm->context.sigpage = (void *)addr;
423 
424 out:
425 	return PTR_ERR_OR_ZERO(ret);
426 }
427 
428 int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
429 {
430 	struct mm_struct *mm = current->mm;
431 	int ret;
432 
433 	if (mmap_write_lock_killable(mm))
434 		return -EINTR;
435 
436 	ret = aarch32_kuser_helpers_setup(mm);
437 	if (ret)
438 		goto out;
439 
440 #ifdef CONFIG_COMPAT_VDSO
441 	ret = __setup_additional_pages(VDSO_ABI_AA32,
442 				       mm,
443 				       bprm,
444 				       uses_interp);
445 	if (ret)
446 		goto out;
447 #endif /* CONFIG_COMPAT_VDSO */
448 
449 	ret = aarch32_sigreturn_setup(mm);
450 out:
451 	mmap_write_unlock(mm);
452 	return ret;
453 }
454 #endif /* CONFIG_COMPAT */
455 
456 static int vdso_mremap(const struct vm_special_mapping *sm,
457 		struct vm_area_struct *new_vma)
458 {
459 	return __vdso_remap(VDSO_ABI_AA64, sm, new_vma);
460 }
461 
462 enum aarch64_map {
463 	AA64_MAP_VVAR,
464 	AA64_MAP_VDSO,
465 };
466 
467 static struct vm_special_mapping aarch64_vdso_maps[] __ro_after_init = {
468 	[AA64_MAP_VVAR] = {
469 		.name	= "[vvar]",
470 		.fault = vvar_fault,
471 	},
472 	[AA64_MAP_VDSO] = {
473 		.name	= "[vdso]",
474 		.mremap = vdso_mremap,
475 	},
476 };
477 
478 static int __init vdso_init(void)
479 {
480 	vdso_info[VDSO_ABI_AA64].dm = &aarch64_vdso_maps[AA64_MAP_VVAR];
481 	vdso_info[VDSO_ABI_AA64].cm = &aarch64_vdso_maps[AA64_MAP_VDSO];
482 
483 	return __vdso_init(VDSO_ABI_AA64);
484 }
485 arch_initcall(vdso_init);
486 
487 int arch_setup_additional_pages(struct linux_binprm *bprm,
488 				int uses_interp)
489 {
490 	struct mm_struct *mm = current->mm;
491 	int ret;
492 
493 	if (mmap_write_lock_killable(mm))
494 		return -EINTR;
495 
496 	ret = __setup_additional_pages(VDSO_ABI_AA64,
497 				       mm,
498 				       bprm,
499 				       uses_interp);
500 
501 	mmap_write_unlock(mm);
502 
503 	return ret;
504 }
505