xref: /linux-6.15/kernel/livepatch/core.c (revision d83a7cb3)
1 /*
2  * core.c - Kernel Live Patching Core
3  *
4  * Copyright (C) 2014 Seth Jennings <[email protected]>
5  * Copyright (C) 2014 SUSE
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License
9  * as published by the Free Software Foundation; either version 2
10  * of the License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, see <http://www.gnu.org/licenses/>.
19  */
20 
21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
22 
23 #include <linux/module.h>
24 #include <linux/kernel.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/list.h>
28 #include <linux/kallsyms.h>
29 #include <linux/livepatch.h>
30 #include <linux/elf.h>
31 #include <linux/moduleloader.h>
32 #include <asm/cacheflush.h>
33 #include "patch.h"
34 #include "transition.h"
35 
36 /*
37  * klp_mutex is a coarse lock which serializes access to klp data.  All
38  * accesses to klp-related variables and structures must have mutex protection,
39  * except within the following functions which carefully avoid the need for it:
40  *
41  * - klp_ftrace_handler()
42  * - klp_update_patch_state()
43  */
44 DEFINE_MUTEX(klp_mutex);
45 
46 static LIST_HEAD(klp_patches);
47 
48 static struct kobject *klp_root_kobj;
49 
50 static bool klp_is_module(struct klp_object *obj)
51 {
52 	return obj->name;
53 }
54 
55 static bool klp_is_object_loaded(struct klp_object *obj)
56 {
57 	return !obj->name || obj->mod;
58 }
59 
60 /* sets obj->mod if object is not vmlinux and module is found */
61 static void klp_find_object_module(struct klp_object *obj)
62 {
63 	struct module *mod;
64 
65 	if (!klp_is_module(obj))
66 		return;
67 
68 	mutex_lock(&module_mutex);
69 	/*
70 	 * We do not want to block removal of patched modules and therefore
71 	 * we do not take a reference here. The patches are removed by
72 	 * klp_module_going() instead.
73 	 */
74 	mod = find_module(obj->name);
75 	/*
76 	 * Do not mess work of klp_module_coming() and klp_module_going().
77 	 * Note that the patch might still be needed before klp_module_going()
78 	 * is called. Module functions can be called even in the GOING state
79 	 * until mod->exit() finishes. This is especially important for
80 	 * patches that modify semantic of the functions.
81 	 */
82 	if (mod && mod->klp_alive)
83 		obj->mod = mod;
84 
85 	mutex_unlock(&module_mutex);
86 }
87 
88 static bool klp_is_patch_registered(struct klp_patch *patch)
89 {
90 	struct klp_patch *mypatch;
91 
92 	list_for_each_entry(mypatch, &klp_patches, list)
93 		if (mypatch == patch)
94 			return true;
95 
96 	return false;
97 }
98 
99 static bool klp_initialized(void)
100 {
101 	return !!klp_root_kobj;
102 }
103 
104 struct klp_find_arg {
105 	const char *objname;
106 	const char *name;
107 	unsigned long addr;
108 	unsigned long count;
109 	unsigned long pos;
110 };
111 
112 static int klp_find_callback(void *data, const char *name,
113 			     struct module *mod, unsigned long addr)
114 {
115 	struct klp_find_arg *args = data;
116 
117 	if ((mod && !args->objname) || (!mod && args->objname))
118 		return 0;
119 
120 	if (strcmp(args->name, name))
121 		return 0;
122 
123 	if (args->objname && strcmp(args->objname, mod->name))
124 		return 0;
125 
126 	args->addr = addr;
127 	args->count++;
128 
129 	/*
130 	 * Finish the search when the symbol is found for the desired position
131 	 * or the position is not defined for a non-unique symbol.
132 	 */
133 	if ((args->pos && (args->count == args->pos)) ||
134 	    (!args->pos && (args->count > 1)))
135 		return 1;
136 
137 	return 0;
138 }
139 
140 static int klp_find_object_symbol(const char *objname, const char *name,
141 				  unsigned long sympos, unsigned long *addr)
142 {
143 	struct klp_find_arg args = {
144 		.objname = objname,
145 		.name = name,
146 		.addr = 0,
147 		.count = 0,
148 		.pos = sympos,
149 	};
150 
151 	mutex_lock(&module_mutex);
152 	kallsyms_on_each_symbol(klp_find_callback, &args);
153 	mutex_unlock(&module_mutex);
154 
155 	/*
156 	 * Ensure an address was found. If sympos is 0, ensure symbol is unique;
157 	 * otherwise ensure the symbol position count matches sympos.
158 	 */
159 	if (args.addr == 0)
160 		pr_err("symbol '%s' not found in symbol table\n", name);
161 	else if (args.count > 1 && sympos == 0) {
162 		pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n",
163 		       name, objname);
164 	} else if (sympos != args.count && sympos > 0) {
165 		pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n",
166 		       sympos, name, objname ? objname : "vmlinux");
167 	} else {
168 		*addr = args.addr;
169 		return 0;
170 	}
171 
172 	*addr = 0;
173 	return -EINVAL;
174 }
175 
176 static int klp_resolve_symbols(Elf_Shdr *relasec, struct module *pmod)
177 {
178 	int i, cnt, vmlinux, ret;
179 	char objname[MODULE_NAME_LEN];
180 	char symname[KSYM_NAME_LEN];
181 	char *strtab = pmod->core_kallsyms.strtab;
182 	Elf_Rela *relas;
183 	Elf_Sym *sym;
184 	unsigned long sympos, addr;
185 
186 	/*
187 	 * Since the field widths for objname and symname in the sscanf()
188 	 * call are hard-coded and correspond to MODULE_NAME_LEN and
189 	 * KSYM_NAME_LEN respectively, we must make sure that MODULE_NAME_LEN
190 	 * and KSYM_NAME_LEN have the values we expect them to have.
191 	 *
192 	 * Because the value of MODULE_NAME_LEN can differ among architectures,
193 	 * we use the smallest/strictest upper bound possible (56, based on
194 	 * the current definition of MODULE_NAME_LEN) to prevent overflows.
195 	 */
196 	BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);
197 
198 	relas = (Elf_Rela *) relasec->sh_addr;
199 	/* For each rela in this klp relocation section */
200 	for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {
201 		sym = pmod->core_kallsyms.symtab + ELF_R_SYM(relas[i].r_info);
202 		if (sym->st_shndx != SHN_LIVEPATCH) {
203 			pr_err("symbol %s is not marked as a livepatch symbol",
204 			       strtab + sym->st_name);
205 			return -EINVAL;
206 		}
207 
208 		/* Format: .klp.sym.objname.symname,sympos */
209 		cnt = sscanf(strtab + sym->st_name,
210 			     ".klp.sym.%55[^.].%127[^,],%lu",
211 			     objname, symname, &sympos);
212 		if (cnt != 3) {
213 			pr_err("symbol %s has an incorrectly formatted name",
214 			       strtab + sym->st_name);
215 			return -EINVAL;
216 		}
217 
218 		/* klp_find_object_symbol() treats a NULL objname as vmlinux */
219 		vmlinux = !strcmp(objname, "vmlinux");
220 		ret = klp_find_object_symbol(vmlinux ? NULL : objname,
221 					     symname, sympos, &addr);
222 		if (ret)
223 			return ret;
224 
225 		sym->st_value = addr;
226 	}
227 
228 	return 0;
229 }
230 
231 static int klp_write_object_relocations(struct module *pmod,
232 					struct klp_object *obj)
233 {
234 	int i, cnt, ret = 0;
235 	const char *objname, *secname;
236 	char sec_objname[MODULE_NAME_LEN];
237 	Elf_Shdr *sec;
238 
239 	if (WARN_ON(!klp_is_object_loaded(obj)))
240 		return -EINVAL;
241 
242 	objname = klp_is_module(obj) ? obj->name : "vmlinux";
243 
244 	/* For each klp relocation section */
245 	for (i = 1; i < pmod->klp_info->hdr.e_shnum; i++) {
246 		sec = pmod->klp_info->sechdrs + i;
247 		secname = pmod->klp_info->secstrings + sec->sh_name;
248 		if (!(sec->sh_flags & SHF_RELA_LIVEPATCH))
249 			continue;
250 
251 		/*
252 		 * Format: .klp.rela.sec_objname.section_name
253 		 * See comment in klp_resolve_symbols() for an explanation
254 		 * of the selected field width value.
255 		 */
256 		cnt = sscanf(secname, ".klp.rela.%55[^.]", sec_objname);
257 		if (cnt != 1) {
258 			pr_err("section %s has an incorrectly formatted name",
259 			       secname);
260 			ret = -EINVAL;
261 			break;
262 		}
263 
264 		if (strcmp(objname, sec_objname))
265 			continue;
266 
267 		ret = klp_resolve_symbols(sec, pmod);
268 		if (ret)
269 			break;
270 
271 		ret = apply_relocate_add(pmod->klp_info->sechdrs,
272 					 pmod->core_kallsyms.strtab,
273 					 pmod->klp_info->symndx, i, pmod);
274 		if (ret)
275 			break;
276 	}
277 
278 	return ret;
279 }
280 
281 static int __klp_disable_patch(struct klp_patch *patch)
282 {
283 	if (klp_transition_patch)
284 		return -EBUSY;
285 
286 	/* enforce stacking: only the last enabled patch can be disabled */
287 	if (!list_is_last(&patch->list, &klp_patches) &&
288 	    list_next_entry(patch, list)->enabled)
289 		return -EBUSY;
290 
291 	klp_init_transition(patch, KLP_UNPATCHED);
292 
293 	/*
294 	 * Enforce the order of the func->transition writes in
295 	 * klp_init_transition() and the TIF_PATCH_PENDING writes in
296 	 * klp_start_transition().  In the rare case where klp_ftrace_handler()
297 	 * is called shortly after klp_update_patch_state() switches the task,
298 	 * this ensures the handler sees that func->transition is set.
299 	 */
300 	smp_wmb();
301 
302 	klp_start_transition();
303 	klp_try_complete_transition();
304 	patch->enabled = false;
305 
306 	return 0;
307 }
308 
309 /**
310  * klp_disable_patch() - disables a registered patch
311  * @patch:	The registered, enabled patch to be disabled
312  *
313  * Unregisters the patched functions from ftrace.
314  *
315  * Return: 0 on success, otherwise error
316  */
317 int klp_disable_patch(struct klp_patch *patch)
318 {
319 	int ret;
320 
321 	mutex_lock(&klp_mutex);
322 
323 	if (!klp_is_patch_registered(patch)) {
324 		ret = -EINVAL;
325 		goto err;
326 	}
327 
328 	if (!patch->enabled) {
329 		ret = -EINVAL;
330 		goto err;
331 	}
332 
333 	ret = __klp_disable_patch(patch);
334 
335 err:
336 	mutex_unlock(&klp_mutex);
337 	return ret;
338 }
339 EXPORT_SYMBOL_GPL(klp_disable_patch);
340 
341 static int __klp_enable_patch(struct klp_patch *patch)
342 {
343 	struct klp_object *obj;
344 	int ret;
345 
346 	if (klp_transition_patch)
347 		return -EBUSY;
348 
349 	if (WARN_ON(patch->enabled))
350 		return -EINVAL;
351 
352 	/* enforce stacking: only the first disabled patch can be enabled */
353 	if (patch->list.prev != &klp_patches &&
354 	    !list_prev_entry(patch, list)->enabled)
355 		return -EBUSY;
356 
357 	pr_notice("enabling patch '%s'\n", patch->mod->name);
358 
359 	klp_init_transition(patch, KLP_PATCHED);
360 
361 	/*
362 	 * Enforce the order of the func->transition writes in
363 	 * klp_init_transition() and the ops->func_stack writes in
364 	 * klp_patch_object(), so that klp_ftrace_handler() will see the
365 	 * func->transition updates before the handler is registered and the
366 	 * new funcs become visible to the handler.
367 	 */
368 	smp_wmb();
369 
370 	klp_for_each_object(patch, obj) {
371 		if (!klp_is_object_loaded(obj))
372 			continue;
373 
374 		ret = klp_patch_object(obj);
375 		if (ret) {
376 			pr_warn("failed to enable patch '%s'\n",
377 				patch->mod->name);
378 
379 			klp_cancel_transition();
380 			return ret;
381 		}
382 	}
383 
384 	klp_start_transition();
385 	klp_try_complete_transition();
386 	patch->enabled = true;
387 
388 	return 0;
389 }
390 
391 /**
392  * klp_enable_patch() - enables a registered patch
393  * @patch:	The registered, disabled patch to be enabled
394  *
395  * Performs the needed symbol lookups and code relocations,
396  * then registers the patched functions with ftrace.
397  *
398  * Return: 0 on success, otherwise error
399  */
400 int klp_enable_patch(struct klp_patch *patch)
401 {
402 	int ret;
403 
404 	mutex_lock(&klp_mutex);
405 
406 	if (!klp_is_patch_registered(patch)) {
407 		ret = -EINVAL;
408 		goto err;
409 	}
410 
411 	ret = __klp_enable_patch(patch);
412 
413 err:
414 	mutex_unlock(&klp_mutex);
415 	return ret;
416 }
417 EXPORT_SYMBOL_GPL(klp_enable_patch);
418 
419 /*
420  * Sysfs Interface
421  *
422  * /sys/kernel/livepatch
423  * /sys/kernel/livepatch/<patch>
424  * /sys/kernel/livepatch/<patch>/enabled
425  * /sys/kernel/livepatch/<patch>/transition
426  * /sys/kernel/livepatch/<patch>/<object>
427  * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
428  */
429 
430 static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
431 			     const char *buf, size_t count)
432 {
433 	struct klp_patch *patch;
434 	int ret;
435 	bool enabled;
436 
437 	ret = kstrtobool(buf, &enabled);
438 	if (ret)
439 		return ret;
440 
441 	patch = container_of(kobj, struct klp_patch, kobj);
442 
443 	mutex_lock(&klp_mutex);
444 
445 	if (patch->enabled == enabled) {
446 		/* already in requested state */
447 		ret = -EINVAL;
448 		goto err;
449 	}
450 
451 	if (patch == klp_transition_patch) {
452 		klp_reverse_transition();
453 	} else if (enabled) {
454 		ret = __klp_enable_patch(patch);
455 		if (ret)
456 			goto err;
457 	} else {
458 		ret = __klp_disable_patch(patch);
459 		if (ret)
460 			goto err;
461 	}
462 
463 	mutex_unlock(&klp_mutex);
464 
465 	return count;
466 
467 err:
468 	mutex_unlock(&klp_mutex);
469 	return ret;
470 }
471 
472 static ssize_t enabled_show(struct kobject *kobj,
473 			    struct kobj_attribute *attr, char *buf)
474 {
475 	struct klp_patch *patch;
476 
477 	patch = container_of(kobj, struct klp_patch, kobj);
478 	return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);
479 }
480 
481 static ssize_t transition_show(struct kobject *kobj,
482 			       struct kobj_attribute *attr, char *buf)
483 {
484 	struct klp_patch *patch;
485 
486 	patch = container_of(kobj, struct klp_patch, kobj);
487 	return snprintf(buf, PAGE_SIZE-1, "%d\n",
488 			patch == klp_transition_patch);
489 }
490 
491 static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
492 static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition);
493 static struct attribute *klp_patch_attrs[] = {
494 	&enabled_kobj_attr.attr,
495 	&transition_kobj_attr.attr,
496 	NULL
497 };
498 
499 static void klp_kobj_release_patch(struct kobject *kobj)
500 {
501 	/*
502 	 * Once we have a consistency model we'll need to module_put() the
503 	 * patch module here.  See klp_register_patch() for more details.
504 	 */
505 }
506 
507 static struct kobj_type klp_ktype_patch = {
508 	.release = klp_kobj_release_patch,
509 	.sysfs_ops = &kobj_sysfs_ops,
510 	.default_attrs = klp_patch_attrs,
511 };
512 
513 static void klp_kobj_release_object(struct kobject *kobj)
514 {
515 }
516 
517 static struct kobj_type klp_ktype_object = {
518 	.release = klp_kobj_release_object,
519 	.sysfs_ops = &kobj_sysfs_ops,
520 };
521 
522 static void klp_kobj_release_func(struct kobject *kobj)
523 {
524 }
525 
526 static struct kobj_type klp_ktype_func = {
527 	.release = klp_kobj_release_func,
528 	.sysfs_ops = &kobj_sysfs_ops,
529 };
530 
531 /*
532  * Free all functions' kobjects in the array up to some limit. When limit is
533  * NULL, all kobjects are freed.
534  */
535 static void klp_free_funcs_limited(struct klp_object *obj,
536 				   struct klp_func *limit)
537 {
538 	struct klp_func *func;
539 
540 	for (func = obj->funcs; func->old_name && func != limit; func++)
541 		kobject_put(&func->kobj);
542 }
543 
544 /* Clean up when a patched object is unloaded */
545 static void klp_free_object_loaded(struct klp_object *obj)
546 {
547 	struct klp_func *func;
548 
549 	obj->mod = NULL;
550 
551 	klp_for_each_func(obj, func)
552 		func->old_addr = 0;
553 }
554 
555 /*
556  * Free all objects' kobjects in the array up to some limit. When limit is
557  * NULL, all kobjects are freed.
558  */
559 static void klp_free_objects_limited(struct klp_patch *patch,
560 				     struct klp_object *limit)
561 {
562 	struct klp_object *obj;
563 
564 	for (obj = patch->objs; obj->funcs && obj != limit; obj++) {
565 		klp_free_funcs_limited(obj, NULL);
566 		kobject_put(&obj->kobj);
567 	}
568 }
569 
570 static void klp_free_patch(struct klp_patch *patch)
571 {
572 	klp_free_objects_limited(patch, NULL);
573 	if (!list_empty(&patch->list))
574 		list_del(&patch->list);
575 	kobject_put(&patch->kobj);
576 }
577 
578 static int klp_init_func(struct klp_object *obj, struct klp_func *func)
579 {
580 	if (!func->old_name || !func->new_func)
581 		return -EINVAL;
582 
583 	INIT_LIST_HEAD(&func->stack_node);
584 	func->patched = false;
585 	func->transition = false;
586 
587 	/* The format for the sysfs directory is <function,sympos> where sympos
588 	 * is the nth occurrence of this symbol in kallsyms for the patched
589 	 * object. If the user selects 0 for old_sympos, then 1 will be used
590 	 * since a unique symbol will be the first occurrence.
591 	 */
592 	return kobject_init_and_add(&func->kobj, &klp_ktype_func,
593 				    &obj->kobj, "%s,%lu", func->old_name,
594 				    func->old_sympos ? func->old_sympos : 1);
595 }
596 
597 /* Arches may override this to finish any remaining arch-specific tasks */
598 void __weak arch_klp_init_object_loaded(struct klp_patch *patch,
599 					struct klp_object *obj)
600 {
601 }
602 
603 /* parts of the initialization that is done only when the object is loaded */
604 static int klp_init_object_loaded(struct klp_patch *patch,
605 				  struct klp_object *obj)
606 {
607 	struct klp_func *func;
608 	int ret;
609 
610 	module_disable_ro(patch->mod);
611 	ret = klp_write_object_relocations(patch->mod, obj);
612 	if (ret) {
613 		module_enable_ro(patch->mod, true);
614 		return ret;
615 	}
616 
617 	arch_klp_init_object_loaded(patch, obj);
618 	module_enable_ro(patch->mod, true);
619 
620 	klp_for_each_func(obj, func) {
621 		ret = klp_find_object_symbol(obj->name, func->old_name,
622 					     func->old_sympos,
623 					     &func->old_addr);
624 		if (ret)
625 			return ret;
626 
627 		ret = kallsyms_lookup_size_offset(func->old_addr,
628 						  &func->old_size, NULL);
629 		if (!ret) {
630 			pr_err("kallsyms size lookup failed for '%s'\n",
631 			       func->old_name);
632 			return -ENOENT;
633 		}
634 
635 		ret = kallsyms_lookup_size_offset((unsigned long)func->new_func,
636 						  &func->new_size, NULL);
637 		if (!ret) {
638 			pr_err("kallsyms size lookup failed for '%s' replacement\n",
639 			       func->old_name);
640 			return -ENOENT;
641 		}
642 	}
643 
644 	return 0;
645 }
646 
647 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
648 {
649 	struct klp_func *func;
650 	int ret;
651 	const char *name;
652 
653 	if (!obj->funcs)
654 		return -EINVAL;
655 
656 	obj->patched = false;
657 	obj->mod = NULL;
658 
659 	klp_find_object_module(obj);
660 
661 	name = klp_is_module(obj) ? obj->name : "vmlinux";
662 	ret = kobject_init_and_add(&obj->kobj, &klp_ktype_object,
663 				   &patch->kobj, "%s", name);
664 	if (ret)
665 		return ret;
666 
667 	klp_for_each_func(obj, func) {
668 		ret = klp_init_func(obj, func);
669 		if (ret)
670 			goto free;
671 	}
672 
673 	if (klp_is_object_loaded(obj)) {
674 		ret = klp_init_object_loaded(patch, obj);
675 		if (ret)
676 			goto free;
677 	}
678 
679 	return 0;
680 
681 free:
682 	klp_free_funcs_limited(obj, func);
683 	kobject_put(&obj->kobj);
684 	return ret;
685 }
686 
687 static int klp_init_patch(struct klp_patch *patch)
688 {
689 	struct klp_object *obj;
690 	int ret;
691 
692 	if (!patch->objs)
693 		return -EINVAL;
694 
695 	mutex_lock(&klp_mutex);
696 
697 	patch->enabled = false;
698 
699 	ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch,
700 				   klp_root_kobj, "%s", patch->mod->name);
701 	if (ret)
702 		goto unlock;
703 
704 	klp_for_each_object(patch, obj) {
705 		ret = klp_init_object(patch, obj);
706 		if (ret)
707 			goto free;
708 	}
709 
710 	list_add_tail(&patch->list, &klp_patches);
711 
712 	mutex_unlock(&klp_mutex);
713 
714 	return 0;
715 
716 free:
717 	klp_free_objects_limited(patch, obj);
718 	kobject_put(&patch->kobj);
719 unlock:
720 	mutex_unlock(&klp_mutex);
721 	return ret;
722 }
723 
724 /**
725  * klp_unregister_patch() - unregisters a patch
726  * @patch:	Disabled patch to be unregistered
727  *
728  * Frees the data structures and removes the sysfs interface.
729  *
730  * Return: 0 on success, otherwise error
731  */
732 int klp_unregister_patch(struct klp_patch *patch)
733 {
734 	int ret = 0;
735 
736 	mutex_lock(&klp_mutex);
737 
738 	if (!klp_is_patch_registered(patch)) {
739 		ret = -EINVAL;
740 		goto out;
741 	}
742 
743 	if (patch->enabled) {
744 		ret = -EBUSY;
745 		goto out;
746 	}
747 
748 	klp_free_patch(patch);
749 
750 out:
751 	mutex_unlock(&klp_mutex);
752 	return ret;
753 }
754 EXPORT_SYMBOL_GPL(klp_unregister_patch);
755 
756 /**
757  * klp_register_patch() - registers a patch
758  * @patch:	Patch to be registered
759  *
760  * Initializes the data structure associated with the patch and
761  * creates the sysfs interface.
762  *
763  * Return: 0 on success, otherwise error
764  */
765 int klp_register_patch(struct klp_patch *patch)
766 {
767 	int ret;
768 
769 	if (!patch || !patch->mod)
770 		return -EINVAL;
771 
772 	if (!is_livepatch_module(patch->mod)) {
773 		pr_err("module %s is not marked as a livepatch module",
774 		       patch->mod->name);
775 		return -EINVAL;
776 	}
777 
778 	if (!klp_initialized())
779 		return -ENODEV;
780 
781 	/*
782 	 * Architectures without reliable stack traces have to set
783 	 * patch->immediate because there's currently no way to patch kthreads
784 	 * with the consistency model.
785 	 */
786 	if (!klp_have_reliable_stack() && !patch->immediate) {
787 		pr_err("This architecture doesn't have support for the livepatch consistency model.\n");
788 		return -ENOSYS;
789 	}
790 
791 	/*
792 	 * A reference is taken on the patch module to prevent it from being
793 	 * unloaded.  Right now, we don't allow patch modules to unload since
794 	 * there is currently no method to determine if a thread is still
795 	 * running in the patched code contained in the patch module once
796 	 * the ftrace registration is successful.
797 	 */
798 	if (!try_module_get(patch->mod))
799 		return -ENODEV;
800 
801 	ret = klp_init_patch(patch);
802 	if (ret)
803 		module_put(patch->mod);
804 
805 	return ret;
806 }
807 EXPORT_SYMBOL_GPL(klp_register_patch);
808 
809 int klp_module_coming(struct module *mod)
810 {
811 	int ret;
812 	struct klp_patch *patch;
813 	struct klp_object *obj;
814 
815 	if (WARN_ON(mod->state != MODULE_STATE_COMING))
816 		return -EINVAL;
817 
818 	mutex_lock(&klp_mutex);
819 	/*
820 	 * Each module has to know that klp_module_coming()
821 	 * has been called. We never know what module will
822 	 * get patched by a new patch.
823 	 */
824 	mod->klp_alive = true;
825 
826 	list_for_each_entry(patch, &klp_patches, list) {
827 		klp_for_each_object(patch, obj) {
828 			if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
829 				continue;
830 
831 			obj->mod = mod;
832 
833 			ret = klp_init_object_loaded(patch, obj);
834 			if (ret) {
835 				pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n",
836 					patch->mod->name, obj->mod->name, ret);
837 				goto err;
838 			}
839 
840 			/*
841 			 * Only patch the module if the patch is enabled or is
842 			 * in transition.
843 			 */
844 			if (!patch->enabled && patch != klp_transition_patch)
845 				break;
846 
847 			pr_notice("applying patch '%s' to loading module '%s'\n",
848 				  patch->mod->name, obj->mod->name);
849 
850 			ret = klp_patch_object(obj);
851 			if (ret) {
852 				pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
853 					patch->mod->name, obj->mod->name, ret);
854 				goto err;
855 			}
856 
857 			break;
858 		}
859 	}
860 
861 	mutex_unlock(&klp_mutex);
862 
863 	return 0;
864 
865 err:
866 	/*
867 	 * If a patch is unsuccessfully applied, return
868 	 * error to the module loader.
869 	 */
870 	pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n",
871 		patch->mod->name, obj->mod->name, obj->mod->name);
872 	mod->klp_alive = false;
873 	klp_free_object_loaded(obj);
874 	mutex_unlock(&klp_mutex);
875 
876 	return ret;
877 }
878 
879 void klp_module_going(struct module *mod)
880 {
881 	struct klp_patch *patch;
882 	struct klp_object *obj;
883 
884 	if (WARN_ON(mod->state != MODULE_STATE_GOING &&
885 		    mod->state != MODULE_STATE_COMING))
886 		return;
887 
888 	mutex_lock(&klp_mutex);
889 	/*
890 	 * Each module has to know that klp_module_going()
891 	 * has been called. We never know what module will
892 	 * get patched by a new patch.
893 	 */
894 	mod->klp_alive = false;
895 
896 	list_for_each_entry(patch, &klp_patches, list) {
897 		klp_for_each_object(patch, obj) {
898 			if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
899 				continue;
900 
901 			/*
902 			 * Only unpatch the module if the patch is enabled or
903 			 * is in transition.
904 			 */
905 			if (patch->enabled || patch == klp_transition_patch) {
906 				pr_notice("reverting patch '%s' on unloading module '%s'\n",
907 					  patch->mod->name, obj->mod->name);
908 				klp_unpatch_object(obj);
909 			}
910 
911 			klp_free_object_loaded(obj);
912 			break;
913 		}
914 	}
915 
916 	mutex_unlock(&klp_mutex);
917 }
918 
919 static int __init klp_init(void)
920 {
921 	int ret;
922 
923 	ret = klp_check_compiler_support();
924 	if (ret) {
925 		pr_info("Your compiler is too old; turning off.\n");
926 		return -EINVAL;
927 	}
928 
929 	klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj);
930 	if (!klp_root_kobj)
931 		return -ENOMEM;
932 
933 	return 0;
934 }
935 
936 module_init(klp_init);
937