93 static int insert_links(struct ctl_table_header *head);
116 	struct ctl_table_header *head;  in find_entry()  local
129 		head = ctl_node->header;  in find_entry()
130 		entry = &head->ctl_table[ctl_node - head->node];  in find_entry()
139 			*phead = head;  in find_entry()
146 static int insert_entry(struct ctl_table_header *head, const struct ctl_table *entry)  in insert_entry()  argument
148 	struct rb_node *node = &head->node[entry - head->ctl_table].node;  in insert_entry()
149 	struct rb_node **p = &head->parent->root.rb_node;  in insert_entry()
174 			sysctl_print_dir(head->parent);  in insert_entry()
181 	rb_insert_color(node, &head->parent->root);  in insert_entry()
185 static void erase_entry(struct ctl_table_header *head, const struct ctl_table *entry)  in erase_entry()  argument
187 	struct rb_node *node = &head->node[entry - head->ctl_table].node;  in erase_entry()
189 	rb_erase(node, &head->parent->root);  in erase_entry()
192 static void init_header(struct ctl_table_header *head,  in init_header()  argument
196 	head->ctl_table = table;  in init_header()
197 	head->ctl_table_size = table_size;  in init_header()
198 	head->ctl_table_arg = table;  in init_header()
199 	head->used = 0;  in init_header()
200 	head->count = 1;  in init_header()
201 	head->nreg = 1;  in init_header()
202 	head->unregistering = NULL;  in init_header()
203 	head->root = root;  in init_header()
204 	head->set = set;  in init_header()
205 	head->parent = NULL;  in init_header()
206 	head->node = node;  in init_header()
207 	INIT_HLIST_HEAD(&head->inodes);  in init_header()
211 		list_for_each_table_entry(entry, head) {  in init_header()
212 			node->header = head;  in init_header()
217 		sysctl_set_perm_empty_ctl_header(head);  in init_header()
220 static void erase_header(struct ctl_table_header *head)  in erase_header()  argument
224 	list_for_each_table_entry(entry, head)  in erase_header()
225 		erase_entry(head, entry);  in erase_header()
287 static void proc_sys_invalidate_dcache(struct ctl_table_header *head)  in proc_sys_invalidate_dcache()  argument
289 	proc_invalidate_siblings_dcache(&head->inodes, &sysctl_lock);  in proc_sys_invalidate_dcache()
325 static struct ctl_table_header *sysctl_head_grab(struct ctl_table_header *head)  in sysctl_head_grab()  argument
327 	BUG_ON(!head);  in sysctl_head_grab()
329 	if (!use_table(head))  in sysctl_head_grab()
330 		head = ERR_PTR(-ENOENT);  in sysctl_head_grab()
332 	return head;  in sysctl_head_grab()
335 static void sysctl_head_finish(struct ctl_table_header *head)  in sysctl_head_finish()  argument
337 	if (!head)  in sysctl_head_finish()
340 	unuse_table(head);  in sysctl_head_finish()
357 	struct ctl_table_header *head;  in lookup_entry()  local
361 	entry = find_entry(&head, dir, name, namelen);  in lookup_entry()
362 	if (entry && use_table(head))  in lookup_entry()
363 		*phead = head;  in lookup_entry()
385 	struct ctl_table_header *head = NULL;  in first_entry()  local
393 		head = ctl_node->header;  in first_entry()
394 		entry = &head->ctl_table[ctl_node - head->node];  in first_entry()
396 	*phead = head;  in first_entry()
402 	struct ctl_table_header *head = *phead;  in next_entry()  local
404 	struct ctl_node *ctl_node = &head->node[entry - head->ctl_table];  in next_entry()
407 	unuse_table(head);  in next_entry()
411 	head = NULL;  in next_entry()
413 		head = ctl_node->header;  in next_entry()
414 		entry = &head->ctl_table[ctl_node - head->node];  in next_entry()
416 	*phead = head;  in next_entry()
436 static int sysctl_perm(struct ctl_table_header *head, const struct ctl_table *table, int op)  in sysctl_perm()  argument
438 	struct ctl_table_root *root = head->root;  in sysctl_perm()
442 		mode = root->permissions(head, table);  in sysctl_perm()
450 		struct ctl_table_header *head, const struct ctl_table *table)  in proc_sys_make_inode()  argument
452 	struct ctl_table_root *root = head->root;  in proc_sys_make_inode()
465 	if (unlikely(head->unregistering)) {  in proc_sys_make_inode()
470 	ei->sysctl = head;  in proc_sys_make_inode()
472 	hlist_add_head_rcu(&ei->sibling_inodes, &head->inodes);  in proc_sys_make_inode()
473 	head->count++;  in proc_sys_make_inode()
486 		if (sysctl_is_perm_empty_ctl_header(head))  in proc_sys_make_inode()
493 		root->set_ownership(head, &inode->i_uid, &inode->i_gid);  in proc_sys_make_inode()
498 void proc_sys_evict_inode(struct inode *inode, struct ctl_table_header *head)  in proc_sys_evict_inode()  argument
502 	if (!--head->count)  in proc_sys_evict_inode()
503 		kfree_rcu(head, rcu);  in proc_sys_evict_inode()
509 	struct ctl_table_header *head = PROC_I(inode)->sysctl;  in grab_header()  local
510 	if (!head)  in grab_header()
511 		head = &sysctl_table_root.default_set.dir.header;  in grab_header()
512 	return sysctl_head_grab(head);  in grab_header()
518 	struct ctl_table_header *head = grab_header(dir);  in proc_sys_lookup()  local
527 	if (IS_ERR(head))  in proc_sys_lookup()
528 		return ERR_CAST(head);  in proc_sys_lookup()
530 	ctl_dir = container_of(head, struct ctl_dir, header);  in proc_sys_lookup()
544 	inode = proc_sys_make_inode(dir->i_sb, h ? h : head, p);  in proc_sys_lookup()
550 	sysctl_head_finish(head);  in proc_sys_lookup()
558 	struct ctl_table_header *head = grab_header(inode);  in proc_sys_call_handler()  local
564 	if (IS_ERR(head))  in proc_sys_call_handler()
565 		return PTR_ERR(head);  in proc_sys_call_handler()
572 	if (sysctl_perm(head, table, write ? MAY_WRITE : MAY_READ))  in proc_sys_call_handler()
595 	error = BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, &kbuf, &count,  in proc_sys_call_handler()
615 	sysctl_head_finish(head);  in proc_sys_call_handler()
632 	struct ctl_table_header *head = grab_header(inode);  in proc_sys_open()  local
636 	if (IS_ERR(head))  in proc_sys_open()
637 		return PTR_ERR(head);  in proc_sys_open()
642 	sysctl_head_finish(head);  in proc_sys_open()
650 	struct ctl_table_header *head = grab_header(inode);  in proc_sys_poll()  local
656 	if (IS_ERR(head))  in proc_sys_poll()
674 	sysctl_head_finish(head);  in proc_sys_poll()
681 				struct ctl_table_header *head,  in proc_sys_fill_cache()  argument
703 			inode = proc_sys_make_inode(dir->d_sb, head, table);  in proc_sys_fill_cache()
725 				    struct ctl_table_header *head,  in proc_sys_link_fill_cache()  argument
730 	head = sysctl_head_grab(head);  in proc_sys_link_fill_cache()
731 	if (IS_ERR(head))  in proc_sys_link_fill_cache()
735 	if (sysctl_follow_link(&head, &table))  in proc_sys_link_fill_cache()
738 	ret = proc_sys_fill_cache(file, ctx, head, table);  in proc_sys_link_fill_cache()
740 	sysctl_head_finish(head);  in proc_sys_link_fill_cache()
744 static int scan(struct ctl_table_header *head, const struct ctl_table *table,  in scan()  argument
754 		res = proc_sys_link_fill_cache(file, ctx, head, table);  in scan()
756 		res = proc_sys_fill_cache(file, ctx, head, table);  in scan()
766 	struct ctl_table_header *head = grab_header(file_inode(file));  in proc_sys_readdir()  local
772 	if (IS_ERR(head))  in proc_sys_readdir()
773 		return PTR_ERR(head);  in proc_sys_readdir()
775 	ctl_dir = container_of(head, struct ctl_dir, header);  in proc_sys_readdir()
789 	sysctl_head_finish(head);  in proc_sys_readdir()
800 	struct ctl_table_header *head;  in proc_sys_permission()  local
808 	head = grab_header(inode);  in proc_sys_permission()
809 	if (IS_ERR(head))  in proc_sys_permission()
810 		return PTR_ERR(head);  in proc_sys_permission()
816 		error = sysctl_perm(head, table, mask & ~MAY_NOT_BLOCK);  in proc_sys_permission()
818 	sysctl_head_finish(head);  in proc_sys_permission()
844 	struct ctl_table_header *head = grab_header(inode);  in proc_sys_getattr()  local
847 	if (IS_ERR(head))  in proc_sys_getattr()
848 		return PTR_ERR(head);  in proc_sys_getattr()
854 	sysctl_head_finish(head);  in proc_sys_getattr()
918 	struct ctl_table_header *head;  in proc_sys_compare()  local
931 	head = rcu_dereference(PROC_I(inode)->sysctl);  in proc_sys_compare()
932 	return !head || !sysctl_is_seen(head);  in proc_sys_compare()
944 	struct ctl_table_header *head;  in find_subdir()  local
947 	entry = find_entry(&head, dir, name, namelen);  in find_subdir()
952 	return container_of(head, struct ctl_dir, header);  in find_subdir()
1058 	struct ctl_table_header *head;  in sysctl_follow_link()  local
1073 		head = NULL;  in sysctl_follow_link()
1074 		entry = find_entry(&head, dir, procname, strlen(procname));  in sysctl_follow_link()
1076 		if (entry && use_table(head)) {  in sysctl_follow_link()
1078 			*phead = head;  in sysctl_follow_link()
1177 static struct ctl_table_header *new_links(struct ctl_dir *dir, struct ctl_table_header *head)  in new_links()  argument
1187 	list_for_each_table_entry(entry, head) {  in new_links()
1192 			sizeof(struct ctl_node)*head->ctl_table_size +  in new_links()
1193 			sizeof(struct ctl_table)*head->ctl_table_size +  in new_links()
1201 	link_table = (struct ctl_table *)(node + head->ctl_table_size);  in new_links()
1202 	link_name = (char *)(link_table + head->ctl_table_size);  in new_links()
1205 	list_for_each_table_entry(entry, head) {  in new_links()
1210 		link->data = head->root;  in new_links()
1215 		    head->ctl_table_size);  in new_links()
1216 	links->nreg = head->ctl_table_size;  in new_links()
1254 static int insert_links(struct ctl_table_header *head)  in insert_links()  argument
1261 	if (head->set == root_set)  in insert_links()
1264 	core_parent = xlate_dir(root_set, head->parent);  in insert_links()
1268 	if (get_links(core_parent, head, head->root))  in insert_links()
1274 	links = new_links(core_parent, head);  in insert_links()
1282 	if (get_links(core_parent, head, head->root)) {  in insert_links()