98 #define shm_unlock(shp)			\  argument
99 	ipc_unlock(&(shp)->shm_perm)
104 static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
125 	struct shmid_kernel *shp;  in do_shm_rmid()  local
127 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in do_shm_rmid()
128 	WARN_ON(ns != shp->ns);  in do_shm_rmid()
130 	if (shp->shm_nattch) {  in do_shm_rmid()
131 		shp->shm_perm.mode |= SHM_DEST;  in do_shm_rmid()
133 		ipc_set_key_private(&shm_ids(ns), &shp->shm_perm);  in do_shm_rmid()
134 		shm_unlock(shp);  in do_shm_rmid()
136 		shm_destroy(ns, shp);  in do_shm_rmid()
233 	struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,  in shm_rcu_free()  local
235 	security_shm_free(&shp->shm_perm);  in shm_rcu_free()
236 	kfree(shp);  in shm_rcu_free()
243 static inline void shm_clist_rm(struct shmid_kernel *shp)  in shm_clist_rm()  argument
254 	if (!list_empty(&shp->shm_clist)) {  in shm_clist_rm()
259 		creator = shp->shm_creator;  in shm_clist_rm()
266 		list_del_init(&shp->shm_clist);  in shm_clist_rm()
281 	struct shmid_kernel *shp;  in __shm_open()  local
283 	shp = shm_lock(sfd->ns, sfd->id);  in __shm_open()
285 	if (IS_ERR(shp))  in __shm_open()
286 		return PTR_ERR(shp);  in __shm_open()
288 	if (shp->shm_file != sfd->file) {  in __shm_open()
290 		shm_unlock(shp);  in __shm_open()
294 	shp->shm_atim = ktime_get_real_seconds();  in __shm_open()
295 	ipc_update_pid(&shp->shm_lprid, task_tgid(current));  in __shm_open()
296 	shp->shm_nattch++;  in __shm_open()
297 	shm_unlock(shp);  in __shm_open()
329 static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)  in shm_destroy()  argument
333 	shm_file = shp->shm_file;  in shm_destroy()
334 	shp->shm_file = NULL;  in shm_destroy()
335 	ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;  in shm_destroy()
336 	shm_rmid(shp);  in shm_destroy()
337 	shm_unlock(shp);  in shm_destroy()
339 		shmem_lock(shm_file, 0, shp->mlock_ucounts);  in shm_destroy()
341 	ipc_update_pid(&shp->shm_cprid, NULL);  in shm_destroy()
342 	ipc_update_pid(&shp->shm_lprid, NULL);  in shm_destroy()
343 	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);  in shm_destroy()
356 static bool shm_may_destroy(struct shmid_kernel *shp)  in shm_may_destroy()  argument
358 	return (shp->shm_nattch == 0) &&  in shm_may_destroy()
359 	       (shp->ns->shm_rmid_forced ||  in shm_may_destroy()
360 		(shp->shm_perm.mode & SHM_DEST));  in shm_may_destroy()
371 	struct shmid_kernel *shp;  in __shm_close()  local
376 	shp = shm_lock(ns, sfd->id);  in __shm_close()
382 	if (WARN_ON_ONCE(IS_ERR(shp)))  in __shm_close()
385 	ipc_update_pid(&shp->shm_lprid, task_tgid(current));  in __shm_close()
386 	shp->shm_dtim = ktime_get_real_seconds();  in __shm_close()
387 	shp->shm_nattch--;  in __shm_close()
388 	if (shm_may_destroy(shp))  in __shm_close()
389 		shm_destroy(ns, shp);  in __shm_close()
391 		shm_unlock(shp);  in __shm_close()
413 	struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in shm_try_destroy_orphaned()  local
421 	if (!list_empty(&shp->shm_clist))  in shm_try_destroy_orphaned()
424 	if (shm_may_destroy(shp)) {  in shm_try_destroy_orphaned()
425 		shm_lock_by_ptr(shp);  in shm_try_destroy_orphaned()
426 		shm_destroy(ns, shp);  in shm_try_destroy_orphaned()
443 		struct shmid_kernel *shp;  in exit_shm()  local
453 		shp = list_first_entry(&task->sysvshm.shm_clist, struct shmid_kernel,  in exit_shm()
463 		ns = shp->ns;  in exit_shm()
481 			list_del_init(&shp->shm_clist);  in exit_shm()
491 		WARN_ON(!ipc_rcu_getref(&shp->shm_perm));  in exit_shm()
500 		list_del_init(&shp->shm_clist);  in exit_shm()
509 		shm_lock_by_ptr(shp);  in exit_shm()
514 		ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);  in exit_shm()
516 		if (ipc_valid_object(&shp->shm_perm)) {  in exit_shm()
517 			if (shm_may_destroy(shp))  in exit_shm()
518 				shm_destroy(ns, shp);  in exit_shm()
520 				shm_unlock(shp);  in exit_shm()
527 			shm_unlock(shp);  in exit_shm()
703 	struct shmid_kernel *shp;  in newseg()  local
719 	shp = kmalloc(sizeof(*shp), GFP_KERNEL_ACCOUNT);  in newseg()
720 	if (unlikely(!shp))  in newseg()
723 	shp->shm_perm.key = key;  in newseg()
724 	shp->shm_perm.mode = (shmflg & S_IRWXUGO);  in newseg()
725 	shp->mlock_ucounts = NULL;  in newseg()
727 	shp->shm_perm.security = NULL;  in newseg()
728 	error = security_shm_alloc(&shp->shm_perm);  in newseg()
730 		kfree(shp);  in newseg()
765 	shp->shm_cprid = get_pid(task_tgid(current));  in newseg()
766 	shp->shm_lprid = NULL;  in newseg()
767 	shp->shm_atim = shp->shm_dtim = 0;  in newseg()
768 	shp->shm_ctim = ktime_get_real_seconds();  in newseg()
769 	shp->shm_segsz = size;  in newseg()
770 	shp->shm_nattch = 0;  in newseg()
771 	shp->shm_file = file;  in newseg()
772 	shp->shm_creator = current;  in newseg()
775 	error = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);  in newseg()
779 	shp->ns = ns;  in newseg()
782 	list_add(&shp->shm_clist, &current->sysvshm.shm_clist);  in newseg()
789 	file_inode(file)->i_ino = shp->shm_perm.id;  in newseg()
792 	error = shp->shm_perm.id;  in newseg()
794 	ipc_unlock_object(&shp->shm_perm);  in newseg()
799 	ipc_update_pid(&shp->shm_cprid, NULL);  in newseg()
800 	ipc_update_pid(&shp->shm_lprid, NULL);  in newseg()
802 	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);  in newseg()
805 	call_rcu(&shp->shm_perm.rcu, shm_rcu_free);  in newseg()
814 	struct shmid_kernel *shp;  in shm_more_checks()  local
816 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in shm_more_checks()
817 	if (shp->shm_segsz < params->u.size)  in shm_more_checks()
929 static void shm_add_rss_swap(struct shmid_kernel *shp,  in shm_add_rss_swap()  argument
934 	inode = file_inode(shp->shm_file);  in shm_add_rss_swap()
936 	if (is_file_hugepages(shp->shm_file)) {  in shm_add_rss_swap()
938 		struct hstate *h = hstate_file(shp->shm_file);  in shm_add_rss_swap()
970 		struct shmid_kernel *shp;  in shm_get_stat()  local
975 		shp = container_of(ipc, struct shmid_kernel, shm_perm);  in shm_get_stat()
977 		shm_add_rss_swap(shp, rss, swp);  in shm_get_stat()
992 	struct shmid_kernel *shp;  in shmctl_down()  local
1005 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in shmctl_down()
1007 	err = security_shm_shmctl(&shp->shm_perm, cmd);  in shmctl_down()
1013 		ipc_lock_object(&shp->shm_perm);  in shmctl_down()
1018 		ipc_lock_object(&shp->shm_perm);  in shmctl_down()
1022 		shp->shm_ctim = ktime_get_real_seconds();  in shmctl_down()
1030 	ipc_unlock_object(&shp->shm_perm);  in shmctl_down()
1080 	struct shmid_kernel *shp;  in shmctl_stat()  local
1087 		shp = shm_obtain_object(ns, shmid);  in shmctl_stat()
1088 		if (IS_ERR(shp)) {  in shmctl_stat()
1089 			err = PTR_ERR(shp);  in shmctl_stat()
1093 		shp = shm_obtain_object_check(ns, shmid);  in shmctl_stat()
1094 		if (IS_ERR(shp)) {  in shmctl_stat()
1095 			err = PTR_ERR(shp);  in shmctl_stat()
1108 		audit_ipc_obj(&shp->shm_perm);  in shmctl_stat()
1111 		if (ipcperms(ns, &shp->shm_perm, S_IRUGO))  in shmctl_stat()
1115 	err = security_shm_shmctl(&shp->shm_perm, cmd);  in shmctl_stat()
1119 	ipc_lock_object(&shp->shm_perm);  in shmctl_stat()
1121 	if (!ipc_valid_object(&shp->shm_perm)) {  in shmctl_stat()
1122 		ipc_unlock_object(&shp->shm_perm);  in shmctl_stat()
1127 	kernel_to_ipc64_perm(&shp->shm_perm, &tbuf->shm_perm);  in shmctl_stat()
1128 	tbuf->shm_segsz	= shp->shm_segsz;  in shmctl_stat()
1129 	tbuf->shm_atime	= shp->shm_atim;  in shmctl_stat()
1130 	tbuf->shm_dtime	= shp->shm_dtim;  in shmctl_stat()
1131 	tbuf->shm_ctime	= shp->shm_ctim;  in shmctl_stat()
1133 	tbuf->shm_atime_high = shp->shm_atim >> 32;  in shmctl_stat()
1134 	tbuf->shm_dtime_high = shp->shm_dtim >> 32;  in shmctl_stat()
1135 	tbuf->shm_ctime_high = shp->shm_ctim >> 32;  in shmctl_stat()
1137 	tbuf->shm_cpid	= pid_vnr(shp->shm_cprid);  in shmctl_stat()
1138 	tbuf->shm_lpid	= pid_vnr(shp->shm_lprid);  in shmctl_stat()
1139 	tbuf->shm_nattch = shp->shm_nattch;  in shmctl_stat()
1152 		err = shp->shm_perm.id;  in shmctl_stat()
1155 	ipc_unlock_object(&shp->shm_perm);  in shmctl_stat()
1163 	struct shmid_kernel *shp;  in shmctl_do_lock()  local
1168 	shp = shm_obtain_object_check(ns, shmid);  in shmctl_do_lock()
1169 	if (IS_ERR(shp)) {  in shmctl_do_lock()
1170 		err = PTR_ERR(shp);  in shmctl_do_lock()
1174 	audit_ipc_obj(&(shp->shm_perm));  in shmctl_do_lock()
1175 	err = security_shm_shmctl(&shp->shm_perm, cmd);  in shmctl_do_lock()
1179 	ipc_lock_object(&shp->shm_perm);  in shmctl_do_lock()
1182 	if (!ipc_valid_object(&shp->shm_perm)) {  in shmctl_do_lock()
1190 		if (!uid_eq(euid, shp->shm_perm.uid) &&  in shmctl_do_lock()
1191 		    !uid_eq(euid, shp->shm_perm.cuid)) {  in shmctl_do_lock()
1201 	shm_file = shp->shm_file;  in shmctl_do_lock()
1209 		if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {  in shmctl_do_lock()
1210 			shp->shm_perm.mode |= SHM_LOCKED;  in shmctl_do_lock()
1211 			shp->mlock_ucounts = ucounts;  in shmctl_do_lock()
1217 	if (!(shp->shm_perm.mode & SHM_LOCKED))  in shmctl_do_lock()
1219 	shmem_lock(shm_file, 0, shp->mlock_ucounts);  in shmctl_do_lock()
1220 	shp->shm_perm.mode &= ~SHM_LOCKED;  in shmctl_do_lock()
1221 	shp->mlock_ucounts = NULL;  in shmctl_do_lock()
1223 	ipc_unlock_object(&shp->shm_perm);  in shmctl_do_lock()
1231 	ipc_unlock_object(&shp->shm_perm);  in shmctl_do_lock()
1517 	struct shmid_kernel *shp;  in do_shmat()  local
1577 	shp = shm_obtain_object_check(ns, shmid);  in do_shmat()
1578 	if (IS_ERR(shp)) {  in do_shmat()
1579 		err = PTR_ERR(shp);  in do_shmat()
1584 	if (ipcperms(ns, &shp->shm_perm, acc_mode))  in do_shmat()
1587 	err = security_shm_shmat(&shp->shm_perm, shmaddr, shmflg);  in do_shmat()
1591 	ipc_lock_object(&shp->shm_perm);  in do_shmat()
1594 	if (!ipc_valid_object(&shp->shm_perm)) {  in do_shmat()
1595 		ipc_unlock_object(&shp->shm_perm);  in do_shmat()
1609 	base = get_file(shp->shm_file);  in do_shmat()
1610 	shp->shm_nattch++;  in do_shmat()
1612 	ipc_unlock_object(&shp->shm_perm);  in do_shmat()
1633 	sfd->id = shp->shm_perm.id;  in do_shmat()
1672 	shp = shm_lock(ns, shmid);  in do_shmat()
1673 	shp->shm_nattch--;  in do_shmat()
1675 	if (shm_may_destroy(shp))  in do_shmat()
1676 		shm_destroy(ns, shp);  in do_shmat()
1678 		shm_unlock(shp);  in do_shmat()
1840 	struct shmid_kernel *shp;  in sysvipc_shm_proc_show()  local
1843 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in sysvipc_shm_proc_show()
1844 	shm_add_rss_swap(shp, &rss, &swp);  in sysvipc_shm_proc_show()
1856 		   shp->shm_perm.key,  in sysvipc_shm_proc_show()
1857 		   shp->shm_perm.id,  in sysvipc_shm_proc_show()
1858 		   shp->shm_perm.mode,  in sysvipc_shm_proc_show()
1859 		   shp->shm_segsz,  in sysvipc_shm_proc_show()
1860 		   pid_nr_ns(shp->shm_cprid, pid_ns),  in sysvipc_shm_proc_show()
1861 		   pid_nr_ns(shp->shm_lprid, pid_ns),  in sysvipc_shm_proc_show()
1862 		   shp->shm_nattch,  in sysvipc_shm_proc_show()
1863 		   from_kuid_munged(user_ns, shp->shm_perm.uid),  in sysvipc_shm_proc_show()
1864 		   from_kgid_munged(user_ns, shp->shm_perm.gid),  in sysvipc_shm_proc_show()
1865 		   from_kuid_munged(user_ns, shp->shm_perm.cuid),  in sysvipc_shm_proc_show()
1866 		   from_kgid_munged(user_ns, shp->shm_perm.cgid),  in sysvipc_shm_proc_show()
1867 		   shp->shm_atim,  in sysvipc_shm_proc_show()
1868 		   shp->shm_dtim,  in sysvipc_shm_proc_show()
1869 		   shp->shm_ctim,  in sysvipc_shm_proc_show()